def __init__(self):
self.cookieObj = CookieUtil()
self.nowTile = time.strftime('%Y%m%d%H%M%S')
self.nowPath = os.getcwd()
if os.path.exists(self.nowPath + "/../log") == False:
os.mkdir(self.nowPath + "/../log")
if os.path.exists(self.nowPath + "/../result") == False:
os.mkdir(self.nowPath + "/../result")
self.resultIniPath = self.nowPath + "/../log/result_" + self.nowTile + ".ini"
self.htmlPath = self.nowPath + "/../result/result_" + self.nowTile + ".html"
self.check = Check()
self.logUtil = LogUtil()
def __init__(self):
self.cookieObj = CookieUtil()
self.nowTile = time.strftime("%Y%m%d%H%M%S")
self.nowPath = os.getcwd()
if os.path.exists(self.nowPath + "/../log") == False:
os.mkdir(self.nowPath + "/../log")
if os.path.exists(self.nowPath + "/../result") == False:
os.mkdir(self.nowPath + "/../result")
self.resultIniPath = self.nowPath + "/../log/result_" + self.nowTile + ".ini"
self.htmlPath = self.nowPath + "/../result/result_" + self.nowTile + ".html"
self.check = Check()
self.logUtil = LogUtil()
from logUtil import LogUtil
from dictUtil import DictUtil
import traceback
import json
def handleData(returnStr):
jsonData=json.loads(returnStr)
dateList=jsonData.get('date')
dataList=jsonData.get('data1')
return dateList,dataList
def storeData(dateStr,dataStr,conn,cur,wdzjPlatId):
sql='insert into platIncomeRate (date,incomeRate,wdzjPlatId) values ("'+dateStr+'","'+dataStr+'","'+wdzjPlatId+'")'
cur.execute(sql)
conn.commit()
conn,cur=DatabaseUtil().getConn()
session=SessionUtil()
logUtil=LogUtil("platIncomeRate.log")
cur.execute('select wdzjPlatId from platData where month="2017-06"')
data=cur.fetchall()
print(data)
mylist=list()
print(data)
for i in range(0,len(data)):
platId=str(data[i].get('wdzjPlatId'))
mylist.append(platId)
print mylist
for i in mylist:
url='http://shuju.wdzj.com/plat-info-target.html'
try:
param={
jsonData = json.loads(returnStr)
dateList = jsonData.get('date')
dataList1 = jsonData.get('data1')
dataList2 = jsonData.get('data2')
return dateList, dataList1, dataList2
def storeData(dateStr, dataStr1, dataStr2, conn, cur, wdzjPlatId):
sql = 'insert into platInvMeanVsBorMean (date,invMean,borMean,wdzjPlatId) values ("' + dateStr + '","' + dataStr1 + '","' + dataStr2 + '","' + wdzjPlatId + '")'
cur.execute(sql)
conn.commit()
conn, cur = DatabaseUtil().getConn()
session = SessionUtil()
logUtil = LogUtil("xinPlanInvest.log")
cur.execute('select wdzjPlatId from platData where month="2017-06"')
data = cur.fetchall()
print(data)
mylist = list()
print(data)
for i in range(0, len(data)):
platId = str(data[i].get('wdzjPlatId'))
mylist.append(platId)
print mylist
for i in mylist:
url = 'http://shuju.wdzj.com/plat-info-target.html'
try:
param = {'wdzjPlatId': i, 'type': '1', 'target1': '7', 'target2': '8'}
def storeData(jsonOne, conn, cur, planId):
amount = jsonOne.get('amount')
createTime = jsonOne.get('createTime')
finalAmout = jsonOne.get('finalAmout')
nickName = jsonOne.get('nickName')
tradeMethod = jsonOne.get('tradeMethod')
ucodeId = jsonOne.get('ucodeId')
userId = jsonOne.get('userId')
sql = 'insert into RRDXinPlanInvest (amount,createTime,finalAmout,nickName,tradeMethod,ucodeId,userId,planId) values ("' + amount + '","' + createTime + '","' + finalAmout + '","' + nickName + '","' + tradeMethod + '","' + ucodeId + '","' + userId + '","' + planId + '")'
cur.execute(sql)
conn.commit()
conn, cur = DatabaseUtil().getConn()
session = SessionUtil()
logUtil = LogUtil("xinPlanInvest.log")
cur.execute('select planId from RRDXinPlanList order by planId')
data = cur.fetchall()
for i in range(0, len(data)):
planId = str(data[i].get('planId'))
url = 'https://www.renrendai.com/autoinvestplan/listPlan!getAutoInvestPlanBuyerRecords.action?autoInvestPlanId=' + planId + '&_=' + str(
int(time.time()))
logUtil.warning(planId)
try:
data = session.getReq(url)
investList = handleData(data)
if len(investList) > 0:
for j in range(len(investList)):
dictObject = DictUtil(investList[j])
storeData(dictObject, conn, cur, planId)
except Exception, e:
import sys
import traceback
from databaseUtil import DatabaseUtil
from sessionUtil import SessionUtil
from htmlUtil import HtmlUtil
from logUtil import LogUtil
from dictUtil import DictUtil
import json
reload(sys)
sys.setdefaultencoding('utf8')
def storeData(htmlObject,conn,cur,logUtil,loanId):
creditinfo=htmlObject.find('script',{'id':'credit-info-data'}).text
jsonData=json.loads(creditinfo)
jsonOne=jsonData.get('data')
loanInfo=jsonOne.get('loan')
beginBidTime=loanInfo.get('beginBidTime')
sql='update RRDZhaiquanDetail set beginBidTime="'+beginBidTime+'" where loanId="'+loanId+'"'
cur.execute(sql)
conn.commit()
logUtil.warning(loanId)
session=SessionUtil()
conn,cur=DatabaseUtil().getConn()
logUtil=LogUtil("zhaiquanUpdate.log")
for i in range(2029567,2209630):
url="https://www.renrendai.com/loan/"+str(i)
try:
htmlObject=HtmlUtil(session.getReq(url))
storeData(htmlObject,conn,cur,logUtil,str(i))
except Exception,e:
logUtil.warning(traceback.print_exc())
planList=jsonData.get('data').get('plans')
return planList
def storeData(jsonOne,conn,cur,logUtil,loanId):
amount=jsonOne.get('amount')
earnInterest=jsonOne.get('earnInterest')
expectedYearRate=jsonOne.get('expectedYearRate')
fundsuserRate=jsonOne.get('fundsUseRate')
planId=jsonOne.get('id')
name=jsonOne.get('name')
status=jsonOne.get('status')
subpointCountActual=jsonOne.get('subpointCountActual')
sql='insert into RRDXinPlanList (amount,earnInterest,expectedYearRate,fundsUseRate,planId,name,status,subpointCountActual) values ("'+amount+'","'+earnInterest+'","'+expectedYearRate+'","'+fundsuserRate+'","'+planId+'","'+name+'","'+status+'","'+subpointCountActual+'")'
print(sql)
logUtil.warning(loanId)
cur.execute(sql)
conn.commit()
session=SessionUtil()
conn,cur=DatabaseUtil().getConn()
logUtil=LogUtil("uplanList.log")
for i in range(1,73):
url='https://www.renrendai.com/autoinvestplan/listPlan!listPlanJson.action?pageIndex='+str(i)+'&_='+str(int(time.time()))
try:
planList=handleData(session.getReq(url))
for j in range(len(planList)):
dictObject=DictUtil(planList[j])
storeData(dictObject,conn,cur,logUtil,str(i))
except Exception,e:
logUtil.warning(traceback.print_exc())
cur.close()
conn.close()
def storeData(jsonOne, conn, cur, planId):
amount = jsonOne.get('amount')
createTime = jsonOne.get('createTime')
finalAmout = jsonOne.get('finalAmout')
nickName = jsonOne.get('nickName')
tradeMethod = jsonOne.get('tradeMethod')
ucodeId = jsonOne.get('ucodeId')
userId = jsonOne.get('userId')
sql = 'insert into RRDPlanInvest (amount,createTime,finalAmout,nickName,tradeMethod,ucodeId,userId,planId) values ("' + amount + '","' + createTime + '","' + finalAmout + '","' + nickName + '","' + tradeMethod + '","' + ucodeId + '","' + userId + '","' + planId + '")'
cur.execute(sql)
conn.commit()
conn, cur = DatabaseUtil().getConn()
session = SessionUtil()
logUtil = LogUtil("planInvest.log")
for i in range(1, 13387):
time.sleep(1)
url = 'https://www.renrendai.com/financeplan/getFinancePlanLenders.action?financePlanStr=' + str(
i) + '&_=' + str(int(time.time()))
logUtil.warning(str(i))
try:
data = session.getReq(url)
investList = handleData(data)
if len(investList) > 0:
for j in range(len(investList)):
dictObject = DictUtil(investList[j])
storeData(dictObject, conn, cur, str(i))
except Exception, e:
logUtil.warning(traceback.print_exc())
class RunCode:
def __init__(self):
self.cookieObj = CookieUtil()
self.nowTile = time.strftime('%Y%m%d%H%M%S')
self.nowPath = os.getcwd()
if os.path.exists(self.nowPath + "/../log") == False:
os.mkdir(self.nowPath + "/../log")
if os.path.exists(self.nowPath + "/../result") == False:
os.mkdir(self.nowPath + "/../result")
self.resultIniPath = self.nowPath + "/../log/result_" + self.nowTile + ".ini"
self.htmlPath = self.nowPath + "/../result/result_" + self.nowTile + ".html"
self.check = Check()
self.logUtil = LogUtil()
def run(self):
checkKeyResult = self.check.checkIniKeys()
#key重复性校验失败
if checkKeyResult:
errorKeyPath = self.nowPath + "/../log/result_" + self.nowTile + "_error.ini"
self.logUtil.setKeyLog(errorKeyPath, checkKeyResult)
#校验成功
else:
checkUrlResult = self.check.checkIniUrl()
#url合法性校验失败
if checkUrlResult:
errorUrlPath = self.nowPath + "/../log/result_" + self.nowTile + "_error.ini"
self.logUtil.setUrlLog(errorUrlPath, checkUrlResult)
#校验成功
else:
self.iniObj = IniUtil(self.nowPath + '/../config/url.ini')
self.iniObjResult = IniUtil(self.resultIniPath)
cookie = self.getCookie()
self.doPost(cookie)
self.setCode()
self.setHtml()
print '执行完毕！！'
def getCookie(self):
cookie = self.cookieObj.getLoginCookie()
return cookie
def doPost(self, cookie):
#获取配置文件的url
keysList = []
urlDict = {}
keysList = self.iniObj.getKeysBySection('url')
for key in keysList:
pageCode = ''
url = ''
url = self.iniObj.getValue('url', key)
urlDict.setdefault(key, url)
#url = "http://kxyesit.cnsuning.com/"
try:
#创建请求的request
req = urllib2.Request(url)
handler = urllib2.HTTPCookieProcessor(cookie)
#利用urllib2的build_opener方法创建一个opener
openner = urllib2.build_opener(handler)
responseCode = openner.open(req).getcode()
#print '页面状态码为：' + str(responseCode)
pageCode = u'状态码:' + str(responseCode)
#iniObj.setValue(section='result',key=key,value=pageCode)
self.iniObjResult.setValue(section='result',
key=key,
value=pageCode)
except urllib2.URLError, e:
#print '错误原因：'+e.reason
reason = e.reason
pageCode = u'状态码:' + str(e.code) + u';错误原因:' + reason
#print pageCode
#iniObj.setValue(section='result',key=key,value=pageCode)
self.iniObjResult.setValue(section='result',
key=key,
value=pageCode)
class RunCode:
def __init__(self):
self.cookieObj = CookieUtil()
self.nowTile = time.strftime('%Y%m%d%H%M%S')
self.nowPath = os.getcwd()
if os.path.exists(self.nowPath + "/../log") == False:
os.mkdir(self.nowPath + "/../log")
if os.path.exists(self.nowPath + "/../result") == False:
os.mkdir(self.nowPath + "/../result")
self.resultIniPath = self.nowPath + "/../log/result_" + self.nowTile + ".ini"
self.errorLogPath = self.nowPath + "/../log/result_" + self.nowTile + "_error.ini"
self.htmlPath = self.nowPath + "/../result/result_" + self.nowTile + ".html"
self.check = Check()
self.logUtil = LogUtil()
def run(self):
checkKeyResult = self.check.checkIniKeys()
#key重复性校验失败
if checkKeyResult:
self.logUtil.setKeyLog(self.errorLogPath, checkKeyResult)
#校验成功
else:
checkUrlResult = self.check.checkIniUrl()
#url合法性校验失败
if checkUrlResult:
self.logUtil.setUrlLog(self.errorLogPath, checkUrlResult)
#校验成功
else:
self.iniObj = IniUtil(self.nowPath + '/../config/url.ini')
self.iniObjResult = IniUtil(self.resultIniPath)
cookie = self.getCookie()
#解析内容
self.getContent(cookie)
#self.doPost(cookie)
#self.setCode()
#self.setHtml()
print '执行完毕！！'
def getCookie(self):
cookie = self.cookieObj.getLoginCookie()
return cookie
def getContent(self, cookie):
#获取配置文件的url
keysList = []
urlDict = {}
keysList = self.iniObj.getKeysBySection('url')
num = 1
excelUtilObj = ExcelUtil()
for key in keysList:
url = self.iniObj.getValue('url', key)
urlDict.setdefault(key, url)
try:
#创建请求的request
req = urllib2.Request(url)
handler = urllib2.HTTPCookieProcessor(cookie)
#利用urllib2的build_opener方法创建一个opener
openner = urllib2.build_opener(handler)
response = openner.open(req)
content = response.read()
htmlLabel = excelUtilObj.htmlLabel
worksheet = excelUtilObj.addSheet(key)
excelUtilObj.setKeys(worksheet)
self.setExcel(worksheet, num, excelUtilObj, content, htmlLabel)
num += 1
except urllib2.URLError, e:
reason = e.reason
pageCode = u'状态码:' + str(e.code) + u';错误原因:' + reason
self.logUtil.setPostErrorLog(self.errorLogPath, url, pageCode)
excelUtilObj.closeExcel()
'class': 'font-40px color-dark-text num-family'
}).text)
amount = str(
htmlObject.find('span', {
'class': 'font-40px color-dark-text num-family '
}).text).strip()
mylimit = str(
htmlObject.find('span', {
'class': 'fn-left basic-value basic-value-new'
}).find('em').text).strip()
totalEarnings = htmlObject.find('i', {
'class': 'font-36px num-family'
}).text
lockEndTime = htmlObject.findAll('td')[5].text
beginJoinTime = htmlObject.findAll('td')[8].text
sql = 'insert into RRDPlanDetail (type,periods,interest,months,amount,mylimit,totalEarnings,lockEndTime,beginJoinTime,planId) values ("' + mytype + '","' + periods + '","' + interest + '","' + months + '","' + amount + '","' + mylimit + '","' + totalEarnings + '","' + lockEndTime + '","' + beginJoinTime + '","' + planId + '")'
cur.execute(sql)
conn.commit()
session = SessionUtil()
conn, cur = DatabaseUtil().getConn()
logUtil = LogUtil("planDetail.log")
for i in range(77, 13387):
url = "https://www.renrendai.com/financeplan/" + str(i)
try:
htmlObject = HtmlUtil(session.getReq(url))
storeData(htmlObject, conn, cur, str(i), logUtil)
except Exception, e:
logUtil.warning(traceback.print_exc())
class RunCode:
def __init__(self):
self.cookieObj = CookieUtil()
self.nowTile = time.strftime("%Y%m%d%H%M%S")
self.nowPath = os.getcwd()
if os.path.exists(self.nowPath + "/../log") == False:
os.mkdir(self.nowPath + "/../log")
if os.path.exists(self.nowPath + "/../result") == False:
os.mkdir(self.nowPath + "/../result")
self.resultIniPath = self.nowPath + "/../log/result_" + self.nowTile + ".ini"
self.errorLogPath = self.nowPath + "/../log/result_" + self.nowTile + "_error.ini"
self.htmlPath = self.nowPath + "/../result/result_" + self.nowTile + ".html"
self.check = Check()
self.logUtil = LogUtil()
def run(self):
checkKeyResult = self.check.checkIniKeys()
# key重复性校验失败
if checkKeyResult:
self.logUtil.setKeyLog(self.errorLogPath, checkKeyResult)
# 校验成功
else:
checkUrlResult = self.check.checkIniUrl()
# url合法性校验失败
if checkUrlResult:
self.logUtil.setUrlLog(self.errorLogPath, checkUrlResult)
# 校验成功
else:
self.iniObj = IniUtil(self.nowPath + "/../config/url.ini")
self.iniObjResult = IniUtil(self.resultIniPath)
cookie = self.getCookie()
# 解析内容
self.getContent(cookie)
# self.doPost(cookie)
# self.setCode()
# self.setHtml()
print "执行完毕！！"
def getCookie(self):
cookie = self.cookieObj.getLoginCookie()
return cookie
def getContent(self, cookie):
# 获取配置文件的url
keysList = []
urlDict = {}
keysList = self.iniObj.getKeysBySection("url")
num = 1
excelUtilObj = ExcelUtil()
for key in keysList:
url = self.iniObj.getValue("url", key)
urlDict.setdefault(key, url)
try:
# 创建请求的request
req = urllib2.Request(url)
handler = urllib2.HTTPCookieProcessor(cookie)
# 利用urllib2的build_opener方法创建一个opener
openner = urllib2.build_opener(handler)
response = openner.open(req)
content = response.read()
htmlLabel = excelUtilObj.htmlLabel
worksheet = excelUtilObj.addSheet(key)
excelUtilObj.setKeys(worksheet)
self.setExcel(worksheet, num, excelUtilObj, content, htmlLabel)
num += 1
except urllib2.URLError, e:
reason = e.reason
pageCode = u"状态码:" + str(e.code) + u";错误原因:" + reason
self.logUtil.setPostErrorLog(self.errorLogPath, url, pageCode)
excelUtilObj.closeExcel()
trustCapital = jsonOne.get('trustCapital')
trustCreditor = jsonOne.get('trustCreditor')
trustCreditorMonth = jsonOne.get('trustCreditorMonth')
trustFunds = jsonOne.get('trustFunds')
tzjPj = jsonOne.get('tzjPj')
vipExpense = jsonOne.get('vipExpense')
withTzj = jsonOne.get('withTzj')
withdrawExpense = jsonOne.get('withdrawExpense')
sql = 'insert into problemPlatDetail (actualCapital,aliasName,association,associationDetail,autoBid,autoBidCode,bankCapital,bankFunds,bidSecurity,bindingFlag,businessType,companyName,credit,creditLevel,delayScore,delayScoreDetail,displayFlg,drawScore,drawScoreDetail,equityVoList,experienceScore,experienceScoreDetail,fundCapital,gjlhhFlag,gjlhhTime,gruarantee,inspection,juridicalPerson,locationArea,locationAreaName,locationCity,locationCityName,manageExpense,manageExpenseDetail,newTrustCreditor,newTrustCreditorCode,officeAddress,onlineDate,payment,paymode,platBackground,platBackgroundDetail,platBackgroundDetailExpand,platBackgroundExpand,platEarnings,platEarningsCode,platName,platStatus,platUrl,problem,problemTime,recordId,recordLicId,registeredCapital,riskCapital,riskFunds,riskReserve,riskcontrol,securityModel,securityModelCode,securityModelOther,serviceScore,serviceScoreDetail,startInvestmentAmout,term,termCodes,termWeight,transferExpense,transferExpenseDetail,trustCapital,trustCreditor,trustCreditorMonth,trustFunds,tzjPj,vipExpense,withTzj,withdrawExpense,platId) values ("' + actualCapital + '","' + aliasName + '","' + association + '","' + associationDetail + '","' + autoBid + '","' + autoBidCode + '","' + bankCapital + '","' + bankFunds + '","' + bidSecurity + '","' + bindingFlag + '","' + businessType + '","' + companyName + '","' + credit + '","' + creditLevel + '","' + delayScore + '","' + delayScoreDetail + '","' + displayFlg + '","' + drawScore + '","' + drawScoreDetail + '","' + equityVoList + '","' + experienceScore + '","' + experienceScoreDetail + '","' + fundCapital + '","' + gjlhhFlag + '","' + gjlhhTime + '","' + gruarantee + '","' + inspection + '","' + juridicalPerson + '","' + locationArea + '","' + locationAreaName + '","' + locationCity + '","' + locationCityName + '","' + manageExpense + '","' + manageExpenseDetail + '","' + newTrustCreditor + '","' + newTrustCreditorCode + '","' + officeAddress + '","' + onlineDate + '","' + payment + '","' + paymode + '","' + platBackground + '","' + platBackgroundDetail + '","' + platBackgroundDetailExpand + '","' + platBackgroundExpand + '","' + platEarnings + '","' + platEarningsCode + '","' + platName + '","' + platStatus + '","' + platUrl + '","' + problem + '","' + problemTime + '","' + recordId + '","' + recordLicId + '","' + registeredCapital + '","' + riskCapital + '","' + riskFunds + '","' + riskReserve + '","' + riskcontrol + '","' + securityModel + '","' + securityModelCode + '","' + securityModelOther + '","' + serviceScore + '","' + serviceScoreDetail + '","' + startInvestmentAmout + '","' + term + '","' + termCodes + '","' + termWeight + '","' + transferExpense + '","' + transferExpenseDetail + '","' + trustCapital + '","' + trustCreditor + '","' + trustCreditorMonth + '","' + trustFunds + '","' + tzjPj + '","' + vipExpense + '","' + withTzj + '","' + withdrawExpense + '","' + platId + '")'
cur.execute(sql)
conn.commit()
conn, cur = DatabaseUtil().getConn()
session = SessionUtil()
logUtil = LogUtil("problemPlatDetail.log")
cur.execute('select platId from problemPlat')
data = cur.fetchall()
print(data)
mylist = list()
print(data)
for i in range(0, len(data)):
platId = str(data[i].get('platId'))
mylist.append(platId)
print mylist
for i in mylist:
url = 'http://wwwservice.wdzj.com/api/plat/platData30Days?platId=' + i
try:
data = session.getReq(url)
class RunCode:
def __init__(self):
self.cookieObj = CookieUtil()
self.nowTile = time.strftime("%Y%m%d%H%M%S")
self.nowPath = os.getcwd()
if os.path.exists(self.nowPath + "/../log") == False:
os.mkdir(self.nowPath + "/../log")
if os.path.exists(self.nowPath + "/../result") == False:
os.mkdir(self.nowPath + "/../result")
self.resultIniPath = self.nowPath + "/../log/result_" + self.nowTile + ".ini"
self.htmlPath = self.nowPath + "/../result/result_" + self.nowTile + ".html"
self.check = Check()
self.logUtil = LogUtil()
def run(self):
checkKeyResult = self.check.checkIniKeys()
# key重复性校验失败
if checkKeyResult:
errorKeyPath = self.nowPath + "/../log/result_" + self.nowTile + "_error.ini"
self.logUtil.setKeyLog(errorKeyPath, checkKeyResult)
# 校验成功
else:
checkUrlResult = self.check.checkIniUrl()
# url合法性校验失败
if checkUrlResult:
errorUrlPath = self.nowPath + "/../log/result_" + self.nowTile + "_error.ini"
self.logUtil.setUrlLog(errorUrlPath, checkUrlResult)
# 校验成功
else:
self.iniObj = IniUtil(self.nowPath + "/../config/url.ini")
self.iniObjResult = IniUtil(self.resultIniPath)
cookie = self.getCookie()
self.doPost(cookie)
self.setCode()
self.setHtml()
print "执行完毕！！"
def getCookie(self):
cookie = self.cookieObj.getLoginCookie()
return cookie
def doPost(self, cookie):
# 获取配置文件的url
keysList = []
urlDict = {}
keysList = self.iniObj.getKeysBySection("url")
for key in keysList:
pageCode = ""
url = ""
url = self.iniObj.getValue("url", key)
urlDict.setdefault(key, url)
# url = "http://kxyesit.cnsuning.com/"
try:
# 创建请求的request
req = urllib2.Request(url)
handler = urllib2.HTTPCookieProcessor(cookie)
# 利用urllib2的build_opener方法创建一个opener
openner = urllib2.build_opener(handler)
responseCode = openner.open(req).getcode()
# print '页面状态码为：' + str(responseCode)
pageCode = u"状态码:" + str(responseCode)
# iniObj.setValue(section='result',key=key,value=pageCode)
self.iniObjResult.setValue(section="result", key=key, value=pageCode)
except urllib2.URLError, e:
# print '错误原因：'+e.reason
reason = e.reason
pageCode = u"状态码:" + str(e.code) + u";错误原因:" + reason
# print pageCode
# iniObj.setValue(section='result',key=key,value=pageCode)
self.iniObjResult.setValue(section="result", key=key, value=pageCode)
