def __init__(self): self.cookieObj = CookieUtil() self.nowTile = time.strftime('%Y%m%d%H%M%S') self.nowPath = os.getcwd() if os.path.exists(self.nowPath + "/../log") == False: os.mkdir(self.nowPath + "/../log") if os.path.exists(self.nowPath + "/../result") == False: os.mkdir(self.nowPath + "/../result") self.resultIniPath = self.nowPath + "/../log/result_" + self.nowTile + ".ini" self.htmlPath = self.nowPath + "/../result/result_" + self.nowTile + ".html" self.check = Check() self.logUtil = LogUtil()
from logUtil import LogUtil from dictUtil import DictUtil import traceback import json def handleData(returnStr): jsonData=json.loads(returnStr) dateList=jsonData.get('date') dataList=jsonData.get('data1') return dateList,dataList def storeData(dateStr,dataStr,conn,cur,wdzjPlatId): sql='insert into platIncomeRate (date,incomeRate,wdzjPlatId) values ("'+dateStr+'","'+dataStr+'","'+wdzjPlatId+'")' cur.execute(sql) conn.commit() conn,cur=DatabaseUtil().getConn() session=SessionUtil() logUtil=LogUtil("platIncomeRate.log") cur.execute('select wdzjPlatId from platData where month="2017-06"') data=cur.fetchall() print(data) mylist=list() print(data) for i in range(0,len(data)): platId=str(data[i].get('wdzjPlatId')) mylist.append(platId) print mylist for i in mylist: url='http://shuju.wdzj.com/plat-info-target.html' try: param={
jsonData = json.loads(returnStr) dateList = jsonData.get('date') dataList1 = jsonData.get('data1') dataList2 = jsonData.get('data2') return dateList, dataList1, dataList2 def storeData(dateStr, dataStr1, dataStr2, conn, cur, wdzjPlatId): sql = 'insert into platInvMeanVsBorMean (date,invMean,borMean,wdzjPlatId) values ("' + dateStr + '","' + dataStr1 + '","' + dataStr2 + '","' + wdzjPlatId + '")' cur.execute(sql) conn.commit() conn, cur = DatabaseUtil().getConn() session = SessionUtil() logUtil = LogUtil("xinPlanInvest.log") cur.execute('select wdzjPlatId from platData where month="2017-06"') data = cur.fetchall() print(data) mylist = list() print(data) for i in range(0, len(data)): platId = str(data[i].get('wdzjPlatId')) mylist.append(platId) print mylist for i in mylist: url = 'http://shuju.wdzj.com/plat-info-target.html' try: param = {'wdzjPlatId': i, 'type': '1', 'target1': '7', 'target2': '8'}
import sys import traceback from databaseUtil import DatabaseUtil from sessionUtil import SessionUtil from htmlUtil import HtmlUtil from logUtil import LogUtil from dictUtil import DictUtil import json reload(sys) sys.setdefaultencoding('utf8') def storeData(htmlObject,conn,cur,logUtil,loanId): creditinfo=htmlObject.find('script',{'id':'credit-info-data'}).text jsonData=json.loads(creditinfo) jsonOne=jsonData.get('data') loanInfo=jsonOne.get('loan') beginBidTime=loanInfo.get('beginBidTime') sql='update RRDZhaiquanDetail set beginBidTime="'+beginBidTime+'" where loanId="'+loanId+'"' cur.execute(sql) conn.commit() logUtil.warning(loanId) session=SessionUtil() conn,cur=DatabaseUtil().getConn() logUtil=LogUtil("zhaiquanUpdate.log") for i in range(2029567,2209630): url="https://www.renrendai.com/loan/"+str(i) try: htmlObject=HtmlUtil(session.getReq(url)) storeData(htmlObject,conn,cur,logUtil,str(i)) except Exception,e: logUtil.warning(traceback.print_exc())
planList=jsonData.get('data').get('plans') return planList def storeData(jsonOne,conn,cur,logUtil,loanId): amount=jsonOne.get('amount') earnInterest=jsonOne.get('earnInterest') expectedYearRate=jsonOne.get('expectedYearRate') fundsuserRate=jsonOne.get('fundsUseRate') planId=jsonOne.get('id') name=jsonOne.get('name') status=jsonOne.get('status') subpointCountActual=jsonOne.get('subpointCountActual') sql='insert into RRDXinPlanList (amount,earnInterest,expectedYearRate,fundsUseRate,planId,name,status,subpointCountActual) values ("'+amount+'","'+earnInterest+'","'+expectedYearRate+'","'+fundsuserRate+'","'+planId+'","'+name+'","'+status+'","'+subpointCountActual+'")' print(sql) logUtil.warning(loanId) cur.execute(sql) conn.commit() session=SessionUtil() conn,cur=DatabaseUtil().getConn() logUtil=LogUtil("uplanList.log") for i in range(1,73): url='https://www.renrendai.com/autoinvestplan/listPlan!listPlanJson.action?pageIndex='+str(i)+'&_='+str(int(time.time())) try: planList=handleData(session.getReq(url)) for j in range(len(planList)): dictObject=DictUtil(planList[j]) storeData(dictObject,conn,cur,logUtil,str(i)) except Exception,e: logUtil.warning(traceback.print_exc()) cur.close() conn.close()
def storeData(jsonOne, conn, cur, planId): amount = jsonOne.get('amount') createTime = jsonOne.get('createTime') finalAmout = jsonOne.get('finalAmout') nickName = jsonOne.get('nickName') tradeMethod = jsonOne.get('tradeMethod') ucodeId = jsonOne.get('ucodeId') userId = jsonOne.get('userId') sql = 'insert into RRDPlanInvest (amount,createTime,finalAmout,nickName,tradeMethod,ucodeId,userId,planId) values ("' + amount + '","' + createTime + '","' + finalAmout + '","' + nickName + '","' + tradeMethod + '","' + ucodeId + '","' + userId + '","' + planId + '")' cur.execute(sql) conn.commit() conn, cur = DatabaseUtil().getConn() session = SessionUtil() logUtil = LogUtil("planInvest.log") for i in range(1, 13387): time.sleep(1) url = 'https://www.renrendai.com/financeplan/getFinancePlanLenders.action?financePlanStr=' + str( i) + '&_=' + str(int(time.time())) logUtil.warning(str(i)) try: data = session.getReq(url) investList = handleData(data) if len(investList) > 0: for j in range(len(investList)): dictObject = DictUtil(investList[j]) storeData(dictObject, conn, cur, str(i)) except Exception, e: logUtil.warning(traceback.print_exc())
lendTime = jsonOne.get('lendTime') lenderType = jsonOne.get('lenderType') loanId = jsonOne.get('loanId') orderNo = jsonOne.get('orderNo') traderMethod = jsonOne.get('tradeMethod') user = jsonOne.get('user') investorId = jsonOne.get('userId') userNickName = jsonOne.get('userNickName') sql = 'insert into RRDZhaiquanInvest (amount,autoCheckout,bussNo,financeCategory,financePlanId,financePlanSubPointId,investId,lendTime,lenderType,loanId,orderNo,tradeMethod,user,investorId,userNickName) values ("' + amount + '","' + autoCheckout + '","' + bussNo + '","' + financeCategory + '","' + financePlanId + '","' + financePlanSubPointId + '","' + investId + '","' + lendTime + '","' + lenderType + '","' + loanId + '","' + orderNo + '","' + traderMethod + '","' + user + '","' + investorId + '","' + userNickName + '")' cur.execute(sql) conn.commit() conn, cur = DatabaseUtil().getConn() session = SessionUtil() logUtil = LogUtil("zhaiquanInvest.log") for i in range(2208789, 2209630): url = 'https://www.renrendai.com/lend/getborrowerandlenderinfo.action?id=lenderRecords&loanId=' + str( i) + '&_=' + str(int(time.time())) logUtil.warning(str(i)) try: data = session.getReq(url) investList = handleData(data) if len(investList) > 0: for j in range(len(investList)): dictObject = DictUtil(investList[j]) storeData(dictObject, conn, cur) except Exception, e: logUtil.warning(traceback.print_exc()) cur.close()
'class': 'font-40px color-dark-text num-family' }).text) amount = str( htmlObject.find('span', { 'class': 'font-40px color-dark-text num-family ' }).text).strip() mylimit = str( htmlObject.find('span', { 'class': 'fn-left basic-value basic-value-new' }).find('em').text).strip() totalEarnings = htmlObject.find('i', { 'class': 'font-36px num-family' }).text lockEndTime = htmlObject.findAll('td')[5].text beginJoinTime = htmlObject.findAll('td')[8].text sql = 'insert into RRDPlanDetail (type,periods,interest,months,amount,mylimit,totalEarnings,lockEndTime,beginJoinTime,planId) values ("' + mytype + '","' + periods + '","' + interest + '","' + months + '","' + amount + '","' + mylimit + '","' + totalEarnings + '","' + lockEndTime + '","' + beginJoinTime + '","' + planId + '")' cur.execute(sql) conn.commit() session = SessionUtil() conn, cur = DatabaseUtil().getConn() logUtil = LogUtil("planDetail.log") for i in range(77, 13387): url = "https://www.renrendai.com/financeplan/" + str(i) try: htmlObject = HtmlUtil(session.getReq(url)) storeData(htmlObject, conn, cur, str(i), logUtil) except Exception, e: logUtil.warning(traceback.print_exc())
trustCapital = jsonOne.get('trustCapital') trustCreditor = jsonOne.get('trustCreditor') trustCreditorMonth = jsonOne.get('trustCreditorMonth') trustFunds = jsonOne.get('trustFunds') tzjPj = jsonOne.get('tzjPj') vipExpense = jsonOne.get('vipExpense') withTzj = jsonOne.get('withTzj') withdrawExpense = jsonOne.get('withdrawExpense') sql = 'insert into problemPlatDetail (actualCapital,aliasName,association,associationDetail,autoBid,autoBidCode,bankCapital,bankFunds,bidSecurity,bindingFlag,businessType,companyName,credit,creditLevel,delayScore,delayScoreDetail,displayFlg,drawScore,drawScoreDetail,equityVoList,experienceScore,experienceScoreDetail,fundCapital,gjlhhFlag,gjlhhTime,gruarantee,inspection,juridicalPerson,locationArea,locationAreaName,locationCity,locationCityName,manageExpense,manageExpenseDetail,newTrustCreditor,newTrustCreditorCode,officeAddress,onlineDate,payment,paymode,platBackground,platBackgroundDetail,platBackgroundDetailExpand,platBackgroundExpand,platEarnings,platEarningsCode,platName,platStatus,platUrl,problem,problemTime,recordId,recordLicId,registeredCapital,riskCapital,riskFunds,riskReserve,riskcontrol,securityModel,securityModelCode,securityModelOther,serviceScore,serviceScoreDetail,startInvestmentAmout,term,termCodes,termWeight,transferExpense,transferExpenseDetail,trustCapital,trustCreditor,trustCreditorMonth,trustFunds,tzjPj,vipExpense,withTzj,withdrawExpense,platId) values ("' + actualCapital + '","' + aliasName + '","' + association + '","' + associationDetail + '","' + autoBid + '","' + autoBidCode + '","' + bankCapital + '","' + bankFunds + '","' + bidSecurity + '","' + bindingFlag + '","' + businessType + '","' + companyName + '","' + credit + '","' + creditLevel + '","' + delayScore + '","' + delayScoreDetail + '","' + displayFlg + '","' + drawScore + '","' + drawScoreDetail + '","' + equityVoList + '","' + experienceScore + '","' + experienceScoreDetail + '","' + fundCapital + '","' + gjlhhFlag + '","' + gjlhhTime + '","' + gruarantee + '","' + inspection + '","' + juridicalPerson + '","' + locationArea + '","' + locationAreaName + '","' + locationCity + '","' + locationCityName + '","' + manageExpense + '","' + manageExpenseDetail + '","' + newTrustCreditor + '","' + newTrustCreditorCode + '","' + officeAddress + '","' + onlineDate + '","' + payment + '","' + paymode + '","' + platBackground + '","' + platBackgroundDetail + '","' + platBackgroundDetailExpand + '","' + platBackgroundExpand + '","' + platEarnings + '","' + platEarningsCode + '","' + platName + '","' + platStatus + '","' + platUrl + '","' + problem + '","' + problemTime + '","' + recordId + '","' + recordLicId + '","' + registeredCapital + '","' + riskCapital + '","' + riskFunds + '","' + riskReserve + '","' + riskcontrol + '","' + securityModel + '","' + securityModelCode + '","' + securityModelOther + '","' + serviceScore + '","' + serviceScoreDetail + '","' + startInvestmentAmout + '","' + term + '","' + termCodes + '","' + termWeight + '","' + transferExpense + '","' + transferExpenseDetail + '","' + trustCapital + '","' + trustCreditor + '","' + trustCreditorMonth + '","' + trustFunds + '","' + tzjPj + '","' + vipExpense + '","' + withTzj + '","' + withdrawExpense + '","' + platId + '")' cur.execute(sql) conn.commit() conn, cur = DatabaseUtil().getConn() session = SessionUtil() logUtil = LogUtil("problemPlatDetail.log") cur.execute('select platId from problemPlat') data = cur.fetchall() print(data) mylist = list() print(data) for i in range(0, len(data)): platId = str(data[i].get('platId')) mylist.append(platId) print mylist for i in mylist: url = 'http://wwwservice.wdzj.com/api/plat/platData30Days?platId=' + i try: data = session.getReq(url)