def step3_bid(self, opener, maturity, loanid, money):
loanurl = "http://invest.ppdai.com/bid/info?source=2&listingId=%d" % (
loanid) + '%20%20%20%20&title=&date=' + "%d" % (
maturity) + '%20%20%20%20&UrlReferrer=1&money=' + "%d" % (
money)
bidurl = "http://invest.ppdai.com/Bid/Bid"
bid_info = {
"Reason": '',
"Amount": money,
"ListingId": loanid,
"UrlReferrer": "1",
"SubListType": '0'
}
headers = {
"Origin": "http://invest.ppdai.com",
"Accept-Encoding": "gzip, deflate",
"Host": "invest.ppdai.com",
"Accept-Language": "zh-CN,zh;q=0.8,en;q=0.6",
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "*/*",
"Referer": loanurl,
"X-Requested-With": "XMLHttpRequest",
"Connection": "keep-alive",
"Content-Length": "63"
}
upost_data = urllib.urlencode(bid_info)
opener = PPBaoUtil.add_headers(opener, headers)
response = opener.open(bidurl, upost_data, 15)
html = PPBaoUtil.get_html_from_response(response)
#logging.debug("Get Response: %s", html)
response_headers = response.info()
for head in response_headers:
logging.debug("%s:%s" % (head, response_headers[head]))
def step1_open_actionlog(opener, loanid):
url = "http://invest.ppdai.com/common/actionlog"
headers = {"Origin":"http://invest.ppdai.com",
"Accept-Encoding": "gzip, deflate",
"Host": "invest.ppdai.com",
"Accept-Language": "zh-CN,zh;q=0.8,en;q=0.6",
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "*/*",
"Referer": "http://invest.ppdai.com/loan/info?id=%d" % (loanid),
"X-Requested-With": "XMLHttpRequest",
"Connection": "keep-alive",
"Content-Length": "109"
}
post_data = {"useraction":"VisitRecord",
"functionname":'%E6%95%A3%E6%A0%87%E6%8A%95%E6%A0%87',
"remark":'%E6%95%A3%E6%A0%87'+"%d+++++" %(loanid)
}
opener = PPBaoUtil.add_headers(opener, headers)
upost_data = urllib.urlencode(post_data)
response = opener.open(url, upost_data, 10)
html = PPBaoUtil.get_html_from_response(response)
logging.debug("Get Response: %s", html)
response_headers = response.info()
for head in response_headers:
logging.info("%s:%s" % (head, response_headers[head]))
def open_blacklist_page(self, page):
try:
if (page > 1):
url = "http://invest.ppdai.com/account/blacklist?PageIndex=%d&IsCalendarRequest=0" % (
page)
refid = page - 1
referer = "http://invest.ppdai.com/account/blacklist?PageIndex=%d&IsCalendarRequest=0" % (
refid)
else:
url = "http://invest.ppdai.com/account/blacklist?UserName=&LateDayTo=1&LateDayFrom=&ListingTitle="
referer = "http://invest.ppdai.com/account/lend"
headers = {
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Upgrade-Insecure-Requests': '1',
'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6',
'Host': 'invest.ppdai.com',
'Accept-Encoding': 'gzip, deflate, sdch',
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36',
'Connection': 'keep-alive'
}
opener = PPBaoUtil.add_headers(self.ppdspider.opener, headers)
response = opener.open(url, None, 15)
html_str = PPBaoUtil.get_html_from_response(response)
response.close()
return html_str
except urllib2.URLError, e:
logging.error("Not able to open %s, %r" % (self.blacklist_url, e))
return None
def init_ppbao(ppbao_config_file, ppbaouserdao):
# Initialize
ppbao_config = PPBaoConfig(ppbao_config_file)
ppdloginid, dbhost, dbuser, dbpwd, dbname = ppbao_config.read_ppbao_config(
)
PPBaoUtil.init_logging(ppdloginid, ppbao_config.logdir)
logging.info("Welcome to PPBao System - Update BlackList Utility!")
logging.info("Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017")
logging.info("PPBao Config: %s,%s,%s,%s,%s" %
(ppdloginid, dbhost, dbuser, dbpwd, dbname))
(ppduserid_db,
ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error(
"Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!"
% (ppdloginid))
exit(2)
# Login to PPDAI!
spider = PPDSpider(ppdloginid, ppbao_config)
(opener, ppduserid) = spider.login(ppdloginid, ppdpasswd)
if (ppduserid == None or ppduserid != ppduserid_db):
logging.error(
"Error: Not able to get PPDAI Username or is not consistent with that in DB! Exit..."
)
exit(4)
return (ppduserid, spider)
def open_blacklist_page(self, page):
try:
if (page > 1):
url = "http://invest.ppdai.com/account/blacklist?PageIndex=%d&IsCalendarRequest=0" % (page)
refid = page - 1
referer = "http://invest.ppdai.com/account/blacklist?PageIndex=%d&IsCalendarRequest=0" % (refid)
else:
url = "http://invest.ppdai.com/account/blacklist?UserName=&LateDayTo=1&LateDayFrom=&ListingTitle="
referer = "http://invest.ppdai.com/account/lend"
headers = {
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Upgrade-Insecure-Requests':'1',
'Accept-Language':'zh-CN,zh;q=0.8,en;q=0.6',
'Host':'invest.ppdai.com',
'Accept-Encoding':'gzip, deflate, sdch',
'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36',
'Connection':'keep-alive'
}
opener = PPBaoUtil.add_headers(self.ppdspider.opener, headers)
response = opener.open(url,None,15)
html_str = PPBaoUtil.get_html_from_response(response)
response.close()
return html_str;
except urllib2.URLError, e:
logging.error("Not able to open %s, %r" % (self.blacklist_url, e))
return None
def init_ppbao(argv):
ppbao_config_file = None
if (len(argv) == 1):
ppbao_config_file = "conf/ppbao.18616856236.config"
ppbao_config_file = "conf/ppbao.18616027065.config"
elif (len(argv) == 2):
me, ppbao_config_file = argv
else:
print "Error: More than 1 argument is provided!"
print "Usage: python update_blacklist.py <ppbao_config_file>"
exit(-1)
# Initialize
ppbao_config = PPBaoConfig(ppbao_config_file)
ppdloginid, dbhost, dbuser, dbpwd, dbname = ppbao_config.read_ppbao_config(
)
PPBaoUtil.init_logging(ppdloginid, ppbao_config.logdir)
logging.info("Welcome to PPBao System - Update BlackList Utility!")
logging.info("Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017")
logging.info("PPBao Config: %s,%s,%s,%s,%s" %
(ppdloginid, dbhost, dbuser, dbpwd, dbname))
# Init DB Modules
ppddao = PPDDAO({
'host': dbhost,
'username': dbuser,
'password': dbpwd,
'database': dbname
})
dbok = ppddao.connect()
if dbok == False:
logging.error(
"Error: Not able to connect to MySQL! Please Fix it. Exiting now")
exit(1)
ppbaouserdao = PPBaoUserDAO(ppddao)
blacklistdao = BlackListDAO(ppddao)
(ppduserid_db,
ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error(
"Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!"
% (ppdloginid))
exit(2)
# Login to PPDAI!
spider = PPDSpider(ppdloginid, ppbao_config)
(opener, ppduserid) = spider.login(ppdloginid, ppdpasswd)
if (ppduserid == None or ppduserid != ppduserid_db):
logging.error(
"Error: Not able to get PPDAI Username or is not consistent with that in DB! Exit..."
)
exit(4)
return (ppduserid, spider, blacklistdao)
def init(self):
"""Init all he global variables and components of PPBao System
Connect to local MySQL DB, and init bid strategies, and spider
"""
for conf in self.config_files:
ppbao_config = PPBaoConfig(conf)
ppdloginid,dbhost,dbuser,dbpwd,dbname = ppbao_config.read_ppbao_config()
# Only do this when ppddao is None as those are common to all PPBao Users
if (self.ppddao == None):
PPBaoUtil.init_logging('new', ppbao_config.logdir)
logging.info("Welcome to PPBao MT System!")
logging.info("Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017")
logging.info("PPBao Config: %s,%s,%s,%s,%s" % (ppdloginid,dbhost,dbuser,dbpwd,dbname))
''' Init DB Modules '''
self.ppddao = PPDDAO({'host':dbhost,'username':dbuser,'password':dbpwd,'database':dbname})
dbok = self.ppddao.connect()
if dbok == False:
logging.error("Error: Not able to connect to MySQL! Please Fix it. Exiting now")
exit (1)
ppbaouserdao = PPBaoUserDAO(self.ppddao)
# The below statements need to be done for each user
(ppduserid_db,ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error("Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!" %(ppdloginid))
exit (2)
self.ppdloginids.append(ppdloginid)
self.ppdid_to_pwd[ppdloginid] = ppdpasswd
strategy_plus = BidStrategyPlus(ppbao_config)
self.ppdid_to_bidstrategy[ppdloginid] = strategy_plus
spider = PPDSpider(ppdloginid, ppbao_config)
self.ppdid_to_spider[ppdloginid] = spider
self.ppdid_to_keepmoney[ppdloginid] = ppbao_config.get_keep_money()
logging.info("%s: Account Minimal Keep Money: %d" % (ppdloginid, self.ppdid_to_keepmoney[ppdloginid]))
'''
Just one instance for below members
'''
self.follower = PPBaoFollower(self.ppdid_to_spider[self.ppdloginids[0]])
self.ppd_parser = PPDHtmlParser()
self.adventurer = PPBaoAdventurer(self.ppdid_to_spider[self.ppdloginids[-1]], self.ppd_parser)
self.autobid = AutoBid()
self.loandao = PPDLoanDAO(self.ppddao)
self.userdao = PPDUserDAO(self.ppddao)
self.mybiddao = MyBidDAO(self.ppddao)
self.blacklistdao = BlackListDAO(self.ppddao)
self.loanids_in_memory = self.loandao.get_last_n_days_loanids(3)
university_to_rank = UniversityDAO(self.ppddao).get_university_ranks()
if university_to_rank is None:
logging.error("Error: Not able to query DB to get University Information. Exiting now")
exit (3)
else:
PPBaoUtil.set_university_to_rank(university_to_rank)
pass
def open_history_loan(self, loanurl):
headers = {
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Upgrade-Insecure-Requests':'1',
'Accept-Language':'zh-CN,zh;q=0.8,en;q=0.6',
'Host':'invest.ppdai.com',
'Accept-Encoding':'gzip, deflate, sdch',
'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36',
'Connection':'keep-alive',
'Cache-Control':'max-age=0'
}
self.opener = PPBaoUtil.add_headers(self.opener, headers)
response = self.opener.open(loanurl,None,10)
html_str = PPBaoUtil.get_html_from_response(response)
response.close()
return html_str
def to_string(self):
from util.PPBaoUtil import PPBaoUtil
rank = PPBaoUtil.get_university_rank(self)
summary = "Edu(%s,%s,%s,%d),%s" \
% (self.education_university, \
self.education_level, self.education_type, rank, self.gender)
certs = None
if self.getihu_cert == 1:
certs = ",Cert(个体户" if certs is None else (certs + ",个体户")
if self.bank_details_cert == 1:
certs = ",Cert(银行流水" if certs is None else (certs + ",银行流水")
if self.job_cert == 1:
certs = ",Cert(工作" if certs is None else (certs + ",工作")
if self.ren_hang_trust_cert == 1:
certs = ",Cert(征信" if certs is None else (certs + ",征信")
if self.shouru_cert == 1:
certs = ",Cert(收入" if certs is None else (certs + ",收入")
if self.alipay_cert == 1:
certs = ",Cert(支付宝" if certs is None else (certs + ",支付宝")
if self.student_cert == 1:
certs = ",Cert(学生证" if certs is None else (certs + ",学生证")
if self.shebao_gjj_cert == 1:
certs = ",Cert(社保" if certs is None else (certs + ",社保")
if self.driver_cert == 1:
certs = ",Cert(驾驶证" if certs is None else (certs + ",驾驶证")
if self.hukou_cert == 1:
certs = ",Cert(户口" if certs is None else (certs + ",户口")
certs = ",Cert(NA)" if certs is None else (certs + ")")
summary += certs
return summary
def testName(self):
ppbao_config_file = "../conf/ppbao.me.config"
# Initialize
ppbao_config = PPBaoConfig(ppbao_config_file)
ppdloginid,dbhost,dbuser,dbpwd,dbname = ppbao_config.read_ppbao_config()
init_logging(ppdloginid)
ppbao_config.print_strategies()
bsp = BidStrategyPlus(ppbao_config)
# Init DB Modules
from dao.PPDDAO import PPDDAO
from dao.UniversityDAO import UniversityDAO
from util.PPBaoUtil import PPBaoUtil
ppddao = PPDDAO({'host':dbhost,'username':dbuser,'password':dbpwd,'database':dbname})
dbok = ppddao.connect()
if dbok == False:
logging.error("Error: Not able to connect to MySQL! Please Fix it. Exiting now")
exit (1)
university_to_rank = UniversityDAO(ppddao).get_university_ranks()
if university_to_rank is None:
logging.error("Error: Not able to query DB to get University Information. Exiting now")
exit (3)
else:
PPBaoUtil.set_university_to_rank(university_to_rank)
" Build Test PPDLoan and PPDUser instance"
from ds.PPDLoan import PPDLoan
from ds.PPDUser import PPDUser
from datetime import datetime
now = datetime.now()
loanid = '1111111'
ppdloan = PPDLoan({'loanid':1111111, 'datetime':now, 'loanrate':22, 'ppdrate':'C', \
'money':5000, 'maturity':12, 'userid':'pdu2517233537', 'age': 31})
ppduser = PPDUser({'userid':'pdu2517233537', 'gender': '男', 'age': 31, 'marriage': '已婚', \
'house': '有', 'car': '无', 'education_level': '本科'})
ppdloan.set_ppduser(ppduser)
ppdloan.set_history_info(8,0,0,12000,3000,0)
ppdloan.history_highest_total_loan = 12000
ppdloan.loantitle = "NA-Test"
ppduser.add_education_cert('保定学院', '本科', '普通')
logging.info(ppdloan.get_loan_summary())
ifbid, money, reason = bsp.check_by_strategy(ppdloan)
self.assertTrue(ifbid, "No Bid for master??")
def open_lend(self, opener):
url = "http://invest.ppdai.com/account/lend"
headers = {
'Host': 'invest.ppdai.com',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, sdch',
'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
}
opener = PPBaoUtil.add_headers(opener, headers)
response = opener.open(url, None, 10)
content = PPBaoUtil.get_html_from_response(response)
info = response.info()
#logging.info("Get Response: %s", data)
for head in info:
logging.debug("%s=%s" % (head, info.get(head)))
def __init__(self, config):
ppbao_config = PPBaoConfig(config)
ppdloginid, dbhost, dbuser, dbpwd, dbname = ppbao_config.read_ppbao_config(
)
PPBaoUtil.init_logging('test_bid_strategy', ppbao_config.logdir)
ppddao = PPDDAO({
'host': dbhost,
'username': dbuser,
'password': dbpwd,
'database': dbname
})
ppddao.connect()
self.loandao = PPDLoanDAO(ppddao)
self.userdao = PPDUserDAO(ppddao)
unidao = UniversityDAO(ppddao)
PPBaoUtil.set_university_to_rank(unidao.get_university_ranks())
self.bid_strategy = BidStrategyPlus(ppbao_config)
self.date_to_bids = {}
def check_university_rank(self, ppdloan):
''' if university rank is <=n, return true, -1 means no check on university_rank '''
if (self.urank_start == -1 and self.urank_end == -1):
return True
rank = PPBaoUtil.get_university_rank(ppdloan.ppduser)
if rank >= self.urank_start and rank <= self.urank_end:
return True
else:
return False
def check_university_rank(self, ppdloan):
''' if university rank is <=n, return true, -1 means no check on university_rank '''
if (self.urank_start == -1 and self.urank_end == -1):
return True
rank = PPBaoUtil.get_university_rank(ppdloan.ppduser)
if rank >= self.urank_start and rank <= self.urank_end:
return True
else:
return False
def step4_check_bid_result(self, opener, maturity, loanid, money):
loanurl = "http://invest.ppdai.com/loan/info?id=%d" % (loanid)
refer_url = "http://invest.ppdai.com/bid/info?source=2&listingId=%d" % (
loanid) + '%20%20%20%20&title=&date=' + "%d" % (
maturity) + '%20%20%20%20&UrlReferrer=1&money=' + "%d" % (
money)
headers = {
"Accept-Encoding": "gzip, deflate, sdch",
"Host": "invest.ppdai.com",
"Accept-Language": "zh-CN,zh;q=0.8,en;q=0.6",
"Upgrade-Insecure-Requests": "1",
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Referer": refer_url,
"Connection": "keep-alive"
}
opener = PPBaoUtil.add_headers(opener, headers)
response = opener.open(loanurl, None, 15)
html = PPBaoUtil.get_html_from_response(response)
#logging.debug("Get Response: %s", html)
#response_headers = response.info()
#for head in response_headers:
# logging.debug("%s:%s" % (head, response_headers[head]))
m = re.search(self.actual_bid_pattern, html)
if (m is not None):
logging.info("Bid %s successfully!!!" % (m.group(1)))
actual_bid = int(m.group(1))
else:
progress_m = re.search(self.pattern_current_progress, html)
if progress_m is not None and progress_m.group(1) == '100%':
logging.warn(
"Bid Failed: It's Already 100% completed! We're too late!")
else:
logging.warn(
"Not match ActualBid Pattern. Most likely Bid is not successful! Do Check it"
)
actual_bid = -1
# -1 means not parsed from html
mymoney = -1
ac = re.search(self.pattern_myaccount_left, html)
if (ac is not None):
mymoney = float(ac.group(1).replace(',', ''))
#logging.info("My Account Left: %4.2f" % (mymoney))
return (actual_bid, mymoney)
def init_ppbao(argv):
ppbao_config_file = None
if (len(argv) == 1):
ppbao_config_file = "../conf/ppbao.18616856236.config"
ppbao_config_file = "../conf/ppbao.18616027065.config"
elif (len(argv) == 2):
me,ppbao_config_file = argv
else:
print "Error: More than 1 argument is provided!"
print "Usage: python update_blacklist.py <ppbao_config_file>"
exit (-1)
# Initialize
ppbao_config = PPBaoConfig(ppbao_config_file)
ppdloginid,dbhost,dbuser,dbpwd,dbname = ppbao_config.read_ppbao_config()
PPBaoUtil.init_logging(ppdloginid,ppbao_config.logdir)
logging.info("Welcome to PPBao System - Update BlackList Utility!")
logging.info("Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017")
logging.info("PPBao Config: %s,%s,%s,%s,%s" % (ppdloginid,dbhost,dbuser,dbpwd,dbname))
# Init DB Modules
ppddao = PPDDAO({'host':dbhost,'username':dbuser,'password':dbpwd,'database':dbname})
dbok = ppddao.connect()
if dbok == False:
logging.error("Error: Not able to connect to MySQL! Please Fix it. Exiting now")
exit (1)
ppbaouserdao = PPBaoUserDAO(ppddao)
blacklistdao = BlackListDAO(ppddao)
(ppduserid_db,ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error("Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!" %(ppdloginid))
exit (2)
# Login to PPDAI!
spider = PPDSpider(ppdloginid, ppbao_config)
(opener, ppduserid) = spider.login(ppdloginid, ppdpasswd)
if (ppduserid == None or ppduserid != ppduserid_db):
logging.error("Error: Not able to get PPDAI Username or is not consistent with that in DB! Exit...")
exit(4)
return (ppduserid, spider, blacklistdao)
def open_blacklist(self, opener):
url = "http://invest.ppdai.com/account/blacklist"
headers = {
'Host': 'invest.ppdai.com',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, sdch',
'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'Referer': 'http://invest.ppdai.com/account/paybacklendreceived'
}
opener = PPBaoUtil.add_headers(opener, headers)
response = opener.open(url, None, 10)
html = PPBaoUtil.get_html_from_response(response)
#logging.debug("Get Response: %s", html)
response_headers = response.info()
for head in response_headers:
logging.debug("%s:%s" % (head, response_headers[head]))
def init(self):
for conf in self.config_files:
ppbao_config = PPBaoConfig(conf)
ppdloginid,dbhost,dbuser,dbpwd,dbname = ppbao_config.read_ppbao_config()
# Only do this when ppddao is None as those are common to all PPBao Users
if (self.ppddao == None):
PPBaoUtil.init_logging('new', ppbao_config.logdir)
logging.info("Welcome to PPBao System!")
logging.info("Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017")
logging.info("PPBao Config: %s,%s,%s,%s,%s" % (ppdloginid,dbhost,dbuser,dbpwd,dbname))
''' Init DB Modules '''
self.ppddao = PPDDAO({'host':dbhost,'username':dbuser,'password':dbpwd,'database':dbname})
dbok = self.ppddao.connect()
if dbok == False:
logging.error("Error: Not able to connect to MySQL! Please Fix it. Exiting now")
exit (1)
ppbaouserdao = PPBaoUserDAO(self.ppddao)
# The below statements need to be done for each user
(ppduserid_db,ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error("Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!" %(ppdloginid))
exit (2)
self.ppdloginids.append(ppdloginid)
self.ppdid_to_pwd[ppdloginid] = ppdpasswd
strategy_plus = BidStrategyPlus(ppbao_config)
self.ppdid_to_bidstrategy[ppdloginid] = strategy_plus
spider = PPDSpider(ppdloginid, ppbao_config)
self.ppdid_to_spider[ppdloginid] = spider
self.ppd_parser = PPDHtmlParser()
self.autobid = AutoBid()
self.loandao = PPDLoanDAO(self.ppddao)
self.userdao = PPDUserDAO(self.ppddao)
self.mybiddao = MyBidDAO(self.ppddao)
self.blacklistdao = BlackListDAO(self.ppddao)
university_to_rank = UniversityDAO(self.ppddao).get_university_ranks()
if university_to_rank is None:
logging.error("Error: Not able to query DB to get University Information. Exiting now")
exit (3)
else:
PPBaoUtil.set_university_to_rank(university_to_rank)
pass
def step2_get_bidloan(opener, maturity, loanid, money):
loanurl = "http://invest.ppdai.com/bid/info?source=2&listingId=%d" % (loanid) \
+ '%20%20%20%20&title=&date=' + "%d" %(maturity) + '%20%20%20%20&UrlReferrer=1&money=' + "%d" %(money)
headers = {"Accept-Encoding": "gzip, deflate, sdch",
"Host": "invest.ppdai.com",
"Accept-Language": "zh-CN,zh;q=0.8,en;q=0.6",
"Upgrade-Insecure-Requests": "1",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Referer": "http://invest.ppdai.com/loan/info?id=%d" % (loanid),
"Connection": "keep-alive"
}
opener = PPBaoUtil.add_headers(opener, headers)
response = opener.open(loanurl, None, 10)
html = PPBaoUtil.get_html_from_response(response)
logging.debug("Get Response: %s", html)
response_headers = response.info()
for head in response_headers:
logging.info("%s:%s" % (head, response_headers[head]))
return loanurl
def step0_open_loan(opener, loanid):
loanurl = "http://invest.ppdai.com/loan/info?id=%d" % (loanid)
headers = {"Accept-Encoding": "gzip, deflate, sdch",
"Host": "invest.ppdai.com",
'Accept':'*/*',
'Cache-Control':'max-age=0',
"Accept-Language": "zh-CN,zh;q=0.8,en;q=0.6",
"Upgrade-Insecure-Requests": "1",
"Referer": "http://invest.ppdai.com/loan/list_riskmiddle_s5_p2?Rate=0",
"Connection": "keep-alive"
}
opener = PPBaoUtil.add_headers(opener, headers)
response = opener.open(loanurl, None, 10)
html = PPBaoUtil.get_html_from_response(response)
logging.debug("Get Response: %s", html)
response_headers = response.info()
for head in response_headers:
logging.info("%s:%s" % (head, response_headers[head]))
return loanurl
def open_loan_detail_page(self, loanurl, referer_url):
try:
headers = {
'Referer':referer_url,
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Upgrade-Insecure-Requests':'1',
'Accept-Language':'zh-CN,zh;q=0.8,en;q=0.6',
'Host':'invest.ppdai.com',
'Accept-Encoding':'gzip, deflate, sdch',
'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36',
'Connection':'keep-alive'
}
self.opener = PPBaoUtil.add_headers(self.opener, headers)
response = self.opener.open(loanurl,None,15)
html_str = PPBaoUtil.get_html_from_response(response)
response.close()
return html_str
except urllib2.URLError, e:
logging.error("Not able to open %s, %r" % (loanurl, e))
return None
def get_pages(self, loan_url, last_url):
try:
'''
headers = {
'Referer':'http://invest.ppdai.com/loan/list_riskmiddle?monthgroup=&rate=0&didibid=&listingispay=',
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Upgrade-Insecure-Requests':'1',
'Accept-Language':'zh-CN,zh;q=0.8,en;q=0.6',
'Host':'invest.ppdai.com',
'Accept-Encoding':'gzip, deflate, sdch',
'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36',
'Connection':'keep-alive'
}
self.opener.addheaders = headers
'''
self.opener.addheaders = [('User-Agent','Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36')]
self.opener.addheaders.append(('Referer',last_url))
self.opener.addheaders.append(('Accept','text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'))
self.opener.addheaders.append(('Accept-Language','zh-CN,zh;q=0.8,en;q=0.6'))
self.opener.addheaders.append(('Accept-Encoding','gzip, deflate, sdch'))
self.opener.addheaders.append(('Connection','keep-alive'))
self.opener.addheaders.append(('Upgrade-Insecure-Requests','1'))
logging.debug("OpenURL: %s" % (loan_url))
response = self.opener.open(loan_url,None,10)
html_str = PPBaoUtil.get_html_from_response(response)
response.close()
#logging.debug("HTML: %s" % (html_str))
m = re.search(self.loan_count_pattern, html_str)
loan_count = -1
skipped = 0
loanid_to_mobile = {}
loanid_to_xueli = {}
if m is not None:
skipped, loanid_to_mobile, loanid_to_xueli = self.get_loaid_list_from_page_html(html_str)
loan_count = int(m.group(1))
if (loan_count == 0):
logging.info("0 Loans detected")
return (0, 0, 0, loanid_to_mobile, loanid_to_xueli)
elif (loan_count <= 10):
logging.debug("%d loans detected" % (loan_count))
return (loan_count, 1, skipped, loanid_to_mobile, loanid_to_xueli)
else:
pass # Continue to check the pages
m = re.search(self.page_pattern, html_str)
if m is not None:
return (loan_count, int(m.group(1)), 0, loanid_to_mobile, loanid_to_xueli)
else:
logging.error( "Not Match the Page Pattern.")
return (loan_count, -1, 0, loanid_to_mobile, loanid_to_xueli)
except Exception, e:
logging.error("Failed to get pages. Ignore and Continue,but do Check it: Exception: %r" %(e))
return (-1,-1,0, None, None)
def step0_open_loan(self, opener, loanid):
loanurl = "http://invest.ppdai.com/loan/info?id=%d" % (loanid)
headers = {
"Accept-Encoding": "gzip, deflate, sdch",
"Host": "invest.ppdai.com",
'Accept': '*/*',
'Cache-Control': 'max-age=0',
"Accept-Language": "zh-CN,zh;q=0.8,en;q=0.6",
"Upgrade-Insecure-Requests": "1",
"Referer":
"http://invest.ppdai.com/loan/list_riskmiddle_s5_p2?Rate=0",
"Connection": "keep-alive"
}
opener = PPBaoUtil.add_headers(opener, headers)
response = opener.open(loanurl, None, 10)
html = PPBaoUtil.get_html_from_response(response)
#logging.debug("Get Response: %s", html)
response_headers = response.info()
for head in response_headers:
logging.debug("%s:%s" % (head, response_headers[head]))
return loanurl
def step2_get_bidloan(self, opener, maturity, loanid, money):
loanurl = "http://invest.ppdai.com/bid/info?source=2&listingId=%d" % (loanid) \
+ '%20%20%20%20&title=&date=' + "%d" %(maturity) + '%20%20%20%20&UrlReferrer=1&money=' + "%d" %(money)
headers = {
"Accept-Encoding": "gzip, deflate, sdch",
"Host": "invest.ppdai.com",
"Accept-Language": "zh-CN,zh;q=0.8,en;q=0.6",
"Upgrade-Insecure-Requests": "1",
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Referer": "http://invest.ppdai.com/loan/info?id=%d" % (loanid),
"Connection": "keep-alive"
}
opener = PPBaoUtil.add_headers(opener, headers)
response = opener.open(loanurl, None, 10)
html = PPBaoUtil.get_html_from_response(response)
#logging.debug("Get Response: %s", html)
response_headers = response.info()
#for head in response_headers:
# logging.info("%s:%s" % (head, response_headers[head]))
return loanurl
def init_ppbao(ppbao_config_file, ppbaouserdao):
# Initialize
ppbao_config = PPBaoConfig(ppbao_config_file)
ppdloginid,dbhost,dbuser,dbpwd,dbname = ppbao_config.read_ppbao_config()
PPBaoUtil.init_logging(ppdloginid,ppbao_config.logdir)
logging.info("Welcome to PPBao System - Update BlackList Utility!")
logging.info("Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017")
logging.info("PPBao Config: %s,%s,%s,%s,%s" % (ppdloginid,dbhost,dbuser,dbpwd,dbname))
(ppduserid_db,ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error("Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!" %(ppdloginid))
exit (2)
# Login to PPDAI!
spider = PPDSpider(ppdloginid, ppbao_config)
(opener, ppduserid) = spider.login(ppdloginid, ppdpasswd)
if (ppduserid == None or ppduserid != ppduserid_db):
logging.error("Error: Not able to get PPDAI Username or is not consistent with that in DB! Exit...")
exit(4)
return (ppduserid, spider)
def step3_bid(opener, maturity, loanid, money):
loanurl = "http://invest.ppdai.com/bid/info?source=2&listingId=%d" % (loanid) + '%20%20%20%20&title=&date=' + "%d" %(maturity) + '%20%20%20%20&UrlReferrer=1&money=' + "%d" %(money)
bidurl = "http://invest.ppdai.com/Bid/Bid"
bid_info = {"Reason":'', "Amount":money, "ListingId":loanid, "UrlReferrer":"1", "SubListType":'0'}
headers = {"Origin": "http://invest.ppdai.com",
"Accept-Encoding": "gzip, deflate",
"Host": "invest.ppdai.com",
"Accept-Language": "zh-CN,zh;q=0.8,en;q=0.6",
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "*/*",
"Referer": loanurl,
"X-Requested-With": "XMLHttpRequest",
"Connection": "keep-alive",
"Content-Length": "63"
}
upost_data = urllib.urlencode(bid_info)
opener = PPBaoUtil.add_headers(opener, headers)
response = opener.open(bidurl, upost_data, 10)
html = PPBaoUtil.get_html_from_response(response)
logging.info("Get Response: %s", html)
response_headers = response.info()
for head in response_headers:
logging.info("%s:%s" % (head, response_headers[head]))
def step1_open_actionlog(self, opener, loanid):
url = "http://invest.ppdai.com/common/actionlog"
headers = {
"Origin": "http://invest.ppdai.com",
"Accept-Encoding": "gzip, deflate",
"Host": "invest.ppdai.com",
"Accept-Language": "zh-CN,zh;q=0.8,en;q=0.6",
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "*/*",
"Referer": "http://invest.ppdai.com/loan/info?id=%d" % (loanid),
"X-Requested-With": "XMLHttpRequest",
"Connection": "keep-alive",
"Content-Length": "109"
}
post_data = {
"useraction": "VisitRecord",
"functionname": '%E6%95%A3%E6%A0%87%E6%8A%95%E6%A0%87',
"remark": '%E6%95%A3%E6%A0%87' + "%d+++++" % (loanid)
}
opener = PPBaoUtil.add_headers(opener, headers)
upost_data = urllib.urlencode(post_data)
response = opener.open(url, upost_data, 10)
html = PPBaoUtil.get_html_from_response(response)
logging.debug("Get Response: %s", html)
def step4_check_bid_result(self, opener, maturity, loanid, money):
loanurl = "http://invest.ppdai.com/loan/info?id=%d" % (loanid)
refer_url = "http://invest.ppdai.com/bid/info?source=2&listingId=%d" % (loanid) + '%20%20%20%20&title=&date=' + "%d" %(maturity) + '%20%20%20%20&UrlReferrer=1&money=' + "%d" %(money)
headers = {"Accept-Encoding": "gzip, deflate, sdch",
"Host": "invest.ppdai.com",
"Accept-Language": "zh-CN,zh;q=0.8,en;q=0.6",
"Upgrade-Insecure-Requests": "1",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Referer": refer_url,
"Connection": "keep-alive"
}
opener = PPBaoUtil.add_headers(opener, headers)
response = opener.open(loanurl, None, 15)
html = PPBaoUtil.get_html_from_response(response)
#logging.debug("Get Response: %s", html)
#response_headers = response.info()
#for head in response_headers:
# logging.debug("%s:%s" % (head, response_headers[head]))
m = re.search(self.actual_bid_pattern, html)
if (m is not None):
logging.info("Bid %s successfully!!!" % (m.group(1)))
actual_bid = int(m.group(1))
else:
progress_m = re.search(self.pattern_current_progress, html)
if progress_m is not None and progress_m.group(1) == '100%':
logging.warn("Bid Failed: It's Already 100% completed! We're too late!")
else:
logging.warn("Not match ActualBid Pattern. Most likely Bid is not successful! Do Check it")
actual_bid = -1
# -1 means not parsed from html
mymoney = -1
ac = re.search(self.pattern_myaccount_left, html)
if (ac is not None):
mymoney = float(ac.group(1).replace(',',''))
#logging.info("My Account Left: %4.2f" % (mymoney))
return (actual_bid, mymoney)
def get_loanid_list_from_listing_page (self, loan_url, referer_url):
" This function is to analsysis the Loan List page and return a list of links point to the detailed loan info page"
try:
self.opener.addheaders = [('User-Agent','Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36')]
self.opener.addheaders.append(('Origin', 'http://invest.ppdai.com'))
self.opener.addheaders.append(('Host','invest.ppdai.com'))
self.opener.addheaders.append(('Accept-Encoding','gzip, deflate, sdch'))
self.opener.addheaders.append(('Referer',referer_url))
self.opener.addheaders.append(('Accept','*/*'))
self.opener.addheaders.append(('Accept-Language','zh-CN,zh;q=0.8,en;q=0.6'))
self.opener.addheaders.append(('Connection','keep-alive'))
self.opener.addheaders.append(('Cache-Control','max-age=0'))
response = self.opener.open(loan_url,None,15)
html_str = PPBaoUtil.get_html_from_response(response)
response.close()
return self.get_loaid_list_from_page_html(html_str)
except HTTPError, e:
logging.debug("Failed to open page: %s - Reason: %r" %( loan_url, e))
return (-1, None, None)
def get_loan_summary(self):
""" Return a Human Readable string of a loan summary """
rank = PPBaoUtil.get_university_rank(self.ppduser)
summary = "Rate(%s),Loan(%d,%d,%d),Edu(%s,%s,%s,%d),Hist(%d,%d,%d),HistLoan(%d,%d,%d,%d,%d),Age(%d),%s" \
% (self.ppdrate, self.money, self.loanrate, self.maturity,self.ppduser.education_university, \
self.ppduser.education_level, self.ppduser.education_type, rank, self.history_return_ontime, self.history_overdue_in15d, \
self.history_overdue_mt15d, self.history_total_loan, self.history_left_loan, self.history_left_lend, self.history_highest_total_loan, self.new_total_loan, self.age, \
self.ppduser.gender)
certs = None
if self.ppduser.getihu_cert == 1:
certs = ",Cert(个体户" if certs is None else (certs + ",个体户")
if self.ppduser.bank_details_cert == 1:
certs = ",Cert(银行流水" if certs is None else (certs + ",银行流水")
if self.ppduser.job_cert == 1:
certs = ",Cert(工作" if certs is None else (certs + ",工作")
if self.ppduser.ren_hang_trust_cert == 1:
certs = ",Cert(征信" if certs is None else (certs + ",征信")
if self.ppduser.shouru_cert == 1:
certs = ",Cert(收入" if certs is None else (certs + ",收入")
if self.ppduser.alipay_cert == 1:
certs = ",Cert(支付宝" if certs is None else (certs + ",支付宝")
if self.ppduser.student_cert == 1:
certs = ",Cert(学生证" if certs is None else (certs + ",学生证")
if self.ppduser.shebao_gjj_cert == 1:
certs = ",Cert(社保" if certs is None else (certs + ",社保")
if self.ppduser.driver_cert == 1:
certs = ",Cert(驾驶证" if certs is None else (certs + ",驾驶证")
if self.ppduser.hukou_cert == 1:
certs = ",Cert(户口" if certs is None else (certs + ",户口")
certs = ",Cert(NA)" if certs is None else (certs + ")")
summary += certs
if self.is_shandian_loan == 1:
summary += ",闪电标"
if self.ppduser.mobile_cert == 0:
summary += ",NoMobile"
if self.ppduser.idcard_cert == 0:
summary += ",NoIDCard"
if self.has_30or36rate_loan_history == 1:
summary += ",HasHighLoanHist"
if (self.source != 'page_walker'):
summary += ",source(%s)" % (self.source)
return summary
def open_page(self,url,referer_url):
'''
Try to Open page and return the HTML String for further analysis
'''
try:
self.opener.addheaders.append(('Origin', 'http://invest.ppdai.com'))
self.opener.addheaders.append(('Host','invest.ppdai.com'))
self.opener.addheaders.append(('Referer',referer_url))
self.opener.addheaders.append(('Accept','*/*'))
self.opener.addheaders.append(('Connection','keep-alive'))
self.opener.addheaders.append(('Cache-Control','max-age=0'))
self.opener.addheaders.append(('Accept-Encoding','gzip, deflate, sdch'))
self.opener.addheaders.append(('Accept-Language','zh-CN,zh;q=0.8,en;q=0.6'))
self.opener.addheaders.append(('User-Agent','Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36'))
response = self.opener.open(url,None,15)
html_str = PPBaoUtil.get_html_from_response(response)
response.close()
return html_str
except urllib2.URLError, e:
logging.debug("Not able to open %s, %r" % (url, e))
return None
ppdloan.ppduser = ppduser
ppdloan.loantitle = results[14]
ppdloan.set_history_info(int(results[7]), int(results[8]), int(results[9]), int(results[10]), int(results[11]), int(results[12]))
ppdloan.has_30or36rate_loan_history = int(results[17])
ppdloan.new_total_loan = int(results[18])
ppdloan.history_highest_total_loan = int(results[19])
ppdloan_list.append(ppdloan)
return ppdloan_list
else:
logging.error("No ppdloan in DB found for in last %d days" % (ndays))
return None
if __name__ == '__main__':
from dao.UniversityDAO import UniversityDAO
from dao.PPDDAO import PPDDAO
from dao.PPDUserDAO import PPDUserDAO
from util.PPBaoUtil import PPBaoUtil
import sys
reload(sys)
sys.setdefaultencoding('utf8')
ppddao = PPDDAO({'host':'localhost','username':'******','password':'******','database':'ppdai'})
ppddao.connect()
loandao = PPDLoanDAO(ppddao)
userdao = PPDUserDAO(ppddao)
unidao = UniversityDAO(ppddao)
PPBaoUtil.set_university_to_rank(unidao.get_university_ranks())
loan = loandao.get_db_ppdloan_by_loanid(12000659, userdao)
print loan.get_loan_summary()
ppddao.disconnect()
def init(self):
"""Init all he global variables and components of PPBao System
Connect to local MySQL DB, and init bid strategies, and spider
"""
for conf in self.config_files:
ppbao_config = PPBaoConfig(conf)
ppdloginid, dbhost, dbuser, dbpwd, dbname = ppbao_config.read_ppbao_config(
)
# Only do this when ppddao is None as those are common to all PPBao Users
if (self.ppddao == None):
PPBaoUtil.init_logging('new', ppbao_config.logdir)
logging.info("Welcome to PPBao MT System!")
logging.info(
"Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017"
)
logging.info("PPBao Config: %s,%s,%s,%s,%s" %
(ppdloginid, dbhost, dbuser, dbpwd, dbname))
''' Init DB Modules '''
self.ppddao = PPDDAO({
'host': dbhost,
'username': dbuser,
'password': dbpwd,
'database': dbname
})
dbok = self.ppddao.connect()
if dbok == False:
logging.error(
"Error: Not able to connect to MySQL! Please Fix it. Exiting now"
)
exit(1)
ppbaouserdao = PPBaoUserDAO(self.ppddao)
# The below statements need to be done for each user
(ppduserid_db,
ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error(
"Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!"
% (ppdloginid))
exit(2)
self.ppdloginids.append(ppdloginid)
self.ppdid_to_pwd[ppdloginid] = ppdpasswd
strategy_plus = BidStrategyPlus(ppbao_config)
self.ppdid_to_bidstrategy[ppdloginid] = strategy_plus
spider = PPDSpider(ppdloginid, ppbao_config)
self.ppdid_to_spider[ppdloginid] = spider
self.ppdid_to_keepmoney[ppdloginid] = ppbao_config.get_keep_money()
logging.info("%s: Account Minimal Keep Money: %d" %
(ppdloginid, self.ppdid_to_keepmoney[ppdloginid]))
'''
Just one instance for below members
'''
self.follower = PPBaoFollower(
self.ppdid_to_spider[self.ppdloginids[0]])
self.ppd_parser = PPDHtmlParser()
self.adventurer = PPBaoAdventurer(
self.ppdid_to_spider[self.ppdloginids[-1]], self.ppd_parser)
self.autobid = AutoBid()
self.loandao = PPDLoanDAO(self.ppddao)
self.userdao = PPDUserDAO(self.ppddao)
self.mybiddao = MyBidDAO(self.ppddao)
self.blacklistdao = BlackListDAO(self.ppddao)
self.loanids_in_memory = self.loandao.get_last_n_days_loanids(3)
university_to_rank = UniversityDAO(self.ppddao).get_university_ranks()
if university_to_rank is None:
logging.error(
"Error: Not able to query DB to get University Information. Exiting now"
)
exit(3)
else:
PPBaoUtil.set_university_to_rank(university_to_rank)
pass
}
opener = PPBaoUtil.add_headers(opener, headers)
response = opener.open(url,None,15)
html_str = PPBaoUtil.get_html_from_response(response)
response.close()
return html_str;
except Exception,e:
logging.error("On OpenPage - Caught Exception %r" %(e))
return None
else:
print "Not supported for test"
if __name__ == '__main__':
ppbao_config = PPBaoConfig("../conf/ppbao.me.config")
ppdloginid,dbhost,dbuser,dbpwd,dbname = ppbao_config.read_ppbao_config()
PPBaoUtil.init_logging(ppdloginid,ppbao_config.logdir)
ppddao, ppbaouserdao = init_dao(dbhost,dbuser,dbpwd,dbname)
spiders = []
for conf in ('18616856236', '18616027065'):
file = "../conf/ppbao.%s.config" % (conf)
print "File: %s" % (file)
ppduserid, spider = init_ppbao(file, ppbaouserdao)
spiders.append(spider)
sleep(5)
for spider in spiders:
logging.info("Opened HTML for ....")
html = open_blacklist_page(spider.opener, 1)
logging.debug(html)
sleep (5)
response = opener.open(url, None, 15)
html_str = PPBaoUtil.get_html_from_response(response)
response.close()
return html_str
except Exception, e:
logging.error("On OpenPage - Caught Exception %r" % (e))
return None
else:
print "Not supported for test"
if __name__ == '__main__':
ppbao_config = PPBaoConfig("../conf/ppbao.me.config")
ppdloginid, dbhost, dbuser, dbpwd, dbname = ppbao_config.read_ppbao_config(
)
PPBaoUtil.init_logging(ppdloginid, ppbao_config.logdir)
ppddao, ppbaouserdao = init_dao(dbhost, dbuser, dbpwd, dbname)
spiders = []
for conf in ('18616856236', '18616027065'):
file = "../conf/ppbao.%s.config" % (conf)
print "File: %s" % (file)
ppduserid, spider = init_ppbao(file, ppbaouserdao)
spiders.append(spider)
sleep(5)
for spider in spiders:
logging.info("Opened HTML for ....")
html = open_blacklist_page(spider.opener, 1)
logging.debug(html)
sleep(5)
def get_all_ppdusers(self):
db_stat = "select * from ppduser"
ppduser_hash = {}
if (self.dao.execute(db_stat)):
data = self.dao.dbcursor.fetchall()
for results in data:
ppduser_hash[results[0]] = self.parse_ppduser_db_row(results)
return ppduser_hash
else:
logging.error("Not able to get PPDUsers")
return None
if __name__ == '__main__':
from dao.UniversityDAO import UniversityDAO
from util.PPBaoUtil import PPBaoUtil
ppddao = PPDDAO({
'host': 'localhost',
'username': '******',
'password': '******',
'database': 'ppdai'
})
ppddao.connect()
ppduserdao = PPDUserDAO(ppddao)
unidao = UniversityDAO(ppddao)
PPBaoUtil.set_university_to_rank(unidao.get_university_ranks())
ppduser = ppduserdao.get_db_ppduser_by_userid('pdu26068827')
print ppduser.to_string()
ppddao.disconnect()