def test_fetch_loanids(self):
ppddao = PPDDAO({'host':'localhost','username':'******','password':'******','database':'ppdai'})
ppddao.connect()
ppdloan_dao = PPDLoanDAO(ppddao)
today = date.today()
loanids = ppdloan_dao.get_loanids_by_date(today)
print loanids;
self.assertTrue(len(loanids)>=1)
def test_insert_ppdloan(self):
ppddao = PPDDAO({'host':'localhost','username':'******','password':'******','database':'ppdai'})
ppddao.connect()
ppdloan_dao = PPDLoanDAO(ppddao)
today = date.today()
ppdloan = PPDLoan({'loanid':8822451, 'date':today, 'loanrate':7.0, 'ppdrate':'AAA', \
'money':3000, 'maturity':12, 'userid':'pdu6310825153', 'age': 46})
ppduser = PPDUser({'userid':'pdu6310825153', 'gender': '男', 'age': 46, 'marriage': '已婚', \
'house': '自住无按揭', 'car': '有', 'education_level': '本科'})
ppdloan.set_ppduser(ppduser)
ppdloan.set_history_info(1, 0, 0, 30000, 29419.12, 108230.77)
result = ppdloan_dao.insert(ppdloan)
ppddao.disconnect()
self.assertFalse(result, "Insert Duplicate records. ")
def __init__(self, config):
ppbao_config = PPBaoConfig(config)
ppdloginid, dbhost, dbuser, dbpwd, dbname = ppbao_config.read_ppbao_config(
)
PPBaoUtil.init_logging('test_bid_strategy', ppbao_config.logdir)
ppddao = PPDDAO({
'host': dbhost,
'username': dbuser,
'password': dbpwd,
'database': dbname
})
ppddao.connect()
self.loandao = PPDLoanDAO(ppddao)
self.userdao = PPDUserDAO(ppddao)
unidao = UniversityDAO(ppddao)
PPBaoUtil.set_university_to_rank(unidao.get_university_ranks())
self.bid_strategy = BidStrategyPlus(ppbao_config)
self.date_to_bids = {}
def init(self):
for conf in self.config_files:
ppbao_config = PPBaoConfig(conf)
ppdloginid,dbhost,dbuser,dbpwd,dbname = ppbao_config.read_ppbao_config()
# Only do this when ppddao is None as those are common to all PPBao Users
if (self.ppddao == None):
PPBaoUtil.init_logging('new', ppbao_config.logdir)
logging.info("Welcome to PPBao System!")
logging.info("Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017")
logging.info("PPBao Config: %s,%s,%s,%s,%s" % (ppdloginid,dbhost,dbuser,dbpwd,dbname))
''' Init DB Modules '''
self.ppddao = PPDDAO({'host':dbhost,'username':dbuser,'password':dbpwd,'database':dbname})
dbok = self.ppddao.connect()
if dbok == False:
logging.error("Error: Not able to connect to MySQL! Please Fix it. Exiting now")
exit (1)
ppbaouserdao = PPBaoUserDAO(self.ppddao)
# The below statements need to be done for each user
(ppduserid_db,ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error("Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!" %(ppdloginid))
exit (2)
self.ppdloginids.append(ppdloginid)
self.ppdid_to_pwd[ppdloginid] = ppdpasswd
strategy_plus = BidStrategyPlus(ppbao_config)
self.ppdid_to_bidstrategy[ppdloginid] = strategy_plus
spider = PPDSpider(ppdloginid, ppbao_config)
self.ppdid_to_spider[ppdloginid] = spider
self.ppd_parser = PPDHtmlParser()
self.autobid = AutoBid()
self.loandao = PPDLoanDAO(self.ppddao)
self.userdao = PPDUserDAO(self.ppddao)
self.mybiddao = MyBidDAO(self.ppddao)
self.blacklistdao = BlackListDAO(self.ppddao)
university_to_rank = UniversityDAO(self.ppddao).get_university_ranks()
if university_to_rank is None:
logging.error("Error: Not able to query DB to get University Information. Exiting now")
exit (3)
else:
PPBaoUtil.set_university_to_rank(university_to_rank)
pass
def init(self):
"""Init all he global variables and components of PPBao System
Connect to local MySQL DB, and init bid strategies, and spider
"""
for conf in self.config_files:
ppbao_config = PPBaoConfig(conf)
ppdloginid,dbhost,dbuser,dbpwd,dbname = ppbao_config.read_ppbao_config()
# Only do this when ppddao is None as those are common to all PPBao Users
if (self.ppddao == None):
PPBaoUtil.init_logging('new', ppbao_config.logdir)
logging.info("Welcome to PPBao MT System!")
logging.info("Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017")
logging.info("PPBao Config: %s,%s,%s,%s,%s" % (ppdloginid,dbhost,dbuser,dbpwd,dbname))
''' Init DB Modules '''
self.ppddao = PPDDAO({'host':dbhost,'username':dbuser,'password':dbpwd,'database':dbname})
dbok = self.ppddao.connect()
if dbok == False:
logging.error("Error: Not able to connect to MySQL! Please Fix it. Exiting now")
exit (1)
ppbaouserdao = PPBaoUserDAO(self.ppddao)
# The below statements need to be done for each user
(ppduserid_db,ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error("Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!" %(ppdloginid))
exit (2)
self.ppdloginids.append(ppdloginid)
self.ppdid_to_pwd[ppdloginid] = ppdpasswd
strategy_plus = BidStrategyPlus(ppbao_config)
self.ppdid_to_bidstrategy[ppdloginid] = strategy_plus
spider = PPDSpider(ppdloginid, ppbao_config)
self.ppdid_to_spider[ppdloginid] = spider
self.ppdid_to_keepmoney[ppdloginid] = ppbao_config.get_keep_money()
logging.info("%s: Account Minimal Keep Money: %d" % (ppdloginid, self.ppdid_to_keepmoney[ppdloginid]))
'''
Just one instance for below members
'''
self.follower = PPBaoFollower(self.ppdid_to_spider[self.ppdloginids[0]])
self.ppd_parser = PPDHtmlParser()
self.adventurer = PPBaoAdventurer(self.ppdid_to_spider[self.ppdloginids[-1]], self.ppd_parser)
self.autobid = AutoBid()
self.loandao = PPDLoanDAO(self.ppddao)
self.userdao = PPDUserDAO(self.ppddao)
self.mybiddao = MyBidDAO(self.ppddao)
self.blacklistdao = BlackListDAO(self.ppddao)
self.loanids_in_memory = self.loandao.get_last_n_days_loanids(3)
university_to_rank = UniversityDAO(self.ppddao).get_university_ranks()
if university_to_rank is None:
logging.error("Error: Not able to query DB to get University Information. Exiting now")
exit (3)
else:
PPBaoUtil.set_university_to_rank(university_to_rank)
pass
class BidStrategyRegression(object):
ppbao_config = None
loandao = None
userdao = None
bid_strategy = None
date_to_bids = {}
date_to_loans = {}
strategy_to_bids = {}
def __init__(self, config):
ppbao_config = PPBaoConfig(config)
ppdloginid, dbhost, dbuser, dbpwd, dbname = ppbao_config.read_ppbao_config(
)
PPBaoUtil.init_logging('test_bid_strategy', ppbao_config.logdir)
ppddao = PPDDAO({
'host': dbhost,
'username': dbuser,
'password': dbpwd,
'database': dbname
})
ppddao.connect()
self.loandao = PPDLoanDAO(ppddao)
self.userdao = PPDUserDAO(ppddao)
unidao = UniversityDAO(ppddao)
PPBaoUtil.set_university_to_rank(unidao.get_university_ranks())
self.bid_strategy = BidStrategyPlus(ppbao_config)
self.date_to_bids = {}
def check_one_loan(self, loanid):
ppdloan = self.loandao.get_db_ppdloan_by_loanid(loanid, self.userdao)
ifbid, bidmoney, reason, bid_strategy = self.bid_strategy.check_by_strategy(
ppdloan)
if (ifbid == True):
print "Bid! %s - %s" % (ppdloan.get_loan_summary(),
bid_strategy.strategy_name)
else:
print "No Bid: %s" % (ppdloan.get_loan_summary())
def post_run(self):
self.loandao.dao.disconnect()
def init(self):
"""Init all he global variables and components of PPBao System
Connect to local MySQL DB, and init bid strategies, and spider
"""
for conf in self.config_files:
ppbao_config = PPBaoConfig(conf)
ppdloginid, dbhost, dbuser, dbpwd, dbname = ppbao_config.read_ppbao_config(
)
# Only do this when ppddao is None as those are common to all PPBao Users
if (self.ppddao == None):
PPBaoUtil.init_logging('new', ppbao_config.logdir)
logging.info("Welcome to PPBao MT System!")
logging.info(
"Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017"
)
logging.info("PPBao Config: %s,%s,%s,%s,%s" %
(ppdloginid, dbhost, dbuser, dbpwd, dbname))
''' Init DB Modules '''
self.ppddao = PPDDAO({
'host': dbhost,
'username': dbuser,
'password': dbpwd,
'database': dbname
})
dbok = self.ppddao.connect()
if dbok == False:
logging.error(
"Error: Not able to connect to MySQL! Please Fix it. Exiting now"
)
exit(1)
ppbaouserdao = PPBaoUserDAO(self.ppddao)
# The below statements need to be done for each user
(ppduserid_db,
ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error(
"Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!"
% (ppdloginid))
exit(2)
self.ppdloginids.append(ppdloginid)
self.ppdid_to_pwd[ppdloginid] = ppdpasswd
strategy_plus = BidStrategyPlus(ppbao_config)
self.ppdid_to_bidstrategy[ppdloginid] = strategy_plus
spider = PPDSpider(ppdloginid, ppbao_config)
self.ppdid_to_spider[ppdloginid] = spider
self.ppdid_to_keepmoney[ppdloginid] = ppbao_config.get_keep_money()
logging.info("%s: Account Minimal Keep Money: %d" %
(ppdloginid, self.ppdid_to_keepmoney[ppdloginid]))
'''
Just one instance for below members
'''
self.follower = PPBaoFollower(
self.ppdid_to_spider[self.ppdloginids[0]])
self.ppd_parser = PPDHtmlParser()
self.adventurer = PPBaoAdventurer(
self.ppdid_to_spider[self.ppdloginids[-1]], self.ppd_parser)
self.autobid = AutoBid()
self.loandao = PPDLoanDAO(self.ppddao)
self.userdao = PPDUserDAO(self.ppddao)
self.mybiddao = MyBidDAO(self.ppddao)
self.blacklistdao = BlackListDAO(self.ppddao)
self.loanids_in_memory = self.loandao.get_last_n_days_loanids(3)
university_to_rank = UniversityDAO(self.ppddao).get_university_ranks()
if university_to_rank is None:
logging.error(
"Error: Not able to query DB to get University Information. Exiting now"
)
exit(3)
else:
PPBaoUtil.set_university_to_rank(university_to_rank)
pass
class PPBaoMT(object):
# TO BE Implemented
config_files = []
ppddao = None
NOBID = False
ppdloginids = [] #
ppdid_to_pwd = {}
ppdid_to_spider = {}
ppdid_to_bidstrategy = {}
ppdid_to_userid = {} # 18616856236 -> pdu2517233537
ppdid_to_leftmoney = {}
ppdid_to_keepmoney = {} # 18616856236 -> 0
ppd_parser = None
error_count = 0
first_page_url = ''
spider = ''
loanids_in_memory = []
ERRORS_BEFORE_RECONNECT = 160
PAGES_FOR_ONE_THREAD = 45
follower = None
adventurer = None
def __init__(self, config_files):
if isinstance(config_files, str) == True:
self.config_files = (config_files, )
elif isinstance(config_files, list) == True or isinstance(
config_files, tuple) == True:
self.config_files = config_files
else:
print "Failed to init PPBao! Wrong PPBao Config file!"
exit(-1)
self.ppdloginids = []
self.ppdid_to_pwd = {}
self.ppdid_to_spider = {}
self.ppdid_to_leftmoney = {}
self.ppdid_to_bidstrategy = {}
self.ppd_parser = None
self.loanids_in_memory = []
self.new_loans = 0
self.follower = None
self.adventurer = None
def init(self):
"""Init all he global variables and components of PPBao System
Connect to local MySQL DB, and init bid strategies, and spider
"""
for conf in self.config_files:
ppbao_config = PPBaoConfig(conf)
ppdloginid, dbhost, dbuser, dbpwd, dbname = ppbao_config.read_ppbao_config(
)
# Only do this when ppddao is None as those are common to all PPBao Users
if (self.ppddao == None):
PPBaoUtil.init_logging('new', ppbao_config.logdir)
logging.info("Welcome to PPBao MT System!")
logging.info(
"Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017"
)
logging.info("PPBao Config: %s,%s,%s,%s,%s" %
(ppdloginid, dbhost, dbuser, dbpwd, dbname))
''' Init DB Modules '''
self.ppddao = PPDDAO({
'host': dbhost,
'username': dbuser,
'password': dbpwd,
'database': dbname
})
dbok = self.ppddao.connect()
if dbok == False:
logging.error(
"Error: Not able to connect to MySQL! Please Fix it. Exiting now"
)
exit(1)
ppbaouserdao = PPBaoUserDAO(self.ppddao)
# The below statements need to be done for each user
(ppduserid_db,
ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error(
"Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!"
% (ppdloginid))
exit(2)
self.ppdloginids.append(ppdloginid)
self.ppdid_to_pwd[ppdloginid] = ppdpasswd
strategy_plus = BidStrategyPlus(ppbao_config)
self.ppdid_to_bidstrategy[ppdloginid] = strategy_plus
spider = PPDSpider(ppdloginid, ppbao_config)
self.ppdid_to_spider[ppdloginid] = spider
self.ppdid_to_keepmoney[ppdloginid] = ppbao_config.get_keep_money()
logging.info("%s: Account Minimal Keep Money: %d" %
(ppdloginid, self.ppdid_to_keepmoney[ppdloginid]))
'''
Just one instance for below members
'''
self.follower = PPBaoFollower(
self.ppdid_to_spider[self.ppdloginids[0]])
self.ppd_parser = PPDHtmlParser()
self.adventurer = PPBaoAdventurer(
self.ppdid_to_spider[self.ppdloginids[-1]], self.ppd_parser)
self.autobid = AutoBid()
self.loandao = PPDLoanDAO(self.ppddao)
self.userdao = PPDUserDAO(self.ppddao)
self.mybiddao = MyBidDAO(self.ppddao)
self.blacklistdao = BlackListDAO(self.ppddao)
self.loanids_in_memory = self.loandao.get_last_n_days_loanids(3)
university_to_rank = UniversityDAO(self.ppddao).get_university_ranks()
if university_to_rank is None:
logging.error(
"Error: Not able to query DB to get University Information. Exiting now"
)
exit(3)
else:
PPBaoUtil.set_university_to_rank(university_to_rank)
pass
def connect_to_ppdai(self):
"""Connect to PPDai for all users
Only return when all users are connected, so it's important that all usernames/passwords are correct.
"""
for ppdid in self.ppdloginids:
ppdpwd = self.ppdid_to_pwd[ppdid]
spider = self.ppdid_to_spider[ppdid]
logging.info("Logging for %s ..." % (ppdid))
(opener, ppduserid) = spider.login_until_success(ppdid, ppdpwd)
if (opener == None or ppduserid == None):
logging.error(
"Error: Not able to get opener or PPDAI Username is None! Exit..."
)
exit(4)
else:
self.ppdid_to_userid[ppdid] = ppduserid
sleep(random.randint(1, 6))
def read_ppdai_pages(self, risk, page_start, page_end, loan_queue):
"""Main Function to Check the loans in each PPDai pages.
Each thread will run this Function for the pages owned by that thread.
"""
last_url = self.first_page_url
new_loans = 0
logging.debug("PPDai Page Walker of %d-%d pages started..." %
(page_start, page_end))
for page in range(page_start, page_end):
pageurl = self.spider.build_loanpage_url(risk, page)
logging.debug("Open page url: %s" % (pageurl))
skipped, loanid_to_mobile, loanid_to_xueli = self.spider.get_loanid_list_from_listing_page(
pageurl, last_url)
last_url = pageurl
if (loanid_to_xueli is None or loanid_to_mobile is None):
self.error_count += 1
st = random.randint(2, 7)
logging.error(
"Can't get loanids! Error Count(%d). Ignore and Continue in %d seconds. Check it Later!"
% (self.error_count, st))
sleep(st)
continue
(new_loan, old_loan) = self.pasrse_loanid_list(
loanid_to_mobile.keys(), pageurl, loan_queue, 'page_walker',
loanid_to_mobile)
new_loans += new_loan
self.new_loans += new_loan
self.old_loans += old_loan
# sleep a few seconds after each page is completed.
sleep(random.randint(1, 7))
logging.debug(
"PPDai Page Walker for %s %d-%d pages completed. Parsed %d new loans."
% (risk, page_start, page_end, new_loans))
def pasrse_loanid_list(self,
loanid_list,
referer_url,
loan_queue,
source,
loanid_to_mobile=None):
"""This function is to go through the loan details page for each loanid one by one
Tried to make this be generic as we'll need to use it for both page worker and good ppdai bidder user followers
"""
(new_loan, old_loan) = (0, 0)
for loanid in loanid_list:
loanurl = self.spider.get_loanurl_by_loanid(loanid)
if (loanid in self.loanids_in_memory):
#logging.debug("Loanid %d is already in DB. Ignore." % (loanid))
old_loan += 1
else:
new_loan += 1
logging.debug("New Loan list: %d" % (loanid))
html = self.spider.open_loan_detail_page(loanurl, referer_url)
referer_url = loanurl
if (html is None):
self.error_count += 1
st = random.randint(3, 7)
logging.error(
"Can't open %s. Error Count:%d. Ignore and Continue in %d seconds."
% (loanurl, self.error_count, st))
sleep(st)
else:
now = datetime.now() # Record Down the current datetime
ppdloan, ppduser, mymoney = self.ppd_parser.parse_loandetail_html(
loanid, now, html)
if ppdloan == None:
if mymoney == None: # if it's -1,then it's just we're too slow as the loan is 100% completed, no error.
self.error_count += 2
logging.error(
"ErrorCount(%d): Not able to parse HTML to get ppdloan. DO CHECK IT. Ignore and Continue for now!"
% (self.error_count))
sleep(random.randint(3, 5))
else:
# This is what we really need
self.ppdid_to_leftmoney[self.ppdloginids[0]] = mymoney
if (loanid_to_mobile != None):
ppduser.mobile_cert = loanid_to_mobile[loanid]
ppdloan.set_source(source)
self.loanids_in_memory.append(loanid)
loan_queue.put(ppdloan)
#logging.info("Put Loan into Queue: %d" % loanid)
sleep(random.randint(3, 6))
return (new_loan, old_loan)
def check_and_bid(self, loan_queue):
"""Check the PPDLoan in loan_queue by the Strategies defined for each user, and bid if any strategy is met.
"""
logging.info("PPBao BidThread is started! Ready to work ^_^")
while (1):
if (loan_queue.empty()):
sleep(random.randint(1, 3))
else:
try:
ppdloan = loan_queue.get(1)
for ppdid in self.ppdloginids:
ppduserid = self.ppdid_to_userid[ppdid]
# AutoBid
ifbid, bidmoney, reason, bid_strategy = self.ppdid_to_bidstrategy[
ppdid].check_by_strategy(ppdloan)
if ifbid == True and bidmoney > 0:
# Reset bidmoney if necessary
if ((bidmoney is not None) and
(self.ppdid_to_leftmoney.has_key(ppdid)) and
((self.ppdid_to_leftmoney[ppdid] -
self.ppdid_to_keepmoney[ppdid]) < bidmoney)):
if ((self.ppdid_to_leftmoney[ppdid] -
self.ppdid_to_keepmoney[ppdid]) < 50):
''' 50 is the minimal number to bid '''
logging.warn(
"%s: NOT ENOUGH MONEY in My Account to Bid(Account Left: %d; Keep: %d; Bid: %d) Will Run without BID!!!"
%
(ppdid, self.ppdid_to_leftmoney[ppdid],
self.ppdid_to_keepmoney[ppdid],
bidmoney))
logging.info(
"%s: No Money to Bid for %d: %s" %
(ppdid, ppdloan.loanid, reason))
bidmoney = 0
continue # No processing further as no money to bid
else:
logging.warn(
"Change to bid %d as we only have this amount of money left in PPDAI Account for %s"
% (self.ppdid_to_leftmoney[ppdid],
ppduserid))
bidmoney = self.ppdid_to_leftmoney[
ppdid] - self.ppdid_to_keepmoney[ppdid]
logging.warn(
"%s: Bid Loanid %d with Money %d. Reason: %s"
% (ppdid, ppdloan.loanid, bidmoney,
reason))
self.error_count = 0
ppdloan.score = bidmoney
if self.NOBID == True:
ppdloan.bid = 0 # override to 0
self.mybiddao.insert_bid_record(
ppdloan.loanid, datetime.now(), 0,
ppduserid, "NoBidMode:" + reason,
bid_strategy.strategy_name)
else:
''' 'Actually bid for it!!! '''
(actual_bid_money,
mymoney_left) = self.autobid.bid(
self.ppdid_to_spider[ppdid].opener,
ppdloan.loanid, ppdloan.maturity,
bidmoney)
if actual_bid_money > 0:
self.mybiddao.insert_bid_record(
ppdloan.loanid, datetime.now(),
actual_bid_money, ppduserid, reason,
bid_strategy.strategy_name)
logging.info(
"DONE Bid %d for loanid %d!!! My Account Left: %4.2f"
% (actual_bid_money, ppdloan.loanid,
mymoney_left))
else:
logging.warn(
"Bid Failed. No Worries. let's keep going!"
)
if (mymoney_left > 0):
self.ppdid_to_leftmoney[
ppdid] = mymoney_left
ppdloan.bid = actual_bid_money
sleep(random.randint(1, 4))
else:
if ppdid == self.ppdloginids[0]:
#logging.info("%s: NoBid: %d: %s" %(ppduserid, loanid, reason))
logging.info("NoBid: %d: %s" %
(ppdloan.loanid, reason))
#self.error_count = 0
self.loandao.insert(ppdloan)
self.userdao.insert_if_not_exists(ppdloan.ppduser)
except Exception, e:
logging.error(
"Encounter Exception in check_and_bid_thread: %r" %
(e))
traceback.print_exc()
sleep(randint(3, 6))
logging.info(
"BidThread is completed! Hard to get to here as it's using while(1)."
)
class PPBaoMT(object):
# TO BE Implemented
config_files = []
ppddao = None
NOBID = False
ppdloginids = [] #
ppdid_to_pwd = {}
ppdid_to_spider = {}
ppdid_to_bidstrategy = {}
ppdid_to_userid = {} # 18616856236 -> pdu2517233537
ppdid_to_leftmoney = {}
ppdid_to_keepmoney = {} # 18616856236 -> 0
ppd_parser = None
error_count = 0
first_page_url = ''
spider = ''
loanids_in_memory = []
ERRORS_BEFORE_RECONNECT = 160
PAGES_FOR_ONE_THREAD = 45
follower = None
adventurer = None
def __init__(self, config_files):
if isinstance(config_files, str) == True:
self.config_files = (config_files,)
elif isinstance(config_files, list) == True or isinstance(config_files, tuple) == True:
self.config_files = config_files
else:
print "Failed to init PPBao! Wrong PPBao Config file!"
exit (-1)
self.ppdloginids = []
self.ppdid_to_pwd = {}
self.ppdid_to_spider = {}
self.ppdid_to_leftmoney = {}
self.ppdid_to_bidstrategy = {}
self.ppd_parser = None
self.loanids_in_memory = []
self.new_loans = 0
self.follower = None
self.adventurer = None
def init(self):
"""Init all he global variables and components of PPBao System
Connect to local MySQL DB, and init bid strategies, and spider
"""
for conf in self.config_files:
ppbao_config = PPBaoConfig(conf)
ppdloginid,dbhost,dbuser,dbpwd,dbname = ppbao_config.read_ppbao_config()
# Only do this when ppddao is None as those are common to all PPBao Users
if (self.ppddao == None):
PPBaoUtil.init_logging('new', ppbao_config.logdir)
logging.info("Welcome to PPBao MT System!")
logging.info("Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017")
logging.info("PPBao Config: %s,%s,%s,%s,%s" % (ppdloginid,dbhost,dbuser,dbpwd,dbname))
''' Init DB Modules '''
self.ppddao = PPDDAO({'host':dbhost,'username':dbuser,'password':dbpwd,'database':dbname})
dbok = self.ppddao.connect()
if dbok == False:
logging.error("Error: Not able to connect to MySQL! Please Fix it. Exiting now")
exit (1)
ppbaouserdao = PPBaoUserDAO(self.ppddao)
# The below statements need to be done for each user
(ppduserid_db,ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error("Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!" %(ppdloginid))
exit (2)
self.ppdloginids.append(ppdloginid)
self.ppdid_to_pwd[ppdloginid] = ppdpasswd
strategy_plus = BidStrategyPlus(ppbao_config)
self.ppdid_to_bidstrategy[ppdloginid] = strategy_plus
spider = PPDSpider(ppdloginid, ppbao_config)
self.ppdid_to_spider[ppdloginid] = spider
self.ppdid_to_keepmoney[ppdloginid] = ppbao_config.get_keep_money()
logging.info("%s: Account Minimal Keep Money: %d" % (ppdloginid, self.ppdid_to_keepmoney[ppdloginid]))
'''
Just one instance for below members
'''
self.follower = PPBaoFollower(self.ppdid_to_spider[self.ppdloginids[0]])
self.ppd_parser = PPDHtmlParser()
self.adventurer = PPBaoAdventurer(self.ppdid_to_spider[self.ppdloginids[-1]], self.ppd_parser)
self.autobid = AutoBid()
self.loandao = PPDLoanDAO(self.ppddao)
self.userdao = PPDUserDAO(self.ppddao)
self.mybiddao = MyBidDAO(self.ppddao)
self.blacklistdao = BlackListDAO(self.ppddao)
self.loanids_in_memory = self.loandao.get_last_n_days_loanids(3)
university_to_rank = UniversityDAO(self.ppddao).get_university_ranks()
if university_to_rank is None:
logging.error("Error: Not able to query DB to get University Information. Exiting now")
exit (3)
else:
PPBaoUtil.set_university_to_rank(university_to_rank)
pass
def connect_to_ppdai(self):
"""Connect to PPDai for all users
Only return when all users are connected, so it's important that all usernames/passwords are correct.
"""
for ppdid in self.ppdloginids:
ppdpwd = self.ppdid_to_pwd[ppdid]
spider = self.ppdid_to_spider[ppdid]
logging.info("Logging for %s ..." % (ppdid))
(opener, ppduserid) = spider.login_until_success(ppdid, ppdpwd)
if (opener == None or ppduserid == None):
logging.error("Error: Not able to get opener or PPDAI Username is None! Exit...")
exit(4)
else:
self.ppdid_to_userid[ppdid] = ppduserid
sleep(random.randint(1,6))
def read_ppdai_pages(self, risk, page_start, page_end, loan_queue):
"""Main Function to Check the loans in each PPDai pages.
Each thread will run this Function for the pages owned by that thread.
"""
last_url = self.first_page_url
new_loans = 0
logging.debug("PPDai Page Walker of %d-%d pages started..." % (page_start, page_end))
for page in range(page_start, page_end):
pageurl = self.spider.build_loanpage_url(risk, page)
logging.debug("Open page url: %s" % (pageurl))
skipped, loanid_to_mobile, loanid_to_xueli = self.spider.get_loanid_list_from_listing_page(pageurl, last_url)
last_url = pageurl
if (loanid_to_xueli is None or loanid_to_mobile is None):
self.error_count += 1
st = random.randint(2, 7)
logging.error("Can't get loanids! Error Count(%d). Ignore and Continue in %d seconds. Check it Later!" % (self.error_count, st))
sleep(st)
continue
(new_loan, old_loan) = self.pasrse_loanid_list(loanid_to_mobile.keys(), pageurl, loan_queue, 'page_walker', loanid_to_mobile)
new_loans += new_loan
self.new_loans += new_loan
self.old_loans += old_loan
# sleep a few seconds after each page is completed.
sleep(random.randint(1,7))
logging.debug("PPDai Page Walker for %s %d-%d pages completed. Parsed %d new loans." % (risk, page_start, page_end, new_loans))
def pasrse_loanid_list(self, loanid_list, referer_url, loan_queue, source, loanid_to_mobile=None):
"""This function is to go through the loan details page for each loanid one by one
Tried to make this be generic as we'll need to use it for both page worker and good ppdai bidder user followers
"""
(new_loan, old_loan) = (0, 0)
for loanid in loanid_list:
loanurl = self.spider.get_loanurl_by_loanid(loanid)
if (loanid in self.loanids_in_memory):
#logging.debug("Loanid %d is already in DB. Ignore." % (loanid))
old_loan += 1
else:
new_loan += 1
logging.debug("New Loan list: %d" % (loanid))
html = self.spider.open_loan_detail_page(loanurl, referer_url)
referer_url = loanurl
if (html is None):
self.error_count += 1
st = random.randint(3,7)
logging.error("Can't open %s. Error Count:%d. Ignore and Continue in %d seconds." %(loanurl, self.error_count, st))
sleep(st)
else:
now = datetime.now() # Record Down the current datetime
ppdloan, ppduser, mymoney = self.ppd_parser.parse_loandetail_html(loanid, now, html)
if ppdloan == None:
if mymoney == None: # if it's -1,then it's just we're too slow as the loan is 100% completed, no error.
self.error_count += 2
logging.error("ErrorCount(%d): Not able to parse HTML to get ppdloan. DO CHECK IT. Ignore and Continue for now!" % (self.error_count))
sleep(random.randint(3,5))
else:
# This is what we really need
self.ppdid_to_leftmoney[self.ppdloginids[0]] = mymoney
if (loanid_to_mobile != None):
ppduser.mobile_cert = loanid_to_mobile[loanid]
ppdloan.set_source(source)
self.loanids_in_memory.append(loanid)
loan_queue.put(ppdloan)
#logging.info("Put Loan into Queue: %d" % loanid)
sleep(random.randint(3,6))
return (new_loan, old_loan)
def check_and_bid(self, loan_queue):
"""Check the PPDLoan in loan_queue by the Strategies defined for each user, and bid if any strategy is met.
"""
logging.info("PPBao BidThread is started! Ready to work ^_^")
while (1):
if (loan_queue.empty()):
sleep(random.randint(1,3))
else:
try:
ppdloan = loan_queue.get(1)
for ppdid in self.ppdloginids:
ppduserid = self.ppdid_to_userid[ppdid]
# AutoBid
ifbid, bidmoney, reason, bid_strategy = self.ppdid_to_bidstrategy[ppdid].check_by_strategy(ppdloan)
if ifbid == True and bidmoney > 0:
# Reset bidmoney if necessary
if ((bidmoney is not None) and (self.ppdid_to_leftmoney.has_key(ppdid)) and ((self.ppdid_to_leftmoney[ppdid]-self.ppdid_to_keepmoney[ppdid]) < bidmoney)):
if ((self.ppdid_to_leftmoney[ppdid]-self.ppdid_to_keepmoney[ppdid]) < 50):
''' 50 is the minimal number to bid '''
logging.warn("%s: NOT ENOUGH MONEY in My Account to Bid(Account Left: %d; Keep: %d; Bid: %d) Will Run without BID!!!" % (ppdid, self.ppdid_to_leftmoney[ppdid], self.ppdid_to_keepmoney[ppdid], bidmoney))
logging.info("%s: No Money to Bid for %d: %s" % (ppdid, ppdloan.loanid, reason))
bidmoney = 0
continue # No processing further as no money to bid
else:
logging.warn("Change to bid %d as we only have this amount of money left in PPDAI Account for %s" %(self.ppdid_to_leftmoney[ppdid], ppduserid))
bidmoney = self.ppdid_to_leftmoney[ppdid] -self.ppdid_to_keepmoney[ppdid]
logging.warn("%s: Bid Loanid %d with Money %d. Reason: %s" %(ppdid, ppdloan.loanid, bidmoney, reason))
self.error_count = 0
ppdloan.score = bidmoney
if self.NOBID == True:
ppdloan.bid = 0 # override to 0
self.mybiddao.insert_bid_record(ppdloan.loanid, datetime.now(), 0, ppduserid, "NoBidMode:" + reason, bid_strategy.strategy_name)
else:
''' 'Actually bid for it!!! '''
(actual_bid_money, mymoney_left) = self.autobid.bid(self.ppdid_to_spider[ppdid].opener, ppdloan.loanid, ppdloan.maturity, bidmoney)
if actual_bid_money > 0:
self.mybiddao.insert_bid_record(ppdloan.loanid, datetime.now(), actual_bid_money, ppduserid, reason, bid_strategy.strategy_name)
logging.info("DONE Bid %d for loanid %d!!! My Account Left: %4.2f" % (actual_bid_money, ppdloan.loanid, mymoney_left))
else:
logging.warn("Bid Failed. No Worries. let's keep going!")
if (mymoney_left > 0):
self.ppdid_to_leftmoney[ppdid] = mymoney_left
ppdloan.bid = actual_bid_money
sleep(random.randint(1,4))
else:
if ppdid == self.ppdloginids[0]:
#logging.info("%s: NoBid: %d: %s" %(ppduserid, loanid, reason))
logging.info("NoBid: %d: %s" %(ppdloan.loanid, reason))
#self.error_count = 0
self.loandao.insert(ppdloan)
self.userdao.insert_if_not_exists(ppdloan.ppduser)
except Exception, e:
logging.error("Encounter Exception in check_and_bid_thread: %r" % (e))
traceback.print_exc()
sleep(randint(3,6))
logging.info("BidThread is completed! Hard to get to here as it's using while(1).")
class PPBao(object):
# TO BE Implemented
config_files = []
ppddao = None
NOBID = False
ppdloginids = [] #
ppdid_to_pwd = {}
ppdid_to_spider = {}
ppdid_to_bidstrategy = {}
ppdid_to_userid = {} # 18616856236 -> pdu2517233537
ppdid_to_leftmoney = {}
ppd_parser = None
def __init__(self, config_files):
if isinstance(config_files, str) == True:
self.config_files = (config_files,)
elif isinstance(config_files, list) == True or isinstance(config_files, tuple) == True:
self.config_files = config_files
else:
print "Failed to init PPBao! Wrong PPBao Config file!"
exit (-1)
self.ppdloginids = []
self.ppdid_to_pwd = {}
self.ppdid_to_spider = {}
self.ppdid_to_leftmoney = {}
self.ppdid_to_bidstrategy = {}
self.ppd_parser = None
def init(self):
for conf in self.config_files:
ppbao_config = PPBaoConfig(conf)
ppdloginid,dbhost,dbuser,dbpwd,dbname = ppbao_config.read_ppbao_config()
# Only do this when ppddao is None as those are common to all PPBao Users
if (self.ppddao == None):
PPBaoUtil.init_logging('new', ppbao_config.logdir)
logging.info("Welcome to PPBao System!")
logging.info("Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017")
logging.info("PPBao Config: %s,%s,%s,%s,%s" % (ppdloginid,dbhost,dbuser,dbpwd,dbname))
''' Init DB Modules '''
self.ppddao = PPDDAO({'host':dbhost,'username':dbuser,'password':dbpwd,'database':dbname})
dbok = self.ppddao.connect()
if dbok == False:
logging.error("Error: Not able to connect to MySQL! Please Fix it. Exiting now")
exit (1)
ppbaouserdao = PPBaoUserDAO(self.ppddao)
# The below statements need to be done for each user
(ppduserid_db,ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error("Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!" %(ppdloginid))
exit (2)
self.ppdloginids.append(ppdloginid)
self.ppdid_to_pwd[ppdloginid] = ppdpasswd
strategy_plus = BidStrategyPlus(ppbao_config)
self.ppdid_to_bidstrategy[ppdloginid] = strategy_plus
spider = PPDSpider(ppdloginid, ppbao_config)
self.ppdid_to_spider[ppdloginid] = spider
self.ppd_parser = PPDHtmlParser()
self.autobid = AutoBid()
self.loandao = PPDLoanDAO(self.ppddao)
self.userdao = PPDUserDAO(self.ppddao)
self.mybiddao = MyBidDAO(self.ppddao)
self.blacklistdao = BlackListDAO(self.ppddao)
university_to_rank = UniversityDAO(self.ppddao).get_university_ranks()
if university_to_rank is None:
logging.error("Error: Not able to query DB to get University Information. Exiting now")
exit (3)
else:
PPBaoUtil.set_university_to_rank(university_to_rank)
pass
def connect_to_ppdai(self):
for ppdid in self.ppdloginids:
ppdpwd = self.ppdid_to_pwd[ppdid]
spider = self.ppdid_to_spider[ppdid]
logging.info("Logging for %s ..." % (ppdid))
(opener, ppduserid) = spider.login_until_success(ppdid, ppdpwd)
if (opener == None or ppduserid == None):
logging.error("Error: Not able to get opener or PPDAI Username is None! Exit...")
exit(4)
else:
self.ppdid_to_userid[ppdid] = ppduserid
sleep(random.randint(2,6))
def read_pages(self, page_start, page_end):
pass
def check_loans(self):
pass
def run(self):
rd = 1 # Round
error_count = 0 # This record down how many errors we have during the run.
last_url = PPDSpider.get_login_url() # This is used to record the last URL to use as Referer
mybid_list = [] # mybid list
ppd_main_id = self.ppdloginids[0]
spider = self.ppdid_to_spider[ppd_main_id]
loanids_in_memory = self.loandao.get_last_2_days_loanids()
while (1):
''' 20160304: Add AutoLogin after 20 Errors so as we can recover from unexpected error/exceptions '''
loanids_in_this_round = []
if error_count >= 20:
self.connect_to_ppdai()
sleep(random.randint(5,20))
for risk in [spider.riskmiddle]: # 20160307: remove: spider.risksafe as I already bid more than 3000 for 12/12 Peibiao
try:
first_page_url = spider.build_loanpage_url(risk, 1)
count, pages,skipped, loanid_to_mobile, loanid_to_xueli = spider.get_pages(first_page_url, last_url)
last_url = first_page_url
old_loans, new_loans, skipped_loans = (0, 0, skipped) # to record how many old/new and skipped loans in this round
if count == 0:
logging.info("No Loan of risktype %s is available. Next..." % risk)
sleep(random.randint(1,4))
continue;
elif count < 0: # -1 means we encountered an error
logging.error("Error: Not able to open %s to get total loans and pages.")
error_count += 5;
continue;
else:
logging.info("****** Round %d ****** Total Loans: %d (%d pages). Checking..." % (rd, count, pages))
sleep(random.randint(1,2))
''' Notice range (1,2) will only returns 1, so need pages +1 '''
for index in range(1,pages+1):
if (index > 1):
pageurl = spider.build_loanpage_url(risk, index)
logging.debug("Open page url: %s" % (pageurl))
skipped, loanid_to_mobile, loanid_to_xueli = spider.get_loanid_list_from_listing_page(pageurl,last_url)
last_url = pageurl
if (loanid_to_xueli is None or loanid_to_mobile is None):
error_count += 1
st = random.randint(2,7)
logging.error("Can't get loanids! Error Count(%d). Ignore and Continue in %d seconds. Check it Later!" % (error_count, st))
sleep(st)
continue
skipped_loans += skipped
else:
pageurl = spider.build_loanpage_url(risk, 1)
last_url = pageurl
for loanid in loanid_to_mobile.keys():
loanurl = spider.get_loanurl_by_loanid(loanid)
loanids_in_this_round.append(loanid)
if (loanid in loanids_in_memory):
logging.debug("Loanid %d is already in DB. Ignore." % (loanid))
old_loans += 1
continue
else:
new_loans += 1
logging.debug("New Loan list: %d" % (loanid))
html = spider.open_loan_detail_page(loanurl, pageurl)
last_url = loanurl
if (html is None):
error_count += 1
st = random.randint(2,7)
logging.error("Can't open %s. Error Count:%d. Ignore and Continue in %d seconds." %(loanurl, error_count, st))
sleep(st)
continue
now = datetime.now() # Record Down the current datetime
ppdloan, ppduser, mymoney = self.ppd_parser.parse_loandetail_html(loanid, now, html)
if ppdloan == None:
if mymoney == None: # if it's -1,then it's just we're too slow as the loan is 100% completed, no error.
error_count += 2
logging.error("ErrorCount(%d): Not able to parse HTML to get ppdloan. DO CHECK IT. Ignore and Continue for now!" % (error_count))
sleep(random.randint(1,5))
continue
else:
ppduser.mobile_cert = loanid_to_mobile[loanid]
self.ppdid_to_leftmoney[ppd_main_id] = mymoney
for ppdid in self.ppdloginids:
ppduserid = self.ppdid_to_userid[ppdid]
# AutoBid
ifbid, bidmoney, reason, bid_strategy = self.ppdid_to_bidstrategy[ppdid].check_by_strategy(ppdloan)
if ((bidmoney is not None) and (self.ppdid_to_leftmoney.has_key(ppdid)) and (self.ppdid_to_leftmoney[ppdid] < bidmoney)):
logging.warn("%s: NOT ENOUGH MONEY in My Account to Bid(%d<%d). Will Run without BID!!!" % (ppdid, mymoney, bidmoney))
logging.info("%s: No Money to Bid for %d: %s" % (ppdid, loanid, reason))
elif ifbid == True:
if loanid not in mybid_list:
mybid_list.append(loanid)
logging.warn("%s: Bid Loanid %d with Money %d (MyAccount Left:%4.2f). Reason: %s" %(ppdid, loanid, bidmoney, mymoney, reason))
ppdloan.score = bidmoney
if self.NOBID == True:
ppdloan.bid = 0 # override to 0
self.mybiddao.insert_bid_record(loanid, now, 0, ppduserid, "NoBidMode:" + reason, bid_strategy.strategy_name)
else:
# Actually bid for it
(actual_bid_money, mymoney_left) = self.autobid.bid(self.ppdid_to_spider[ppdid].opener, loanid, ppdloan.maturity, bidmoney)
if actual_bid_money > 0:
self.mybiddao.insert_bid_record(loanid, now, actual_bid_money, ppduserid, reason, bid_strategy.strategy_name)
logging.info("DONE!!! Bid %d for loanid %d!!!" % (actual_bid_money, loanid))
else:
logging.warn("Bid Failed. No Worries. let's keep going!")
if (mymoney_left > 0):
self.ppdid_to_leftmoney[ppdid] = mymoney_left
ppdloan.bid = actual_bid_money
sleep(random.randint(1,4))
else:
if ppdid == ppd_main_id:
#logging.info("%s: NoBid: %d: %s" %(ppduserid, loanid, reason))
logging.info("NoBid: %d: %s" %(loanid, reason))
# Write to MYSQL
loanids_in_memory.append(loanid)
self.loandao.insert(ppdloan)
self.userdao.insert_if_not_exists(ppduser)
# If we can reach here, means everything is fine. We shall reset error_count = 0
error_count = 0
if ppdloan.loanrate < 10: # No need to check more Loans as we've sorted the pages
sleep(random.randint(1,3))
break
else:
sleep(random.randint(1,3))
sleep(random.randint(1,4))
# End of parsing all the pages
logging.info("Parsed %d loans: new_loans(%d), old_loans(%d), skipped_loans(%d)" % (len(loanids_in_this_round), new_loans, old_loans, skipped_loans))
except Exception, e:
error_count += 1
logging.error("Un-Caught Error!!! Continue with next round - Please do Check IT!! %r" %(e))
traceback.print_exc()
sleep(random.randint(6,12))
# Reset loanids_in_memory as we don't need to keep history old loanids.
''' Seems there is a BUG which will cause duplicate BIDs if we reset loanids in memory everytime'''
''' Probably caused by ppdai not stable caused it read the page url fail and miss some Loans '''
''' Solution is to only reset on Round 1!!! '''
if (rd == 1 and len(loanids_in_this_round) > 1):
loanids_in_memory = loanids_in_this_round
if (rd == 1 or (rd % 120 == 0)):
sleep(1)
logging.info("Updating Black List...")
self.update_black_list()
if (rd % 20 == 0):
rdint = random.randint(20,60) # Sleep more time on every 20 round.
elif (rd % 100 == 0):
rdint = random.randint(30,90)
elif (rd % 400 == 0):
rdint = random.randint(90,600)
else:
rdint = random.randint(6,20)
logging.info("****** Done Round %d ****** Sleep for %d seconds before next run." % (rd, rdint))
rd += 1
sleep(rdint)