def init_ppbao(ppbao_config_file, ppbaouserdao):
# Initialize
ppbao_config = PPBaoConfig(ppbao_config_file)
ppdloginid, dbhost, dbuser, dbpwd, dbname = ppbao_config.read_ppbao_config(
)
PPBaoUtil.init_logging(ppdloginid, ppbao_config.logdir)
logging.info("Welcome to PPBao System - Update BlackList Utility!")
logging.info("Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017")
logging.info("PPBao Config: %s,%s,%s,%s,%s" %
(ppdloginid, dbhost, dbuser, dbpwd, dbname))
(ppduserid_db,
ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error(
"Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!"
% (ppdloginid))
exit(2)
# Login to PPDAI!
spider = PPDSpider(ppdloginid, ppbao_config)
(opener, ppduserid) = spider.login(ppdloginid, ppdpasswd)
if (ppduserid == None or ppduserid != ppduserid_db):
logging.error(
"Error: Not able to get PPDAI Username or is not consistent with that in DB! Exit..."
)
exit(4)
return (ppduserid, spider)
def init_ppbao(argv):
ppbao_config_file = None
if (len(argv) == 1):
ppbao_config_file = "conf/ppbao.18616856236.config"
ppbao_config_file = "conf/ppbao.18616027065.config"
elif (len(argv) == 2):
me, ppbao_config_file = argv
else:
print "Error: More than 1 argument is provided!"
print "Usage: python update_blacklist.py <ppbao_config_file>"
exit(-1)
# Initialize
ppbao_config = PPBaoConfig(ppbao_config_file)
ppdloginid, dbhost, dbuser, dbpwd, dbname = ppbao_config.read_ppbao_config(
)
PPBaoUtil.init_logging(ppdloginid, ppbao_config.logdir)
logging.info("Welcome to PPBao System - Update BlackList Utility!")
logging.info("Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017")
logging.info("PPBao Config: %s,%s,%s,%s,%s" %
(ppdloginid, dbhost, dbuser, dbpwd, dbname))
# Init DB Modules
ppddao = PPDDAO({
'host': dbhost,
'username': dbuser,
'password': dbpwd,
'database': dbname
})
dbok = ppddao.connect()
if dbok == False:
logging.error(
"Error: Not able to connect to MySQL! Please Fix it. Exiting now")
exit(1)
ppbaouserdao = PPBaoUserDAO(ppddao)
blacklistdao = BlackListDAO(ppddao)
(ppduserid_db,
ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error(
"Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!"
% (ppdloginid))
exit(2)
# Login to PPDAI!
spider = PPDSpider(ppdloginid, ppbao_config)
(opener, ppduserid) = spider.login(ppdloginid, ppdpasswd)
if (ppduserid == None or ppduserid != ppduserid_db):
logging.error(
"Error: Not able to get PPDAI Username or is not consistent with that in DB! Exit..."
)
exit(4)
return (ppduserid, spider, blacklistdao)
def init(self):
"""Init all he global variables and components of PPBao System
Connect to local MySQL DB, and init bid strategies, and spider
"""
for conf in self.config_files:
ppbao_config = PPBaoConfig(conf)
ppdloginid,dbhost,dbuser,dbpwd,dbname = ppbao_config.read_ppbao_config()
# Only do this when ppddao is None as those are common to all PPBao Users
if (self.ppddao == None):
PPBaoUtil.init_logging('new', ppbao_config.logdir)
logging.info("Welcome to PPBao MT System!")
logging.info("Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017")
logging.info("PPBao Config: %s,%s,%s,%s,%s" % (ppdloginid,dbhost,dbuser,dbpwd,dbname))
''' Init DB Modules '''
self.ppddao = PPDDAO({'host':dbhost,'username':dbuser,'password':dbpwd,'database':dbname})
dbok = self.ppddao.connect()
if dbok == False:
logging.error("Error: Not able to connect to MySQL! Please Fix it. Exiting now")
exit (1)
ppbaouserdao = PPBaoUserDAO(self.ppddao)
# The below statements need to be done for each user
(ppduserid_db,ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error("Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!" %(ppdloginid))
exit (2)
self.ppdloginids.append(ppdloginid)
self.ppdid_to_pwd[ppdloginid] = ppdpasswd
strategy_plus = BidStrategyPlus(ppbao_config)
self.ppdid_to_bidstrategy[ppdloginid] = strategy_plus
spider = PPDSpider(ppdloginid, ppbao_config)
self.ppdid_to_spider[ppdloginid] = spider
self.ppdid_to_keepmoney[ppdloginid] = ppbao_config.get_keep_money()
logging.info("%s: Account Minimal Keep Money: %d" % (ppdloginid, self.ppdid_to_keepmoney[ppdloginid]))
'''
Just one instance for below members
'''
self.follower = PPBaoFollower(self.ppdid_to_spider[self.ppdloginids[0]])
self.ppd_parser = PPDHtmlParser()
self.adventurer = PPBaoAdventurer(self.ppdid_to_spider[self.ppdloginids[-1]], self.ppd_parser)
self.autobid = AutoBid()
self.loandao = PPDLoanDAO(self.ppddao)
self.userdao = PPDUserDAO(self.ppddao)
self.mybiddao = MyBidDAO(self.ppddao)
self.blacklistdao = BlackListDAO(self.ppddao)
self.loanids_in_memory = self.loandao.get_last_n_days_loanids(3)
university_to_rank = UniversityDAO(self.ppddao).get_university_ranks()
if university_to_rank is None:
logging.error("Error: Not able to query DB to get University Information. Exiting now")
exit (3)
else:
PPBaoUtil.set_university_to_rank(university_to_rank)
pass
def testName(self):
ppbao_config_file = "../conf/ppbao.me.config"
# Initialize
ppbao_config = PPBaoConfig(ppbao_config_file)
ppdloginid,dbhost,dbuser,dbpwd,dbname = ppbao_config.read_ppbao_config()
init_logging(ppdloginid)
ppbao_config.print_strategies()
bsp = BidStrategyPlus(ppbao_config)
# Init DB Modules
from dao.PPDDAO import PPDDAO
from dao.UniversityDAO import UniversityDAO
from util.PPBaoUtil import PPBaoUtil
ppddao = PPDDAO({'host':dbhost,'username':dbuser,'password':dbpwd,'database':dbname})
dbok = ppddao.connect()
if dbok == False:
logging.error("Error: Not able to connect to MySQL! Please Fix it. Exiting now")
exit (1)
university_to_rank = UniversityDAO(ppddao).get_university_ranks()
if university_to_rank is None:
logging.error("Error: Not able to query DB to get University Information. Exiting now")
exit (3)
else:
PPBaoUtil.set_university_to_rank(university_to_rank)
" Build Test PPDLoan and PPDUser instance"
from ds.PPDLoan import PPDLoan
from ds.PPDUser import PPDUser
from datetime import datetime
now = datetime.now()
loanid = '1111111'
ppdloan = PPDLoan({'loanid':1111111, 'datetime':now, 'loanrate':22, 'ppdrate':'C', \
'money':5000, 'maturity':12, 'userid':'pdu2517233537', 'age': 31})
ppduser = PPDUser({'userid':'pdu2517233537', 'gender': '男', 'age': 31, 'marriage': '已婚', \
'house': '有', 'car': '无', 'education_level': '本科'})
ppdloan.set_ppduser(ppduser)
ppdloan.set_history_info(8,0,0,12000,3000,0)
ppdloan.history_highest_total_loan = 12000
ppdloan.loantitle = "NA-Test"
ppduser.add_education_cert('保定学院', '本科', '普通')
logging.info(ppdloan.get_loan_summary())
ifbid, money, reason = bsp.check_by_strategy(ppdloan)
self.assertTrue(ifbid, "No Bid for master??")
def __init__(self, config):
ppbao_config = PPBaoConfig(config)
ppdloginid, dbhost, dbuser, dbpwd, dbname = ppbao_config.read_ppbao_config(
)
PPBaoUtil.init_logging('test_bid_strategy', ppbao_config.logdir)
ppddao = PPDDAO({
'host': dbhost,
'username': dbuser,
'password': dbpwd,
'database': dbname
})
ppddao.connect()
self.loandao = PPDLoanDAO(ppddao)
self.userdao = PPDUserDAO(ppddao)
unidao = UniversityDAO(ppddao)
PPBaoUtil.set_university_to_rank(unidao.get_university_ranks())
self.bid_strategy = BidStrategyPlus(ppbao_config)
self.date_to_bids = {}
def init_ppbao(argv):
ppbao_config_file = None
if (len(argv) == 1):
ppbao_config_file = "../conf/ppbao.18616856236.config"
ppbao_config_file = "../conf/ppbao.18616027065.config"
elif (len(argv) == 2):
me,ppbao_config_file = argv
else:
print "Error: More than 1 argument is provided!"
print "Usage: python update_blacklist.py <ppbao_config_file>"
exit (-1)
# Initialize
ppbao_config = PPBaoConfig(ppbao_config_file)
ppdloginid,dbhost,dbuser,dbpwd,dbname = ppbao_config.read_ppbao_config()
PPBaoUtil.init_logging(ppdloginid,ppbao_config.logdir)
logging.info("Welcome to PPBao System - Update BlackList Utility!")
logging.info("Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017")
logging.info("PPBao Config: %s,%s,%s,%s,%s" % (ppdloginid,dbhost,dbuser,dbpwd,dbname))
# Init DB Modules
ppddao = PPDDAO({'host':dbhost,'username':dbuser,'password':dbpwd,'database':dbname})
dbok = ppddao.connect()
if dbok == False:
logging.error("Error: Not able to connect to MySQL! Please Fix it. Exiting now")
exit (1)
ppbaouserdao = PPBaoUserDAO(ppddao)
blacklistdao = BlackListDAO(ppddao)
(ppduserid_db,ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error("Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!" %(ppdloginid))
exit (2)
# Login to PPDAI!
spider = PPDSpider(ppdloginid, ppbao_config)
(opener, ppduserid) = spider.login(ppdloginid, ppdpasswd)
if (ppduserid == None or ppduserid != ppduserid_db):
logging.error("Error: Not able to get PPDAI Username or is not consistent with that in DB! Exit...")
exit(4)
return (ppduserid, spider, blacklistdao)
def init(self):
for conf in self.config_files:
ppbao_config = PPBaoConfig(conf)
ppdloginid,dbhost,dbuser,dbpwd,dbname = ppbao_config.read_ppbao_config()
# Only do this when ppddao is None as those are common to all PPBao Users
if (self.ppddao == None):
PPBaoUtil.init_logging('new', ppbao_config.logdir)
logging.info("Welcome to PPBao System!")
logging.info("Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017")
logging.info("PPBao Config: %s,%s,%s,%s,%s" % (ppdloginid,dbhost,dbuser,dbpwd,dbname))
''' Init DB Modules '''
self.ppddao = PPDDAO({'host':dbhost,'username':dbuser,'password':dbpwd,'database':dbname})
dbok = self.ppddao.connect()
if dbok == False:
logging.error("Error: Not able to connect to MySQL! Please Fix it. Exiting now")
exit (1)
ppbaouserdao = PPBaoUserDAO(self.ppddao)
# The below statements need to be done for each user
(ppduserid_db,ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error("Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!" %(ppdloginid))
exit (2)
self.ppdloginids.append(ppdloginid)
self.ppdid_to_pwd[ppdloginid] = ppdpasswd
strategy_plus = BidStrategyPlus(ppbao_config)
self.ppdid_to_bidstrategy[ppdloginid] = strategy_plus
spider = PPDSpider(ppdloginid, ppbao_config)
self.ppdid_to_spider[ppdloginid] = spider
self.ppd_parser = PPDHtmlParser()
self.autobid = AutoBid()
self.loandao = PPDLoanDAO(self.ppddao)
self.userdao = PPDUserDAO(self.ppddao)
self.mybiddao = MyBidDAO(self.ppddao)
self.blacklistdao = BlackListDAO(self.ppddao)
university_to_rank = UniversityDAO(self.ppddao).get_university_ranks()
if university_to_rank is None:
logging.error("Error: Not able to query DB to get University Information. Exiting now")
exit (3)
else:
PPBaoUtil.set_university_to_rank(university_to_rank)
pass
def init_ppbao(ppbao_config_file, ppbaouserdao):
# Initialize
ppbao_config = PPBaoConfig(ppbao_config_file)
ppdloginid,dbhost,dbuser,dbpwd,dbname = ppbao_config.read_ppbao_config()
PPBaoUtil.init_logging(ppdloginid,ppbao_config.logdir)
logging.info("Welcome to PPBao System - Update BlackList Utility!")
logging.info("Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017")
logging.info("PPBao Config: %s,%s,%s,%s,%s" % (ppdloginid,dbhost,dbuser,dbpwd,dbname))
(ppduserid_db,ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error("Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!" %(ppdloginid))
exit (2)
# Login to PPDAI!
spider = PPDSpider(ppdloginid, ppbao_config)
(opener, ppduserid) = spider.login(ppdloginid, ppdpasswd)
if (ppduserid == None or ppduserid != ppduserid_db):
logging.error("Error: Not able to get PPDAI Username or is not consistent with that in DB! Exit...")
exit(4)
return (ppduserid, spider)
def init(self):
"""Init all he global variables and components of PPBao System
Connect to local MySQL DB, and init bid strategies, and spider
"""
for conf in self.config_files:
ppbao_config = PPBaoConfig(conf)
ppdloginid, dbhost, dbuser, dbpwd, dbname = ppbao_config.read_ppbao_config(
)
# Only do this when ppddao is None as those are common to all PPBao Users
if (self.ppddao == None):
PPBaoUtil.init_logging('new', ppbao_config.logdir)
logging.info("Welcome to PPBao MT System!")
logging.info(
"Developed By Xiaoqi Ouyang. All Rights Reserved@2016-2017"
)
logging.info("PPBao Config: %s,%s,%s,%s,%s" %
(ppdloginid, dbhost, dbuser, dbpwd, dbname))
''' Init DB Modules '''
self.ppddao = PPDDAO({
'host': dbhost,
'username': dbuser,
'password': dbpwd,
'database': dbname
})
dbok = self.ppddao.connect()
if dbok == False:
logging.error(
"Error: Not able to connect to MySQL! Please Fix it. Exiting now"
)
exit(1)
ppbaouserdao = PPBaoUserDAO(self.ppddao)
# The below statements need to be done for each user
(ppduserid_db,
ppdpasswd) = ppbaouserdao.get_ppduserid_and_passwd(ppdloginid)
if (ppduserid_db is None or ppdpasswd is None):
logging.error(
"Error: Not able to get PPDAI loginid/passwd for %s. Invalid PPBao User!! Exiting!"
% (ppdloginid))
exit(2)
self.ppdloginids.append(ppdloginid)
self.ppdid_to_pwd[ppdloginid] = ppdpasswd
strategy_plus = BidStrategyPlus(ppbao_config)
self.ppdid_to_bidstrategy[ppdloginid] = strategy_plus
spider = PPDSpider(ppdloginid, ppbao_config)
self.ppdid_to_spider[ppdloginid] = spider
self.ppdid_to_keepmoney[ppdloginid] = ppbao_config.get_keep_money()
logging.info("%s: Account Minimal Keep Money: %d" %
(ppdloginid, self.ppdid_to_keepmoney[ppdloginid]))
'''
Just one instance for below members
'''
self.follower = PPBaoFollower(
self.ppdid_to_spider[self.ppdloginids[0]])
self.ppd_parser = PPDHtmlParser()
self.adventurer = PPBaoAdventurer(
self.ppdid_to_spider[self.ppdloginids[-1]], self.ppd_parser)
self.autobid = AutoBid()
self.loandao = PPDLoanDAO(self.ppddao)
self.userdao = PPDUserDAO(self.ppddao)
self.mybiddao = MyBidDAO(self.ppddao)
self.blacklistdao = BlackListDAO(self.ppddao)
self.loanids_in_memory = self.loandao.get_last_n_days_loanids(3)
university_to_rank = UniversityDAO(self.ppddao).get_university_ranks()
if university_to_rank is None:
logging.error(
"Error: Not able to query DB to get University Information. Exiting now"
)
exit(3)
else:
PPBaoUtil.set_university_to_rank(university_to_rank)
pass
'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36',
'Connection':'keep-alive'
}
opener = PPBaoUtil.add_headers(opener, headers)
response = opener.open(url,None,15)
html_str = PPBaoUtil.get_html_from_response(response)
response.close()
return html_str;
except Exception,e:
logging.error("On OpenPage - Caught Exception %r" %(e))
return None
else:
print "Not supported for test"
if __name__ == '__main__':
ppbao_config = PPBaoConfig("../conf/ppbao.me.config")
ppdloginid,dbhost,dbuser,dbpwd,dbname = ppbao_config.read_ppbao_config()
PPBaoUtil.init_logging(ppdloginid,ppbao_config.logdir)
ppddao, ppbaouserdao = init_dao(dbhost,dbuser,dbpwd,dbname)
spiders = []
for conf in ('18616856236', '18616027065'):
file = "../conf/ppbao.%s.config" % (conf)
print "File: %s" % (file)
ppduserid, spider = init_ppbao(file, ppbaouserdao)
spiders.append(spider)
sleep(5)
for spider in spiders:
logging.info("Opened HTML for ....")
html = open_blacklist_page(spider.opener, 1)
'Connection': 'keep-alive'
}
opener = PPBaoUtil.add_headers(opener, headers)
response = opener.open(url, None, 15)
html_str = PPBaoUtil.get_html_from_response(response)
response.close()
return html_str
except Exception, e:
logging.error("On OpenPage - Caught Exception %r" % (e))
return None
else:
print "Not supported for test"
if __name__ == '__main__':
ppbao_config = PPBaoConfig("../conf/ppbao.me.config")
ppdloginid, dbhost, dbuser, dbpwd, dbname = ppbao_config.read_ppbao_config(
)
PPBaoUtil.init_logging(ppdloginid, ppbao_config.logdir)
ppddao, ppbaouserdao = init_dao(dbhost, dbuser, dbpwd, dbname)
spiders = []
for conf in ('18616856236', '18616027065'):
file = "../conf/ppbao.%s.config" % (conf)
print "File: %s" % (file)
ppduserid, spider = init_ppbao(file, ppbaouserdao)
spiders.append(spider)
sleep(5)
for spider in spiders:
logging.info("Opened HTML for ....")
html = open_blacklist_page(spider.opener, 1)