def test_new_loan(self):
tempLoan = Loan(
1000000,
'Network 1',
'Mar',
'Loan Product 1',
1000.50,
)
loanProcessor = LoanProcessor(tempLoan)
newLoan = Loan(
1000001,
'Network 1',
'Mar',
'Loan Product 1',
2000.50,
)
loanProcessor.processNewLoan(newLoan)
self.assertEqual(3001, loanProcessor.getAggregateAmount())
newLoan = Loan(
1000002,
'Network 3',
'Mar',
'Loan Product 1',
2000.50,
)
loanProcessor.processNewLoan(newLoan)
self.assertEqual(2000.50, loanProcessor.getAggregateAmount())
def test_reject_loan():
loan = Loan(amount=100_000)
assert not reject_loan(loan).rejected()
loan = Loan(amount=250_001)
assert reject_loan(loan).rejected()
loan = Loan(amount=250_000)
assert not reject_loan(loan).rejected()
def main():
print ('========== Exersize 2.1.4 ==========\n')
print('Testing Loan classmethod for monthly payment : {}'.format(Loan.calcMonthlyPmt(100000, .025, 360)))
print('Testing Loan classmethod for balance: {}'.format(Loan.calcBal(100000, .025, 360, 60)))
myLoan = Loan(360,.025,100000)
print("Monthly Payment: {}".format(myLoan.monthlyPayment()))
print("Balance after 60 periods: {}".format(myLoan.balance(60)))
print("Interest due on period 60: {}".format(myLoan.interestDue(60)))
print("Principal due on period 60: {}".format(myLoan.principlaDue(60)))
print("The total payment should equal interest plus principal which is {}".format(myLoan.interestDue(5) + myLoan.principlaDue(5)))
print("Total Interest paid is {}".format(myLoan.totalInterest()))
print("Total Payment is {}".format(myLoan.totalPayments()))
print("Total Interest paid is {}".format(myLoan.totalInterest()))
"""
The benefit of the cls level method is that it allows us to compute a payment or balance with out initiating an object
"""
print("Old rate {}".format(myLoan.rate))
print("Old term {}".format(myLoan.term))
print("Old face {}".format(myLoan.face))
myLoan.rate = .035
myLoan.term = 60
myLoan.face = 20000
print("New rate {}".format(myLoan.rate))
print("New term {}".format(myLoan.term))
print("New face {}".format(myLoan.face))
def main(args):
db = Database()
if args['initialize']:
samples = get_input()
for sample in samples:
loan = Loan(sample)
loan.generatePaymentSchedule(months=3)
print("Finished populating db!")
elif args['debits']:
debits = db.queryLoanDebits(args['debits'])
if len(debits) == 0:
print(f"No debits found for loan {args['debits']}")
else:
for debit in debits:
debObj = Debit(*list(debit))
pprint(vars(debObj))
elif args['payments']:
payments = db.queryLoanPayments(args['payments'])
if len(payments) == 0:
print(f"No payments found for loan {args['payments']}")
else:
for payment in payments:
payObj = Payment(*list(payment))
pprint(vars(payObj))
elif args['payment_debits']:
debits = db.queryPaymentDebits(args['payment_debits'])
if len(debits) == 0:
print(f"No debits found for payment {args['payment_debits']}")
else:
for debit in debits:
debObj = Debit(*list(debit))
pprint(vars(debObj))
else:
print("No action specified!")
def processInput(line):
network, product, month_year, amount = line.split(',')
global loanProcessor
newLoan = Loan('', network, month_year, product, amount)
if loanProcessor is not None:
loanProcessor.processNewLoan(newLoan, reducer=True)
else:
loanProcessor = LoanProcessor(newLoan)
def test_loan_aggregation(self):
tempLoan = Loan(
1000000,
'Network 1',
'Mar',
'Loan Product 1',
1000.50,
)
loanProcessor = LoanProcessor(tempLoan)
newLoan = Loan(
1000000,
'Network 1',
'Mar',
'Loan Product 1',
2000.50,
)
loanProcessor.aggregate(newLoan)
self.assertEqual(3001, loanProcessor.getAggregateAmount())
def retrieve_loan():
ssn = construct_random_ssn()
borrowerName = random.choice(firstNames) + " " + random.choice(lastNames)
salary = random.randrange(4000,10000)
generatedLoan = Loan(borrowerName, ssn,salary)
return generatedLoan
def test_if_loan_can_be_aggregated(self):
tempLoan = Loan(
1000000,
'Network 1',
'Mar',
'Loan Product 1',
1000.50,
)
loanProcessor = LoanProcessor(tempLoan)
tempLoan = Loan(
1000000,
'Network 1',
'Mar',
'Loan Product 1',
1000.50,
)
self.assertTrue(loanProcessor.canBeAggregated(tempLoan))
tempLoanWithDifferentNetwork = Loan(
1000000,
'Network 2',
'Mar',
'Loan Product 1',
1000.50,
)
self.assertFalse(
loanProcessor.canBeAggregated(tempLoanWithDifferentNetwork))
tempLoanWithDifferentProduct = Loan(
1000000,
'Network 1',
'Mar',
'Loan Product 2',
1000.50,
)
self.assertFalse(
loanProcessor.canBeAggregated(tempLoanWithDifferentProduct))
tempLoanWithDifferentMonth = Loan(
1000000,
'Network 1',
'Apr',
'Loan Product 1',
1000.50,
)
self.assertFalse(
loanProcessor.canBeAggregated(tempLoanWithDifferentMonth))
def getLoan(self, principal, interestRate, annualPayments, duration, startDate, loanPaymentExtra):
"""
Allocates a loan to the profile.
"""
self.hasLoan = True
self.noLoans += 1
# print(f'Number of loans: {self.noLoans}')
loanID = str(self.noLoans)
self.loanDict[loanID] = Loan(loanID, principal, interestRate, annualPayments, duration, startDate, loanPaymentExtra)
def processInput(line):
msisdn, network, loan_date, product, amount = line.split(',')
msisdn = msisdn.strip("'")
network = network.strip("'").strip().lower()
loan_date = loan_date.strip("'")
product = product.strip("'").strip().lower()
amount = amount.strip("'")
if (Loan.isAmountValid(amount) and Loan.isDateValid(loan_date)):
month_year = Loan.extractMonthYearFromDate(loan_date)
tempLoan = Loan(msisdn, network, month_year, product, amount)
sys.stdout.write(tempLoan.display() + '\n')
def map_loan_row(self, row):
return Loan(loan_id=row['id'],
grade=row['grade'],
int_rate=row['int_rate'],
term=row['term'],
amount=row['funded_amnt'],
issue_date=row['issue_d'],
last_date=row['last_pymnt_d'],
investment=self.buy_size,
defaults=row['defaulted'],
total_payment=row['total_pymnt'],
total_principle=row['total_rec_prncp'],
recoveries=row['recoveries'])
def main():
a = Asset(1000)
normal_loan = Loan(12, 0.3, 1000, a)
memoizable_loan = MemoizedLoan(12, 0.3, 1000, a)
print "First normal loan time cost:"
normal_loan.interestDue(10)
print "Second normal loan time cost:"
normal_loan.interestDue(10)
print "First memoizable loan time cost:"
memoizable_loan.interestDue(10)
print "Second memoizable loan time cost:"
memoizable_loan.interestDue(10)
def disponibilizar_livro(usuario):
id_livro = int(input("INFORME O ID DO LIVRO QUE SERÁ DISPONIBILIZADO: "))
local = input("INFORME O LOCAL ONDE O LIVRO SERÁ DISPONIBILIZADO: ")
livro = lista_book(id_livro, usuario)
if (len(livro) > 0):
novo_loan = Loan(0, usuario.id, livro[0][0], local, 'DISPONIVEL')
sql_loan = f"INSERT INTO tb_loan (id_user, id_book, begin_date, coletion_location, status) values('{usuario.id}', '{novo_loan.id_book}', null, '{novo_loan.collect_location}', '{novo_loan.status}')"
try:
exec_command(sql_loan)
print("LIVRO DISPONIBILIZADO COM SUCESSO!")
except:
print("FALHA AO DISPONIBILIZAR LIVRO.")
else:
print("ID ESCOLHIDO NÃO É VÁLIDO!")
def save_loan(self, request, context):
email = request.email
interest_rate = request.interest_rate
repayment_terms = request.repayment_terms
loan_amount = request.loan_amount
self.loan = Loan(email, interest_rate, repayment_terms, loan_amount)
self.loan.save_loan()
response = loan_pb2.empty()
return response
def main():
print ('========== Exersize 2.1.3 ==========\n')
myLoan = Loan(360,.025,100000)
print("Monthly Payment: {}".format(myLoan.monthlyPayment()))
t = timer()
t.start()
print("Balance after 60 periods: {}".format(myLoan.balance(60)))
t.end()
t.start()
print('Balance in period 60 computed recursivly {}'.format(myLoan.balanceRecursive(60, myLoan.face)))
t.end()
t.start()
print("Interest due on period 60: {}".format(myLoan.interestDue(60)))
t.end()
t.start()
print('Interest in period 60 computed recursivly {}'.format(myLoan.interestDueRecursive(60, myLoan.face)))
t.end()
t.start()
print("Principal due on period 60: {}".format(myLoan.principlaDue(60)))
t.end()
t.start()
print('Principal in period 60 computed recursivly {}'.format(myLoan.principalDueRecursive(60, myLoan.face)))
t.end()
"""
On my system, in both instances the direct and recursive versions of the function run to fast to comeup with a time besides 0
However, I know that the recursive function is likely much slower
"""
print("The total payment should equal interest plus principal which is {}".format(myLoan.interestDue(5) + myLoan.principlaDue(5)))
print("Total Interest paid is {}".format(myLoan.totalInterest()))
print("Total Payment is {}".format(myLoan.totalPayments()))
print("Total Interest paid is {}".format(myLoan.totalInterest()))
print("Old rate {}".format(myLoan.rate))
print("Old term {}".format(myLoan.term))
print("Old face {}".format(myLoan.face))
myLoan.rate = .035
myLoan.term = 60
myLoan.face = 20000
print("New rate {}".format(myLoan.rate))
print("New term {}".format(myLoan.term))
print("New face {}".format(myLoan.face))
def main(lenders_info, amount):
"""
Takes in data from user and feeds classes. Captures errors and prints to user
:param lenders_info: csv file from user
:param amount: amount to borrow input from user
:return: Prints Request Amount, Rate, Total & Monthly Repayments to screen
"""
try:
lenders = Lenders(lenders_info)
selected_lenders = lenders.get_lenders(amount)
loan_offer = Loan(selected_lenders, amount)
loan_offer.input_validation()
loan_offer.calc_offer()
loan_offer.print_output()
except IOError:
print("No market data found for file specified")
except Exception as error:
print(error)
else:
return loan_offer
def init_loan(self):
""" Open a loan with third party lending platform. We use Poloniex
for this demo. Future versions can include integrations with
multiple platforms, and the ability to automatically select the
optimal (most stable and profitable) asset and lending provider.
Loan class has a LoanAgent which cancels old loan offers, turns
auto-renew off on active loans, and creates new loan offers at
fair price (fair = average of the lowest three loan offers).
"""
#Loan(logger, api, asset, deposit, duration, agent)
return Loan(
self.logger,
self.LENDER_API,
self.LOAN_ASSET,
self.initial_deposit,
self.duration,
LoanAgent(
self.logger,
self.LENDER_API,
{self.LOAN_ASSET: self.MIN_LOAN}, #{"USDC" : 50}
self.initial_deposit)) #50
def main():
print('========== Exersize 2.1.5 ==========\n')
print('Testing static method for monthly rate : {}'.format(
Loan.monthlyRate(.025)))
print('Testing testing static method for annual rate: {}'.format(
Loan.annualRate(.01)))
"""
The benefit of the static method is that neither the class nor the instance is passed in. This allows us
to include functions that may be useful for the class, but do not directly rely on information in the class.
From an organizational standpoint this makes it easier to organize the code. For example, the rate conversion is
not logically part of a loan object, however the loan class is where it makes most sense to include it.
"""
myLoan = Loan(360, .025, 100000)
print("Monthly Payment: {}".format(myLoan.monthlyPayment()))
print("Balance after 60 periods: {}".format(myLoan.balance(60)))
print("Interest due on period 60: {}".format(myLoan.interestDue(60)))
print("Principal due on period 60: {}".format(myLoan.principlaDue(60)))
print("The total payment should equal interest plus principal which is {}".
format(myLoan.interestDue(5) + myLoan.principlaDue(5)))
print("Total Interest paid is {}".format(myLoan.totalInterest()))
print("Total Payment is {}".format(myLoan.totalPayments()))
print("Total Interest paid is {}".format(myLoan.totalInterest()))
print("Old rate {}".format(myLoan.rate))
print("Old term {}".format(myLoan.term))
print("Old face {}".format(myLoan.face))
myLoan.rate = .035
myLoan.term = 60
myLoan.face = 20000
print("New rate {}".format(myLoan.rate))
print("New term {}".format(myLoan.term))
print("New face {}".format(myLoan.face))
def main():
print('========== Exercise 2.1.2 ==========')
myLoan = Loan(360, .025, 100000)
print("Monthly Payment: {}".format(myLoan.monthlyPayment()))
print("Balance after 360 periods: {}".format(myLoan.balance(360)))
print("Interest due on period 360: {}".format(myLoan.interestDue(360)))
print("Principal due on period 360: {}".format(myLoan.principlaDue(360)))
print("The total payment should equal interest plus principal which is {}".
format(myLoan.interestDue(5) + myLoan.principlaDue(5)))
print("Total Interest paid is {}".format(myLoan.totalInterest()))
print("Total Payment is {}".format(myLoan.totalPayments()))
print("Old rate {}".format(myLoan.rate))
print("Old term {}".format(myLoan.term))
print("Old face {}".format(myLoan.face))
myLoan.rate = .035
myLoan.term = 60
myLoan.face = 20000
print("New rate {}".format(myLoan.rate))
print("New term {}".format(myLoan.term))
print("New face {}".format(myLoan.face))
def crawl():
company_id = 3
url = "http://www.91wangcai.com/invest/index.html"
request_headers = {
'Referee': "http://www.91wangcai.com",
'User-Agent': DEFAULT_UA
}
db = get_db_engine()
db_ids = list(
db.execute(
"select original_id from loan where company_id=%s and status=0",
company_id))
# db all
db_ids_set = set()
# 在线的所有id
online_ids_set = set()
# new
new_ids_set = set()
# update
update_ids_set = set()
for id in db_ids:
db_ids_set.add(id[0].encode("utf-8"))
# debug
if FLAGS.debug_parser:
import pdb
pdb.set_trace()
try:
loan_htm = download_page(url, request_headers)
loan_htm_parse = parse_html(loan_htm, encoding="gb2312")
loans = loan_htm_parse.xpath("//div[@class='proBoxNew']")
if len(loans) > 0:
for loan in loans:
href = str(loan.xpath("div[@class='hd']/a/@href")[0])
original_id = href.split(".")[0].split("/")[2].encode("utf-8")
if original_id:
online_ids_set.add(original_id)
if original_id in db_ids_set:
update_ids_set.add(original_id)
loan_obj = Loan(company_id, original_id)
loan_obj.schedule = autodecode(str(loan.xpath("div[@class='bd']/table/tr[2]/td[2]/text()")[0].encode("gb2312"))) \
.encode("utf-8").replace("融资进度：", "").replace("借款进度：", "").strip().replace("%", "")
loan_obj.db_update(db)
else:
new_ids_set.add(original_id)
loan_obj = Loan(company_id, original_id)
loan_obj.href = "http://www.91wangcai.com" + href
loan_obj.title = autodecode(
str(
loan.xpath("div[@class='hd']/a/text()")[0].encode(
"gb2312"))).encode("utf-8")
loan_obj.borrow_amount = autodecode(str(loan.xpath("div[@class='bd']/table/tr[1]/td[1]/em/text()")[0].encode("gb2312"))) \
.encode("utf-8").replace("￥", "")
loan_obj.rate = str(
loan.xpath(
"div[@class='bd']/table/tr[1]/td[2]/em/text()")
[0]).strip().replace("%", "")
loan_period_text = lxml.html.tostring(loan.xpath("div[@class='bd']/table/tr[1]/td[3]/*")[0]) \
.replace("<em>", "").replace("</em>", "")
html_parser = HTMLParser.HTMLParser()
period = html_parser.unescape(loan_period_text).encode(
"utf-8").strip()
if period.find(loan_obj.PERIOD_UNIT_DAY) > 0:
loan_obj.period = period.replace(
loan_obj.PERIOD_UNIT_DAY, "")
loan_obj.period_unit = loan_obj.PERIOD_UNIT_DAY
else:
loan_obj.period = period.replace("个", "").replace(
loan_obj.PERIOD_UNIT_MONTH, "")
loan_obj.period_unit = loan_obj.PERIOD_UNIT_MONTH
loan_obj.repayment = autodecode(str(loan.xpath("div[@class='bd']/table/tr[2]/td[1]/text()")[0].encode("gb2312"))) \
.encode("utf-8").replace("还款方式：", "")
loan_obj.schedule = autodecode(str(loan.xpath("div[@class='bd']/table/tr[2]/td[2]/text()")[0].encode("gb2312"))) \
.encode("utf-8").replace("融资进度：", "").replace("借款进度：", "").strip().replace("%", "")
loan_obj.db_create(db)
logger.info("company %s crawler loan: new size %s, update size %s",
company_id, len(new_ids_set), len(update_ids_set))
# db - 新抓取的 = 就是要下线的
off_ids_set = db_ids_set - online_ids_set
if off_ids_set:
loan_obj = Loan(company_id)
loan_obj.db_offline(db, off_ids_set)
logger.info("company %s crawler loan: offline %s", company_id,
len(off_ids_set))
except:
logger.error("url: %s xpath failed:%s", url, traceback.format_exc())
def crawl():
company_id = 11
url = "https://www.tzydb.com"
request_headers = {'User-Agent': DEFAULT_UA}
db = get_db_engine()
db_ids = list(db.execute("select original_id from loan where company_id=%s and status=0", company_id))
# db all
db_ids_set = set()
# 在线的所有id
online_ids_set = set()
# new
new_ids_set = set()
# update
update_ids_set = set()
for id in db_ids:
db_ids_set.add(id[0].encode("utf-8"))
# debug
if FLAGS.debug_parser:
import pdb
pdb.set_trace()
try:
htm = download_page(url, request_headers)
htm_obj = parse_html(htm)
loans = htm_obj.xpath("//div[@id='proList']/ul[@class='item_li']")
if len(loans) > 0:
for loan in loans:
schedule = str(loan.xpath("li/div[last()]/div[1]/span[2]/strong/text()")[0].encode("UTF-8")).strip()
if schedule == "100%" or schedule == "100.0%":
#放弃已经结束的
continue
# link = https://www.tzydb.com/boot/lookup/971,1017
a_script = str(loan.xpath("li/div[1]/div[1]/div/a/@href")[0].encode("utf-8"))
o_id = ID_RE.findall(a_script)[0]
original_id = o_id.replace(",", "-")
if original_id:
online_ids_set.add(original_id)
if original_id in db_ids_set:
update_ids_set.add(original_id)
loan_obj = Loan(company_id, original_id)
loan_obj.schedule = str(loan.xpath("li/div[last()]/div[1]/span[2]/strong/text()")[0].encode("UTF-8")).strip().replace("%", "")
loan_obj.db_update(db)
else:
new_ids_set.add(original_id)
loan_obj = Loan(company_id, original_id)
loan_obj.href = "https://www.tzydb.com/boot/lookup/" + o_id
loan_obj.title = str(loan.xpath("li/div[1]/div[1]/div/a/text()")[0].encode("utf-8"))
loan_obj.borrow_amount = str(loan.xpath("li/div[2]/div[1]/span/text()")[0].encode("utf-8")).strip()\
.replace(" ", "").replace(",", "")
loan_obj.period = str(loan.xpath("li/div[2]/div[3]/span/text()")[0].encode("UTF-8")).strip()
loan_obj.period_unit = loan_obj.PERIOD_UNIT_MONTH
loan_obj.rate = str(loan.xpath("li/div[2]/div[2]/span/text()")[0]).strip().replace("%", "")
loan_obj.schedule = str(loan.xpath("li/div[last()]/div[1]/span[2]/strong/text()")[0].encode("UTF-8")).strip().replace("%", "")
loan_obj.db_create(db)
logger.info("company %s crawler loan: new size %s, update size %s", company_id, len(new_ids_set), len(update_ids_set))
# db - 新抓取的 = 就是要下线的
off_ids_set = db_ids_set - online_ids_set
if off_ids_set:
loan_obj = Loan(company_id)
loan_obj.db_offline(db, off_ids_set)
logger.info("company %s crawler loan: offline %s", company_id, len(off_ids_set))
except:
logger.error("url: %s xpath failed:%s", url, traceback.format_exc())
def crawl():
company_id = 9
url = "https://list.lufax.com/list/service/product/fuying-product-list/listing/1"
request_headers = {
'Referee': "https://list.lufax.com/list/listing/fuying",
'User-Agent': DEFAULT_UA
}
db = get_db_engine()
db_ids = list(
db.execute(
"select original_id from loan where company_id=%s and status=0",
company_id))
# db all
db_ids_set = set()
# 在线的所有id
online_ids_set = set()
# new
new_ids_set = set()
# update
update_ids_set = set()
for id in db_ids:
db_ids_set.add(id[0].encode("utf-8"))
# debug
if FLAGS.debug_parser:
import pdb
pdb.set_trace()
try:
loan_htm = download_page(url, request_headers)
loans_json = loads(loan_htm, encoding="UTF-8")
loan_num = loans_json["totalCount"]
if loans_json and loan_num:
for i in range(0, loan_num):
original_id = str(loans_json["data"][i]["productId"])
online_ids_set.add(original_id)
if original_id in db_ids_set:
update_ids_set.add(original_id)
loan_obj = Loan(company_id, original_id)
loan_obj.schedule = str(
float(loans_json["data"][i]["progress"]) * 100)
loan_obj.cast = str(
int(loans_json["data"][i]["raisedAmount"]))
loan_obj.db_update(db)
else:
new_ids_set.add(original_id)
loan_obj = Loan(company_id, original_id)
loan_obj.href = "https://list.lufax.com/list/productDetail?productId=%s" % original_id
loan_obj.title = loans_json["data"][i][
"productNameDisplay"]
loan_obj.rate = str(
float(loans_json["data"][i]["interestRate"]) * 100)
period = str(loans_json["data"][i]
["investPeriodDisplay"].encode("utf-8"))
if period.find(loan_obj.PERIOD_UNIT_DAY) > 0:
loan_obj.period = period.replace(
loan_obj.PERIOD_UNIT_DAY, "")
loan_obj.period_unit = loan_obj.PERIOD_UNIT_DAY
else:
loan_obj.period = period.replace("个", "").replace(
loan_obj.PERIOD_UNIT_MONTH, "")
loan_obj.period_unit = loan_obj.PERIOD_UNIT_MONTH
loan_obj.repayment = loans_json["data"][i][
"collectionModeDisplay"]
loan_obj.borrow_amount = str(
int(loans_json["data"][i]["price"]))
loan_obj.schedule = str(
float(loans_json["data"][i]["progress"]) * 100)
loan_obj.cast = str(
int(loans_json["data"][i]["raisedAmount"]))
loan_obj.db_create(db)
logger.info("company %s crawler loan: new size %s, update size %s",
company_id, len(new_ids_set), len(update_ids_set))
# db - 新抓取的 = 就是要下线的
off_ids_set = db_ids_set - online_ids_set
if off_ids_set:
loan_obj = Loan(company_id)
loan_obj.db_offline(db, off_ids_set)
logger.info("company %s crawler loan: offline %s", company_id,
len(off_ids_set))
except:
logger.error("url: %s xpath failed:%s", url, traceback.format_exc())
def process_input(avalue, lface, lrate, lterm):
a = Asset(float(avalue))
l = Loan(int(lterm), float(lrate), float(lface), a)
return l
def crawl():
company_id = 19
url = "https://www.qian360.com/bq/queryProductList.html?currentPage=1&pernum=12&type=0"
request_headers = {
'Referee': "https://www.qian360.com/tl/select.html",
'User-Agent': DEFAULT_UA
}
db = get_db_engine()
db_ids = list(
db.execute(
"select original_id from loan where company_id=%s and status=0",
company_id))
# db all
db_ids_set = set()
# 在线的所有id
online_ids_set = set()
# new
new_ids_set = set()
# update
update_ids_set = set()
for id in db_ids:
db_ids_set.add(id[0].encode("utf-8"))
# debug
if FLAGS.debug_parser:
import pdb
pdb.set_trace()
try:
loan_htm = download_page(url, request_headers)
loans_json = loads(loan_htm, encoding="UTF-8")
if loans_json["list"]:
for i in range(0, len(loans_json["list"])):
if int(loans_json["list"][i]["status"]) != 1:
continue
original_id = str(loans_json["list"][i]["borrowId"])
online_ids_set.add(original_id)
if original_id in db_ids_set:
update_ids_set.add(original_id)
loan_obj = Loan(company_id, original_id)
loan_obj.schedule = str(loans_json["list"][i]["percent"])
loan_obj.cast = str(
int(loans_json["list"][i]["accountYes"]))
loan_obj.db_update(db)
else:
new_ids_set.add(original_id)
loan_obj = Loan(company_id, original_id)
loan_obj.href = "https://www.qian360.com/ti/detail.html?borrowId=%s" % original_id
loan_obj.title = loans_json["list"][i]["name"]
loan_obj.rate = str(loans_json["list"][i]["apr"])
loan_obj.period = str(loans_json["list"][i]["totalPeriod"])
loan_obj.period_unit = loan_obj.PERIOD_UNIT_MONTH
loan_obj.borrow_amount = str(
int(loans_json["list"][i]["account"]))
loan_obj.schedule = str(loans_json["list"][i]["percent"])
loan_obj.cast = str(
int(loans_json["list"][i]["accountYes"]))
loan_obj.db_create(db)
logger.info("company %s crawler loan: new size %s, update size %s",
company_id, len(new_ids_set), len(update_ids_set))
# db - 新抓取的 = 就是要下线的
off_ids_set = db_ids_set - online_ids_set
if off_ids_set:
loan_obj = Loan(company_id)
loan_obj.db_offline(db, off_ids_set)
logger.info("company %s crawler loan: offline %s", company_id,
len(off_ids_set))
except:
logger.error("url: %s xpath failed:%s", url, traceback.format_exc())
from borrower import Borrower
from investor import Investor
from loan import Loan
# create Borrower
borrower1 = Borrower(balance=1000)
print('borrower1 is created!')
# List borrower1 loan Requests
print('List borrower1 loan Requests')
borrower1.checkMyLoans()
# creat Loan instance
print('create loan')
loan1 = Loan()
print('loan1 created')
# test submitting request validation
print('test submitting request validation "invalid borrower instance"')
loan1.submitLoanRequest('borrower', amount=5000, installment_period=6)
print('test submitting request validation invalid amount')
loan1.submitLoanRequest(borrower1, amount=000, installment_period=6)
print('test submitting request validation invalid installment_period')
loan1.submitLoanRequest(borrower1, amount=5000, installment_period=-6)
# Submit valid request
print('Submit valid request')
loan1.submitLoanRequest(borrower1, amount=5000, installment_period=6)
# List borrower1 loan Requests
print('List borrower1 loan Requests')
def crawl():
company_id = 23
url = "https://member.niwodai.com/xiangmu/"
request_headers = {'Referee': REFEREE, 'User-Agent': DEFAULT_UA}
db = get_db_engine()
db_ids = list(
db.execute(
"select original_id from loan where company_id=%s and status=0",
company_id))
# db all
db_ids_set = set()
# 在线的所有id
online_ids_set = set()
# new
new_ids_set = set()
# update
update_ids_set = set()
for id in db_ids:
db_ids_set.add(id[0].encode("utf-8"))
# debug
if FLAGS.debug_parser:
import pdb
pdb.set_trace()
try:
htm = download_page(url, request_headers)
htm_obj = parse_html(htm, encoding="utf-8")
loan_size = int(str(htm_obj.xpath("//div[@class='biaoList']/table/tbody/tr[1]/th[last()]/text()")[0].encode("utf-8"))\
.replace("共", "").replace("个标", "").strip())
if loan_size > 0:
page = loan_size / 10
if loan_size % 10 > 0:
page += 1
for p in range(1, page + 1):
page_url = "https://member.niwodai.com/loan/loan.do?pageNo=%d&totalCount=%d" % (
p, loan_size)
page_html = download_page(page_url, request_headers)
page_obj = parse_html(page_html, encoding="utf-8")
loans = page_obj.xpath(
"//div[@class='biaoList']/table/tbody/tr")
for loan in loans:
if lxml.html.tostring(loan).find("<th>") > 0:
continue
href = str(loan.xpath("td[1]/a/@href")[0])
original_id = href.replace(".html", "").split("/")[2]
if original_id:
online_ids_set.add(original_id)
if original_id in db_ids_set:
update_ids_set.add(original_id)
loan_obj = Loan(company_id, original_id)
loan_obj.schedule = str(
loan.xpath("td[5]/text()")[0].encode(
"utf-8")).strip().replace("%", "")
loan_obj.db_update(db)
else:
new_ids_set.add(original_id)
loan_obj = Loan(company_id, original_id)
loan_obj.href = REFEREE + href
loan_obj.title = str(
loan.xpath("td[1]/a/text()")[0].encode(
"utf-8")).strip()
loan_obj.borrow_amount = str(
loan.xpath("td[4]/em/text()")[0].encode(
"utf-8")).strip().replace(",", "")
loan_obj.rate = str(
loan.xpath("td[2]/em/text()")[0].encode(
"utf-8")).strip().replace("%", "")
loan_obj.period = str(
loan.xpath("td[3]/em/text()")[0].encode(
"utf-8")).strip()
loan_obj.period_unit = loan_obj.PERIOD_UNIT_DAY
loan_obj.schedule = str(
loan.xpath("td[5]/text()")[0].encode(
"utf-8")).strip().replace("%", "")
loan_obj.db_create(db)
logger.info("company %s crawler loan: new size %s, update size %s",
company_id, len(new_ids_set), len(update_ids_set))
# db - 新抓取的 = 就是要下线的
off_ids_set = db_ids_set - online_ids_set
if off_ids_set:
loan_obj = Loan(company_id)
loan_obj.db_offline(db, off_ids_set)
logger.info("company %s crawler loan: offline %s", company_id,
len(off_ids_set))
except:
logger.error("url: %s xpath failed:%s", url, traceback.format_exc())
def crawl():
company_id = 4
url = "https://www.yinhu.com/loan/loan_list.bl"
request_headers = {'Referee': "https://www.yinhu.com", 'User-Agent': DEFAULT_UA}
db = get_db_engine()
db_ids = list(db.execute("select original_id from loan where company_id=%s and status=0", company_id))
# db all
db_ids_set = set()
# 在线的所有id
online_ids_set = set()
# new
new_ids_set = set()
# update
update_ids_set = set()
# offline
off_ids_set = set()
for id in db_ids:
db_ids_set.add(id[0].encode("utf-8"))
# debug
if FLAGS.debug_parser:
import pdb
pdb.set_trace()
try:
loan_htm = download_page(url, request_headers)
loan_htm_parse = parse_html(loan_htm, encoding="UTF-8")
loans = loan_htm_parse.xpath("//div[@id='loan_list']/table/tbody/tr")
if len(loans) > 0:
for loan in loans:
href = str(loan.xpath("td[1]/p/a/@href")[0])
original_id = href.split("=")[1].encode("utf-8")
try:
loan_status = str(loan.xpath("td[last()]/em/span/text()")[0].encode("utf-8")).strip()
except:
loan_status = str(loan.xpath("td[last()]/a/span/text()")[0].encode("utf-8")).strip()
if original_id and loan_status != "还款中":
online_ids_set.add(original_id)
if loan_status == "还款中" or loan_status == "满标":
if original_id in db_ids_set:
off_ids_set.add(original_id)
continue
if original_id in db_ids_set:
update_ids_set.add(original_id)
loan_obj = Loan(company_id, original_id)
loan_obj.schedule = str(loan.xpath("td[6]/div[@class='bar_bg']/div/span/span/text()")[0].encode("utf-8"))\
.strip().replace("%", "")
loan_obj.db_update(db)
else:
new_ids_set.add(original_id)
loan_obj = Loan(company_id, original_id)
loan_obj.href = "https://www.yinhu.com" + href
loan_obj.title = str(loan.xpath("td[1]/p/a/text()")[0].encode("utf-8")).strip()
loan_obj.borrow_amount = str(loan.xpath("td[4]/text()")[0].encode("utf-8")).strip().replace(",", "")\
.replace("元", "")
loan_obj.rate = str(loan.xpath("td[3]/text()")[0].encode("utf-8")).strip()
period = str(loan.xpath("td[5]/text()")[0].encode("utf-8")).strip()
if period.find(loan_obj.PERIOD_UNIT_DAY) > 0:
loan_obj.period = period.replace(loan_obj.PERIOD_UNIT_DAY, "")
loan_obj.period_unit = loan_obj.PERIOD_UNIT_DAY
else:
loan_obj.period = period.replace("个", "").replace(loan_obj.PERIOD_UNIT_MONTH, "")
loan_obj.period_unit = loan_obj.PERIOD_UNIT_MONTH
loan_obj.schedule = str(loan.xpath("td[6]/div[@class='bar_bg']/div/span/span/text()")[0].encode("utf-8"))\
.strip().replace("%", "")
loan_obj.db_create(db)
logger.info("company %s crawler loan: new size %s, update size %s", company_id, len(new_ids_set), len(update_ids_set))
# db - 新抓取的 = 就是要下线的
off_ids_set = db_ids_set - online_ids_set
if off_ids_set:
loan_obj = Loan(company_id)
loan_obj.db_offline(db, off_ids_set)
logger.info("company %s crawler loan: offline %s", company_id, len(off_ids_set))
except:
logger.error("url: %s xpath failed:%s", url, traceback.format_exc())
def crawl():
company_id = 18
url = "https://www.my089.com/Loan/default.aspx?pid=1"
request_headers = {
'Referee': "http://www.ppdai.com",
'User-Agent': DEFAULT_UA
}
db = get_db_engine()
db_ids = list(
db.execute(
"select original_id from loan where company_id=%s and status=0",
company_id))
# db all
db_ids_set = set()
# 在线的所有id
online_ids_set = set()
# new
new_ids_set = set()
# update
update_ids_set = set()
for id in db_ids:
db_ids_set.add(id[0].encode("utf-8"))
# debug
if FLAGS.debug_parser:
import pdb
pdb.set_trace()
try:
htm = download_page(url, request_headers)
htm_obj = parse_html(htm)
page = str(htm_obj.xpath("//div[@class='yema rt']/span[@class='z_page']/text()")[0].encode("UTF-8"))\
.replace("共", "").replace("页", "")
for p in range(1, int(page) + 1):
url = "https://www.my089.com/Loan/default.aspx?pid=" + str(p)
logger.info("page url: %s", url)
loan_htm = download_page(url, request_headers)
loan_obj = parse_html(loan_htm)
loans = loan_obj.xpath(
"//div[@class='Loan_box']/dl[@class='LoanList']")
if len(loans) > 0:
for loan in loans:
if str(loan.xpath("dd[last()]/p/span/text()")
[0]) == "100%":
continue
href = str(
loan.xpath("dd[2]/div[@class='txt_tou']/a/@href")[0])
original_id = href.split("=")[1].encode("utf-8")
if original_id:
online_ids_set.add(original_id)
if original_id in db_ids_set:
update_ids_set.add(original_id)
loan_obj = Loan(company_id, original_id)
loan_obj.schedule = str(
loan.xpath("dd[last()]/p/span/text()")[0].encode(
"UTF-8")).strip().replace("%", "")
loan_obj.db_update(db)
else:
new_ids_set.add(original_id)
loan_obj = Loan(company_id, original_id)
loan_obj.href = "https://www.my089.com/Loan/" + href
loan_obj.title = str(
loan.xpath("dd[2]/div[@class='txt_tou']/a/@title")
[0].encode("UTF-8"))
loan_obj.borrow_amount = str(loan.xpath("dd[4]/span/text()")[0].encode("UTF-8")).strip().replace("￥", "")\
.replace(",", "")
loan_obj.rate = str(
loan.xpath("dd[3]/span/text()")[0].encode(
"UTF-8")).strip().replace("%/年", "")
loan_obj.period = str(
loan.xpath("dd[5]/span/text()")[0].encode(
"UTF-8")).strip().replace(" ", "")
s = str(loan.xpath("dd[5]/text()")[0].encode(
"UTF-8")).strip().replace(" ",
"").replace("个", "")
loan_obj.period_unit = s.split("/")[0].strip()
loan_obj.repayment = s.split("/")[1].strip()
loan_obj.schedule = str(
loan.xpath("dd[last()]/p/span/text()")[0].encode(
"UTF-8")).strip().replace("%", "")
loan_obj.db_create(db)
logger.info("company %s crawler loan: new size %s, update size %s",
company_id, len(new_ids_set), len(update_ids_set))
# db - 新抓取的 = 就是要下线的
off_ids_set = db_ids_set - online_ids_set
if off_ids_set:
loan_obj = Loan(company_id)
loan_obj.db_offline(db, off_ids_set)
logger.info("company %s crawler loan: offline %s", company_id,
len(off_ids_set))
except:
logger.error("url: %s xpath failed:%s", url, traceback.format_exc())
def crawl():
company_id = 10
url = "https://www.xinhehui.com/Financing/Invest/ajaxplist"
request_headers = {'Referee': REFEREE, 'User-Agent': DEFAULT_UA}
db = get_db_engine()
db_ids = list(
db.execute(
"select original_id from loan where company_id=%s and status=0",
company_id))
# db all
db_ids_set = set()
# 在线的所有id
online_ids_set = set()
# new
new_ids_set = set()
# update
update_ids_set = set()
for id in db_ids:
db_ids_set.add(id[0].encode("utf-8"))
# debug
if FLAGS.debug_parser:
import pdb
pdb.set_trace()
try:
htm = download_page(url, request_headers)
htm_obj = parse_html(htm)
loans = htm_obj.xpath(
"//table[@class='ui-record-table percentTable mt10']/tbody/tr")
if len(loans) > 0:
for loan in loans:
if loan.xpath("td[last()]/a/@href")[0].encode(
"utf-8") == "javascript:;":
#放弃已经结束的
continue
href = str(loan.xpath("td[1]/p[1]/a/@href")[0].encode("utf-8"))
original_id = href.split("id%3D")[1].encode("utf-8").strip()
if original_id:
online_ids_set.add(original_id)
if original_id in db_ids_set:
update_ids_set.add(original_id)
loan_obj = Loan(company_id, original_id)
if loan.xpath("td[7]/div/a"):
loan_obj.schedule = str(
loan.xpath("td[7]/div/a/text()")[0].encode(
"UTF-8")).strip().replace("%", "")
else:
loan_obj.schedule = "0"
loan_obj.db_update(db)
else:
new_ids_set.add(original_id)
loan_obj = Loan(company_id, original_id)
loan_obj.href = "https://www.xinhehui.com" + href
title_1 = str(
loan.xpath("td[1]/p[1]/a/text()")[0].encode(
"utf-8")).strip()
if loan.xpath("td[1]/p[1]/a/em"):
title_2 = str(
loan.xpath("td[1]/p[1]/a/em/text()")[0].encode(
"utf-8")).strip()
else:
title_2 = str(
loan.xpath("td[1]/p[1]/a/span/text()")[0].encode(
"utf-8")).strip()
loan_obj.title = title_1 + title_2
borrow_amount = str(
loan.xpath("td[2]/span/text()")[0].encode(
"utf-8")).strip().replace(" ", "")
if borrow_amount.find("万") > 0:
loan_obj.borrow_amount = float(
borrow_amount.replace("万", "")) * 10000
else:
loan_obj.borrow_amount = float(
borrow_amount.replace("元", "").replace(",", ""))
if loan.xpath("td[4]/span"):
period = str(
loan.xpath("td[4]/span/@title")[0].encode(
"UTF-8")).strip()
else:
period = str(
loan.xpath("td[4]/text()")[0].encode(
"UTF-8")).strip()
if period.find(loan_obj.PERIOD_UNIT_DAY) > 0:
loan_obj.period = period.replace(
loan_obj.PERIOD_UNIT_DAY, "")
loan_obj.period_unit = loan_obj.PERIOD_UNIT_DAY
else:
loan_obj.period = period.replace("个", "").replace(
loan_obj.PERIOD_UNIT_MONTH, "")
loan_obj.period_unit = loan_obj.PERIOD_UNIT_MONTH
loan_obj.rate = str(
loan.xpath("td[3]/p/text()")[0]).strip().replace(
"%", "")
loan_obj.repayment = str(
loan.xpath("td[5]/text()")[0].encode("UTF-8")).strip()
if loan.xpath("td[7]/div/a"):
loan_obj.schedule = str(
loan.xpath("td[7]/div/a/text()")[0].encode(
"UTF-8")).strip().replace("%", "")
else:
loan_obj.schedule = "0"
loan_obj.db_create(db)
logger.info("company %s crawler loan: new size %s, update size %s",
company_id, len(new_ids_set), len(update_ids_set))
# db - 新抓取的 = 就是要下线的
off_ids_set = db_ids_set - online_ids_set
if off_ids_set:
loan_obj = Loan(company_id)
loan_obj.db_offline(db, off_ids_set)
logger.info("company %s crawler loan: offline %s", company_id,
len(off_ids_set))
except:
logger.error("url: %s xpath failed:%s", url, traceback.format_exc())
class Cleaner():
account = Account()
card = Card()
client = Client()
disp = Disp()
district = District()
loan = Loan()
order = Order()
trans = Trans()
def cleanData(self):
print("Error in district:")
self.district.clean()
self.district.output("cleaned_district.csv")
print("Error in account:")
self.account.clean(self.district.cur)
print("Error in client:")
self.client.clean(self.district.cur)
print("Error in disp:")
self.disp.clean(self.client.cur, self.account.cur)
dat = []
i = 0
j = 0
k = 0
a_len = len(self.account.cleaned_data)
d_len = len(self.disp.cleaned_data)
c_len = len(self.client.cleaned_data)
while True:
a = self.account.cleaned_data[i]
d = self.disp.cleaned_data[j]
c = self.client.cleaned_data[k]
a_a_id = a[0]
c_c_id = c[0]
d_a_id = d[2]
d_c_id = d[1]
if a_a_id != d_a_id:
while a_a_id < d_a_id:
account_id, district_id, frequency, date, time = a
g = 'unknown'
t = 'UNKNOWN'
dat.append(
[account_id, district_id, frequency, date, time, g, t])
i = i + 1
a = self.account.cleaned_data[i]
a_a_id = a[0]
if a_a_id > d_a_id:
print("Sort error")
break
if (c_c_id != d_c_id):
print("Sort error")
break
g = c[2]
t = d[3]
if (j != d_len - 1) and (k != c_len - 1):
_d = self.disp.cleaned_data[j + 1]
_c = self.client.cleaned_data[k + 1]
_d_a_id = _d[2]
if d_a_id == _d_a_id:
_d_c_id = _d[1]
_c_c_id = _c[0]
if _c_c_id != _d_c_id:
print("Sort error")
break
_g = _c[2]
_t = _d[3]
if g != _g:
g = 'couple'
if t != _t:
t = 'DOUBLE'
j = j + 1
k = k + 1
account_id, district_id, frequency, date, time = a
dat.append([account_id, district_id, frequency, date, time, g, t])
i = i + 1
j = j + 1
k = k + 1
if (i == a_len) or (j == d_len) or (k == c_len):
break
self.account.cleaned_data = dat
self.account.output("cleaned_account.csv")
self.client.output("cleaned_client.csv")
self.disp.output("cleaned_disp.csv")
print("Error in card:")
self.card.clean(self.disp.cur)
self.card.output("cleaned_card.csv")
print("Error in loan:")
self.loan.clean(self.account.cur)
self.loan.output("cleaned_loan.csv")
print("Error in order:")
self.order.clean(self.account.cur)
self.order.output("cleaned_order.csv")
print("Error in trans:")
self.trans.clean(self.account.cur)
self.trans.output("cleaned_trans.csv")
