def get_CCBH_product():
"""
建设银行 C10105
请求格式：http post
返回格式：html
:return:
"""
legal_group = 'CCBH'
issuer_code = 'C10105'
root_url = 'http://finance.ccb.com'
index_url = 'http://finance.ccb.com/Channel/3080'
product_data = []
logger_local.info(unicode(legal_group) + ' - Begin parsing...')
response = requests.post(index_url, data={"querytype": "query", "saleStatus": "2", "investmentCurrency": "14"})
soup = bs4.BeautifulSoup(response.text, "html.parser")
pl_data_table = soup.find("table", id="pl_data_list")
for child in pl_data_table.children:
if isinstance(child, bs4.element.Tag):
if child.has_attr("onmouseover"):
if child["onmouseover"] == "this.className='table_select_bg AcqProductItem'":
cells = child.find_all("td")
product_data.append([issuer_code,
cells[13]["id"],
cells[0]["title"],
u'USD',
cells[4].string.strip(),
decode(re.sub(r'[^\d.]+', '', cells[11].string),
u'', u'0',
re.sub(r'[^\d.]+', '', cells[11].string)),
decode(cells[12].string.strip(), u'低风险', u'L', u''),
cells[1].string.strip(),
u'',
]
)
logger_local.debug(unicode(legal_group) + ' - ' + repr_zh(product_data[-1]))
# DB manipulation:
cursor = cnx.cursor()
cursor.execute("""DELETE FROM t_product WHERE data_source='OW' and issuer_code=%s
and date(update_time)=curdate()""", (issuer_code,))
logger_local.info(unicode(legal_group) + ' - ' + unicode(cursor.rowcount) + ' rows deleted')
add_product = ("""INSERT INTO t_product
(issuer_code, prod_code, prod_name,currency, tenor_desc, expected_highest_yield, risk_desc,
status, remark, buyable, data_source, update_time)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'Y', 'OW', now())""")
for p in product_data:
cursor.execute(add_product, p)
cnx.commit()
cursor.close()
logger_local.info(unicode(legal_group) + ' - ' + unicode(len(product_data)) + ' products imported')
def currency_decoder(tenor_desc):
result = decode(
tenor_desc,
u"美元",
"USD",
u"澳元",
"AUD",
u"澳大利亚元",
"AUD",
u"欧元",
"EUR",
u"英镑",
"GBP",
u"日元",
"JPY",
u"港元",
"HKD",
u"港币",
"HKD",
u"加元",
"CAD",
u"加拿大元",
"CAD",
"",
)
return result
def currency_encoder(tenor_desc):
result = butils.decode(tenor_desc,
'USD', u'美元',
'AUD', u'澳元',
'EUR', u'欧元',
'GBP', u'英镑',
'')
return result
def tenor_decoder(tenor_desc):
result = butils.decode(tenor_desc,
u'1个月', '30',
u'2个月', '60',
u'3个月', '90',
u'6个月', '180',
u'9个月', '270',
u'12个月', '360',
u'半年', '180',
u'1年', '360',
u'一年', '360',
u'2年', '720',
u'二年', '720',
u'3年', '1080',
u'三年', '1080',
u'5年', '1800',
u'五年', '1800',
'')
return result
def get_ABCI_product():
"""
农业银行 C10103
请求格式：http get
返回格式：json
:return:
"""
legal_group = 'ABCI'
issuer_code = 'C10103'
# page_index = 1
# page_max = 150
root_url = 'http://ewealth.abchina.com'
manual_url = 'http://ewealth.abchina.com/fs/filter/'
index_url = 'http://ewealth.abchina.com/app/data/api/DataService/BoeProductV2?i=1&s=100&o=0&w=%25E5%258F%25AF%2' \
'5E5%2594%25AE%257C%257C%257C%25E7%25BE%258E%25E5%2585%2583%257C%257C%257C%257C1%257C%257C0%257C%257C0'
product_data = []
logger_local.info(unicode(legal_group) + ' - Begin parsing...')
response = requests.get(index_url)
soup = bs4.BeautifulSoup(response.text, "html.parser")
data_string = json.loads(soup.text)
# <ProductNo>BF161002</ProductNo>
# <ProdName>“金钥匙·本利丰”2016年第1002期美元理财产品</ProdName>
# <ProdClass>封闭</ProdClass>
# <ProdLimit>363天</ProdLimit>
# <ProdProfit>1.3%</ProdProfit>
# <ProdYildType>保证收益</ProdYildType>
# <PrdYildTypeOrder>0</PrdYildTypeOrder>
# <ProdArea>全国发行</ProdArea>
# <szComDat>2016.01.12</szComDat>
# <ProdSaleDate>16.01.12-16.01.18</ProdSaleDate>
# <IsCanBuy>0</IsCanBuy>
# <PurStarAmo>9000.00</PurStarAmo>
# <RowNumber>1</RowNumber>
for p in data_string["Data"]["Table"]:
product_data.append([issuer_code,
issuer_list[issuer_code],
u'USD',
p["ProductNo"],
p["ProdName"],
p["ProdClass"],
re.sub(r'[^\d]+', '', p["ProdLimit"]),
decode(re.sub(r'[^\d.]+', '', p["ProdProfit"]),
u'', u'0',
re.sub(r'[^\d.]+', '', p["ProdProfit"])),
decode(p["ProdYildType"], u'保证收益', u'Y', u'N'),
p["ProdArea"],
re.sub(r'[^\d]+', '', p["szComDat"]),
'20' + re.sub(r'[^\d]+', '', p["ProdSaleDate"].split("-")[0]),
'20' + re.sub(r'[^\d]+', '', p["ProdSaleDate"].split("-")[1])])
logger_local.debug(unicode(legal_group) + ' - ' + repr_zh(product_data[-1]))
# DB manipulation:
cursor = cnx.cursor()
cursor.execute("""DELETE FROM t_product WHERE data_source='OW' and issuer_code=%s
and date(update_time)=curdate()""", (issuer_code,))
logger_local.info(unicode(legal_group) + ' - ' + unicode(cursor.rowcount) + ' rows deleted')
add_product = ("""INSERT INTO t_product
(issuer_code, issuer_name, currency, prod_code, prod_name, redeemable, tenor, expected_highest_yield,
preservable, sales_region_desc, start_date, open_start_date, open_end_date, buyable, data_source,
update_time)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'Y', 'OW', now())""")
for p in product_data:
cursor.execute(add_product, p)
cnx.commit()
cursor.close()
logger_local.info(unicode(legal_group) + ' - ' + unicode(len(product_data)) + ' products imported')
def get_ICBC_product():
"""
工商银行 C10102
请求格式：http get
返回格式：json
:return:
"""
legal_group = 'ICBC'
issuer_code = 'C10102'
root_url = 'http://www.icbc.com.cn'
index_url = 'http://www.icbc.com.cn/ICBCDynamicSite2/money/services/MoenyListService.ashx?ctl1=4&ctl2=6&keyword='
product_data = []
logger_local.info(unicode(legal_group) + ' - Begin parsing...')
data_string = json.loads(requests.get(index_url).text.encode('utf-8'))
# logger_local.debug('data_string = ' + unicode(data_string))
for pd in data_string:
product_data.append([issuer_code,
pd["buyPaamt"],
decode(pd["buyflag"], '1', 'Y', 'N'),
# pd["categoryL1"],
# pd["categoryL2"],
re.sub(r'[^\d.]+', '', pd["intendYield"]),
# pd["introJs"],
# pd["isUnitValue"],
pd["matudate"],
pd["offerPeriod"][:8],
pd["offerPeriod"][-8:],
pd["prodID"],
filter(unicode.isalpha, pd["prodID"]),
filter(unicode.isdigit, pd["prodID"]),
# pd["prodintro"],
pd["productName"],
pd["productTerm"],
pd["saleZone"],
pd["sellStatus"]])
# pd["totalValue"],
# pd["value"],
# pd["valueLink"],
# pd["workdate"]])
logger_local.debug(unicode(legal_group) + ' - ' + repr_zh(product_data[-1]))
# DB manipulation:
cursor = cnx.cursor()
cursor.execute("""DELETE FROM t_product WHERE data_source='OW' and issuer_code=%s
and date(update_time)=curdate()""", (issuer_code,))
logger_local.info(unicode(legal_group) + ' - ' + unicode(cursor.rowcount) + ' rows deleted')
add_product = ("""INSERT INTO t_product
(issuer_code, starting_amount, buyable, expected_highest_yield, maturity_date, start_date,
end_date, prod_code, currency, tenor, prod_name, tenor_desc, sales_region_desc, status,
data_source, update_time)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'OW', now())""")
for p in product_data:
cursor.execute(add_product, p)
cnx.commit()
cursor.close()
logger_local.info(unicode(legal_group) + ' - ' + unicode(len(product_data)) + ' products imported')