def active_department_spider():
active_department = SuperSpider(
host='47.102.40.81',
passwd='Abc12345',
db='bryframe',
table_name='active_department',
field_list=('spider_date', 'up_date', 'name', 'buy_number',
'sell_number', 'buy_sum', 'sell_sum', 'business_amount',
'code', 'stock_name'))
page = 1
while True:
try:
json_data = active_department.use_requests_to_html(
f'http://data.eastmoney.com/DataCenter_V3/stock2016/ActiveStatistics/pagesize=50,page={page},sortRule=-1,sortType=JmMoney,startDate={active_department.spider_date},endDate={active_department.spider_date},gpfw=0,js=var%20data_tab_1.html?rt=25754772',
'GB2312')
data_list = active_department.json_to_py(json_data,
deal=True)['data']
except:
print(f'第{page}页获取失败')
page += 1
continue
if not data_list or page == 500:
break
print(f'第{page}页')
for data in data_list:
active_department.up_date = active_department.spider_date
active_department.name = data['YybName']
active_department.buy_number = active_department.to_null(
data['YybBCount'])
active_department.sell_number = active_department.to_null(
data['YybSCount'])
active_department.buy_sum = active_department.to_null(
data['Bmoney'])
active_department.sell_sum = active_department.to_null(
data['Smoney'])
active_department.business_amount = active_department.to_null(
data['JmMoney'])
if not data['SName']:
active_department.code = 'null'
active_department.stock_name = 'null'
active_department.data_save()
else:
for data_s in active_department.json_to_py(data['SName']):
active_department.code = data_s['SCode']
active_department.stock_name = data_s['CodeName']
active_department.data_save()
print(
f'每日活跃营业部：{active_department.up_date}-{active_department.name}-导入完成'
)
page += 1
active_department.spider_end()
print('end：每日活跃营业部')
def institution_business_spider():
institution_business = SuperSpider(
host='47.102.40.81',
passwd='Abc12345',
db='bryframe',
table_name='institution_business',
field_list=('spider_date', 'up_date', 'code', 'name', 'buy_number',
'sell_number', 'buy_sum', 'sell_sum', 'buy_amount'))
page = 1
while True:
try:
json_data = institution_business.use_requests_to_html(
f'http://data.eastmoney.com/DataCenter_V3/stock2016/DailyStockListStatistics/pagesize=50,page={page},sortRule=-1,sortType=PBuy,startDate={institution_business.spider_date},endDate={institution_business.spider_date},gpfw=0,js=var%20data_tab_1.html?rt=25754580',
'GB2312')
data_list = institution_business.json_to_py(json_data,
deal=True)['data']
except:
print(f'第{page}页获取失败')
page += 1
continue
if not data_list or page == 500:
break
print(f'第{page}页')
for data in data_list:
institution_business.up_date = institution_business.spider_date
institution_business.code = data['SCode']
institution_business.name = data['SName']
institution_business.buy_number = institution_business.to_null(
data['BSL'])
institution_business.sell_number = institution_business.to_null(
data['SSL'])
institution_business.buy_sum = institution_business.to_null(
data['BMoney'])
institution_business.sell_sum = institution_business.to_null(
data['SMoney'])
institution_business.buy_amount = institution_business.to_null(
data['PBuy'])
institution_business.data_save()
print(
f'机构买卖情况：{institution_business.up_date}-{institution_business.code}-{institution_business.name}-导入完成'
)
page += 1
institution_business.spider_end()
print('end：机构买卖情况')
