def stock_count_spider():
stock_count_list = []
stock_count = SuperSpider(host='47.102.40.81',
passwd='Abc12345',
db='bryframe',
table_name='stock_count',
field_list=('spider_date', 'up_date', 'code',
'name', 'list_time', 'buy_sum',
'sell_sum', 'buy_amount'))
month_ago = stock_count.date_ago(30)
page = 1
while True:
try:
json_data = stock_count.get_html(
f'http://data.eastmoney.com/DataCenter_V3/stock2016/StockStatistic/pagesize=50,page={page},sortRule=-1,sortType=,startDate={month_ago},endDate={stock_count.spider_date},gpfw=0,js=var%20data_tab_3.html?rt=25754758',
'GB2312')
data_list = stock_count.json_to_py(json_data, deal=True)['data']
except:
print(f'第{page}页获取失败')
page += 1
continue
if not data_list or page == 500:
break
print(f'第{page}页')
for data in data_list:
stock_count.up_date = data['Tdate']
stock_count.code = data['SCode']
stock_count.name = data['SName']
if (stock_count.up_date, stock_count.code) not in stock_count_list:
stock_count_list.append(
(stock_count.up_date, stock_count.code))
else:
print(
f'{stock_count.up_date}-{stock_count.code}-{stock_count.name}-数据重复'
)
continue
sql = f'select code from stock_count where code="{stock_count.code}" and spider_date="{stock_count.spider_date}" and up_date="{stock_count.up_date}"'
same_data = stock_count.sql_search(sql)
if same_data:
stock_count.sql_search(
f'delete from stock_count where code="{stock_count.code}" and spider_date="{stock_count.spider_date}" and up_date="{stock_count.up_date}"'
)
print(
f'重新爬取-{stock_count.spider_date}-{stock_count.code}-{stock_count.name}'
)
stock_count.list_time = stock_count.to_null(data['SumCount'])
stock_count.buy_sum = stock_count.to_null(data['Bmoney'])
stock_count.sell_sum = stock_count.to_null(data['Smoney'])
stock_count.buy_amount = stock_count.to_null(data['JmMoney'])
stock_count.data_save()
print(
f'个股龙虎榜统计：{stock_count.up_date}-{stock_count.code}-{stock_count.name}-导入完成'
)
page += 1
stock_count.spider_end()
print('end：个股龙虎榜统计')
def lhb_rank_spider():
lhb_rank_list = []
lhb_rank = SuperSpider(
host='47.102.40.81',
passwd='Abc12345',
db='bryframe',
table_name='lhb_rank',
field_list=('spider_date', 'up_date', 'code', 'name', 'close_price',
'up_down', 'buy_amount', 'change_rate', 'currency_market'))
page = 1
while True:
try:
json_data = lhb_rank.get_html(
url=
f'http://data.eastmoney.com/DataCenter_V3/stock2016/TradeDetail/pagesize=200,page={page},sortRule=-1,sortType=,startDate={lhb_rank.spider_date},endDate={lhb_rank.spider_date},gpfw=0,js=var%20data_tab_1.html?rt=25754497',
charset='GB2312')
data_list = lhb_rank.json_to_py(json_data, deal=True)['data']
except:
print(f'第{page}页获取失败')
page += 1
continue
if not data_list or page == 500:
break
print(f'第{page}页')
for data in data_list:
lhb_rank.up_date = lhb_rank.spider_date
lhb_rank.code = data['SCode']
lhb_rank.name = data['SName']
if lhb_rank.code not in lhb_rank_list:
lhb_rank_list.append(lhb_rank.code)
else:
print(f'{lhb_rank.code}-{lhb_rank.name}-数据重复')
continue
sql = f'select code from lhb_rank where code="{lhb_rank.code}" and spider_date="{lhb_rank.spider_date}"'
same_data = lhb_rank.sql_search(sql)
if same_data:
lhb_rank.sql_search(
f'delete from lhb_rank where code="{lhb_rank.code}" and spider_date="{lhb_rank.spider_date}"'
)
print(
f'重新爬取-{lhb_rank.spider_date}-{lhb_rank.code}-{lhb_rank.name}'
)
lhb_rank.close_price = lhb_rank.to_null(data['ClosePrice'])
lhb_rank.up_down = lhb_rank.to_null(data['Chgradio'])
lhb_rank.buy_amount = lhb_rank.to_null(data['JmMoney'])
lhb_rank.change_rate = lhb_rank.to_null(data['Dchratio'])
lhb_rank.currency_market = lhb_rank.to_null(data['Ltsz'])
lhb_rank.data_save()
print(
f'当日龙虎榜涨跌幅排名：{lhb_rank.up_date}-{lhb_rank.code}-{lhb_rank.name}-导入完成'
)
page += 1
lhb_rank.spider_end()
print('end：龙虎榜当日跌幅排名')
def department_track_spider():
department_track = SuperSpider(
host='47.102.40.81',
passwd='Abc12345',
db='bryframe',
table_name='department_track',
field_list=('spider_date', 'up_date', 'code', 'name', 'list_time',
'buy_sum', 'buy_time', 'sell_time', 'buy_amount',
'up_down'))
month_ago = department_track.date_ago(30)
page = 1
while True:
try:
json_data = department_track.use_requests_to_html(
f'http://data.eastmoney.com/DataCenter_V3/stock2016/JgStatistic/pagesize=50,page={page},sortRule=-1,sortType=,startDate={month_ago},endDate={department_track.spider_date},gpfw=0,js=var%20data_tab_3.html?rt=25754592',
'GB2312')
data_list = department_track.json_to_py(json_data,
deal=True)['data']
except:
print(f'第{page}页获取失败')
page += 1
continue
if not data_list or page == 500:
break
print(f'第{page}页')
for data in data_list:
department_track.up_date = department_track.spider_date
department_track.code = data['SCode']
department_track.name = data['SName']
department_track.list_time = department_track.to_null(
data['UPCount'])
department_track.buy_sum = department_track.to_null(
data['JGBMoney'])
department_track.buy_time = department_track.to_null(
data['JGBCount'])
department_track.sell_time = department_track.to_null(
data['JGSCount'])
department_track.buy_amount = department_track.to_null(
data['JGPBuy'])
department_track.up_down = department_track.to_null(
data['RChange1M'])
department_track.data_save()
print(
f'机构席位买卖追踪：{department_track.up_date}-{department_track.code}-{department_track.name}-导入完成'
)
page += 1
department_track.spider_end()
print('end：机构席位买卖追踪')
def institution_business_spider():
institution_business = SuperSpider(
host='47.102.40.81',
passwd='Abc12345',
db='bryframe',
table_name='institution_business',
field_list=('spider_date', 'up_date', 'code', 'name', 'buy_number',
'sell_number', 'buy_sum', 'sell_sum', 'buy_amount'))
page = 1
while True:
try:
json_data = institution_business.use_requests_to_html(
f'http://data.eastmoney.com/DataCenter_V3/stock2016/DailyStockListStatistics/pagesize=50,page={page},sortRule=-1,sortType=PBuy,startDate={institution_business.spider_date},endDate={institution_business.spider_date},gpfw=0,js=var%20data_tab_1.html?rt=25754580',
'GB2312')
data_list = institution_business.json_to_py(json_data,
deal=True)['data']
except:
print(f'第{page}页获取失败')
page += 1
continue
if not data_list or page == 500:
break
print(f'第{page}页')
for data in data_list:
institution_business.up_date = institution_business.spider_date
institution_business.code = data['SCode']
institution_business.name = data['SName']
institution_business.buy_number = institution_business.to_null(
data['BSL'])
institution_business.sell_number = institution_business.to_null(
data['SSL'])
institution_business.buy_sum = institution_business.to_null(
data['BMoney'])
institution_business.sell_sum = institution_business.to_null(
data['SMoney'])
institution_business.buy_amount = institution_business.to_null(
data['PBuy'])
institution_business.data_save()
print(
f'机构买卖情况：{institution_business.up_date}-{institution_business.code}-{institution_business.name}-导入完成'
)
page += 1
institution_business.spider_end()
print('end：机构买卖情况')
