def department_count_spider():
department_count_list = []
department_count = SuperSpider(
host='47.102.40.81',
passwd='Abc12345',
db='bryframe',
table_name='department_count',
field_list=('spider_date', 'up_date', 'name', 'list_time', 'buy_time',
'buy_sum', 'sell_time', 'sell_sum'))
month_ago = department_count.date_ago(30)
page = 1
while True:
try:
json_data = department_count.get_html(
f'http://data.eastmoney.com/DataCenter_V3/stock2016/TraderStatistic/pagesize=50,page={page},sortRule=-1,sortType=,startDate={month_ago},endDate={department_count.spider_date},gpfw=0,js=var%20data_tab_1.html?rt=25754789',
'GB2312')
data_list = department_count.json_to_py(json_data,
deal=True)['data']
except:
print(f'第{page}页获取失败')
page += 1
continue
if not data_list or page == 500:
break
print(f'第{page}页')
for data in data_list:
department_count.up_date = department_count.spider_date
department_count.name = data['SalesName']
if department_count.name not in department_count_list:
department_count_list.append(department_count.name)
else:
print(f'{department_count.name}-数据重复')
continue
sql = f'select name from department_count where name="{department_count.name}" and spider_date="{department_count.spider_date}"'
same_data = department_count.sql_search(sql)
if same_data:
department_count.sql_search(
f'delete from department_count where name="{department_count.name}" and spider_date="{department_count.spider_date}"'
)
print(
f'重新爬取-{department_count.spider_date}-{department_count.name}'
)
department_count.list_time = department_count.to_null(
data['UpCount'])
department_count.buy_time = department_count.to_null(
data['BCount'])
department_count.buy_sum = department_count.to_null(
data['SumActBMoney'])
department_count.sell_time = department_count.to_null(
data['SCount'])
department_count.sell_sum = department_count.to_null(
data['SumActSMoney'])
department_count.data_save()
print(
f'证券营业部上榜统计：{department_count.up_date}-{department_count.name}-导入完成'
)
page += 1
department_count.spider_end()
print('end：证券营业部上榜统计')
def stock_count_spider():
stock_count_list = []
stock_count = SuperSpider(host='47.102.40.81',
passwd='Abc12345',
db='bryframe',
table_name='stock_count',
field_list=('spider_date', 'up_date', 'code',
'name', 'list_time', 'buy_sum',
'sell_sum', 'buy_amount'))
month_ago = stock_count.date_ago(30)
page = 1
while True:
try:
json_data = stock_count.get_html(
f'http://data.eastmoney.com/DataCenter_V3/stock2016/StockStatistic/pagesize=50,page={page},sortRule=-1,sortType=,startDate={month_ago},endDate={stock_count.spider_date},gpfw=0,js=var%20data_tab_3.html?rt=25754758',
'GB2312')
data_list = stock_count.json_to_py(json_data, deal=True)['data']
except:
print(f'第{page}页获取失败')
page += 1
continue
if not data_list or page == 500:
break
print(f'第{page}页')
for data in data_list:
stock_count.up_date = data['Tdate']
stock_count.code = data['SCode']
stock_count.name = data['SName']
if (stock_count.up_date, stock_count.code) not in stock_count_list:
stock_count_list.append(
(stock_count.up_date, stock_count.code))
else:
print(
f'{stock_count.up_date}-{stock_count.code}-{stock_count.name}-数据重复'
)
continue
sql = f'select code from stock_count where code="{stock_count.code}" and spider_date="{stock_count.spider_date}" and up_date="{stock_count.up_date}"'
same_data = stock_count.sql_search(sql)
if same_data:
stock_count.sql_search(
f'delete from stock_count where code="{stock_count.code}" and spider_date="{stock_count.spider_date}" and up_date="{stock_count.up_date}"'
)
print(
f'重新爬取-{stock_count.spider_date}-{stock_count.code}-{stock_count.name}'
)
stock_count.list_time = stock_count.to_null(data['SumCount'])
stock_count.buy_sum = stock_count.to_null(data['Bmoney'])
stock_count.sell_sum = stock_count.to_null(data['Smoney'])
stock_count.buy_amount = stock_count.to_null(data['JmMoney'])
stock_count.data_save()
print(
f'个股龙虎榜统计：{stock_count.up_date}-{stock_count.code}-{stock_count.name}-导入完成'
)
page += 1
stock_count.spider_end()
print('end：个股龙虎榜统计')
def active_department_spider():
active_department = SuperSpider(
host='47.102.40.81',
passwd='Abc12345',
db='bryframe',
table_name='active_department',
field_list=('spider_date', 'up_date', 'name', 'buy_number',
'sell_number', 'buy_sum', 'sell_sum', 'business_amount',
'code', 'stock_name'))
page = 1
while True:
try:
json_data = active_department.use_requests_to_html(
f'http://data.eastmoney.com/DataCenter_V3/stock2016/ActiveStatistics/pagesize=50,page={page},sortRule=-1,sortType=JmMoney,startDate={active_department.spider_date},endDate={active_department.spider_date},gpfw=0,js=var%20data_tab_1.html?rt=25754772',
'GB2312')
data_list = active_department.json_to_py(json_data,
deal=True)['data']
except:
print(f'第{page}页获取失败')
page += 1
continue
if not data_list or page == 500:
break
print(f'第{page}页')
for data in data_list:
active_department.up_date = active_department.spider_date
active_department.name = data['YybName']
active_department.buy_number = active_department.to_null(
data['YybBCount'])
active_department.sell_number = active_department.to_null(
data['YybSCount'])
active_department.buy_sum = active_department.to_null(
data['Bmoney'])
active_department.sell_sum = active_department.to_null(
data['Smoney'])
active_department.business_amount = active_department.to_null(
data['JmMoney'])
if not data['SName']:
active_department.code = 'null'
active_department.stock_name = 'null'
active_department.data_save()
else:
for data_s in active_department.json_to_py(data['SName']):
active_department.code = data_s['SCode']
active_department.stock_name = data_s['CodeName']
active_department.data_save()
print(
f'每日活跃营业部：{active_department.up_date}-{active_department.name}-导入完成'
)
page += 1
active_department.spider_end()
print('end：每日活跃营业部')
def department_track_spider():
department_track = SuperSpider(
host='47.102.40.81',
passwd='Abc12345',
db='bryframe',
table_name='department_track',
field_list=('spider_date', 'up_date', 'code', 'name', 'list_time',
'buy_sum', 'buy_time', 'sell_time', 'buy_amount',
'up_down'))
month_ago = department_track.date_ago(30)
page = 1
while True:
try:
json_data = department_track.use_requests_to_html(
f'http://data.eastmoney.com/DataCenter_V3/stock2016/JgStatistic/pagesize=50,page={page},sortRule=-1,sortType=,startDate={month_ago},endDate={department_track.spider_date},gpfw=0,js=var%20data_tab_3.html?rt=25754592',
'GB2312')
data_list = department_track.json_to_py(json_data,
deal=True)['data']
except:
print(f'第{page}页获取失败')
page += 1
continue
if not data_list or page == 500:
break
print(f'第{page}页')
for data in data_list:
department_track.up_date = department_track.spider_date
department_track.code = data['SCode']
department_track.name = data['SName']
department_track.list_time = department_track.to_null(
data['UPCount'])
department_track.buy_sum = department_track.to_null(
data['JGBMoney'])
department_track.buy_time = department_track.to_null(
data['JGBCount'])
department_track.sell_time = department_track.to_null(
data['JGSCount'])
department_track.buy_amount = department_track.to_null(
data['JGPBuy'])
department_track.up_down = department_track.to_null(
data['RChange1M'])
department_track.data_save()
print(
f'机构席位买卖追踪：{department_track.up_date}-{department_track.code}-{department_track.name}-导入完成'
)
page += 1
department_track.spider_end()
print('end：机构席位买卖追踪')
def institution_business_spider():
institution_business = SuperSpider(
host='47.102.40.81',
passwd='Abc12345',
db='bryframe',
table_name='institution_business',
field_list=('spider_date', 'up_date', 'code', 'name', 'buy_number',
'sell_number', 'buy_sum', 'sell_sum', 'buy_amount'))
page = 1
while True:
try:
json_data = institution_business.use_requests_to_html(
f'http://data.eastmoney.com/DataCenter_V3/stock2016/DailyStockListStatistics/pagesize=50,page={page},sortRule=-1,sortType=PBuy,startDate={institution_business.spider_date},endDate={institution_business.spider_date},gpfw=0,js=var%20data_tab_1.html?rt=25754580',
'GB2312')
data_list = institution_business.json_to_py(json_data,
deal=True)['data']
except:
print(f'第{page}页获取失败')
page += 1
continue
if not data_list or page == 500:
break
print(f'第{page}页')
for data in data_list:
institution_business.up_date = institution_business.spider_date
institution_business.code = data['SCode']
institution_business.name = data['SName']
institution_business.buy_number = institution_business.to_null(
data['BSL'])
institution_business.sell_number = institution_business.to_null(
data['SSL'])
institution_business.buy_sum = institution_business.to_null(
data['BMoney'])
institution_business.sell_sum = institution_business.to_null(
data['SMoney'])
institution_business.buy_amount = institution_business.to_null(
data['PBuy'])
institution_business.data_save()
print(
f'机构买卖情况：{institution_business.up_date}-{institution_business.code}-{institution_business.name}-导入完成'
)
page += 1
institution_business.spider_end()
print('end：机构买卖情况')