def stock_count_spider(): stock_count_list = [] stock_count = SuperSpider(host='47.102.40.81', passwd='Abc12345', db='bryframe', table_name='stock_count', field_list=('spider_date', 'up_date', 'code', 'name', 'list_time', 'buy_sum', 'sell_sum', 'buy_amount')) month_ago = stock_count.date_ago(30) page = 1 while True: try: json_data = stock_count.get_html( f'http://data.eastmoney.com/DataCenter_V3/stock2016/StockStatistic/pagesize=50,page={page},sortRule=-1,sortType=,startDate={month_ago},endDate={stock_count.spider_date},gpfw=0,js=var%20data_tab_3.html?rt=25754758', 'GB2312') data_list = stock_count.json_to_py(json_data, deal=True)['data'] except: print(f'第{page}页获取失败') page += 1 continue if not data_list or page == 500: break print(f'第{page}页') for data in data_list: stock_count.up_date = data['Tdate'] stock_count.code = data['SCode'] stock_count.name = data['SName'] if (stock_count.up_date, stock_count.code) not in stock_count_list: stock_count_list.append( (stock_count.up_date, stock_count.code)) else: print( f'{stock_count.up_date}-{stock_count.code}-{stock_count.name}-数据重复' ) continue sql = f'select code from stock_count where code="{stock_count.code}" and spider_date="{stock_count.spider_date}" and up_date="{stock_count.up_date}"' same_data = stock_count.sql_search(sql) if same_data: stock_count.sql_search( f'delete from stock_count where code="{stock_count.code}" and spider_date="{stock_count.spider_date}" and up_date="{stock_count.up_date}"' ) print( f'重新爬取-{stock_count.spider_date}-{stock_count.code}-{stock_count.name}' ) stock_count.list_time = stock_count.to_null(data['SumCount']) stock_count.buy_sum = stock_count.to_null(data['Bmoney']) stock_count.sell_sum = stock_count.to_null(data['Smoney']) stock_count.buy_amount = stock_count.to_null(data['JmMoney']) stock_count.data_save() print( f'个股龙虎榜统计:{stock_count.up_date}-{stock_count.code}-{stock_count.name}-导入完成' ) page += 1 stock_count.spider_end() print('end:个股龙虎榜统计')
def lhb_rank_spider(): lhb_rank_list = [] lhb_rank = SuperSpider( host='47.102.40.81', passwd='Abc12345', db='bryframe', table_name='lhb_rank', field_list=('spider_date', 'up_date', 'code', 'name', 'close_price', 'up_down', 'buy_amount', 'change_rate', 'currency_market')) page = 1 while True: try: json_data = lhb_rank.get_html( url= f'http://data.eastmoney.com/DataCenter_V3/stock2016/TradeDetail/pagesize=200,page={page},sortRule=-1,sortType=,startDate={lhb_rank.spider_date},endDate={lhb_rank.spider_date},gpfw=0,js=var%20data_tab_1.html?rt=25754497', charset='GB2312') data_list = lhb_rank.json_to_py(json_data, deal=True)['data'] except: print(f'第{page}页获取失败') page += 1 continue if not data_list or page == 500: break print(f'第{page}页') for data in data_list: lhb_rank.up_date = lhb_rank.spider_date lhb_rank.code = data['SCode'] lhb_rank.name = data['SName'] if lhb_rank.code not in lhb_rank_list: lhb_rank_list.append(lhb_rank.code) else: print(f'{lhb_rank.code}-{lhb_rank.name}-数据重复') continue sql = f'select code from lhb_rank where code="{lhb_rank.code}" and spider_date="{lhb_rank.spider_date}"' same_data = lhb_rank.sql_search(sql) if same_data: lhb_rank.sql_search( f'delete from lhb_rank where code="{lhb_rank.code}" and spider_date="{lhb_rank.spider_date}"' ) print( f'重新爬取-{lhb_rank.spider_date}-{lhb_rank.code}-{lhb_rank.name}' ) lhb_rank.close_price = lhb_rank.to_null(data['ClosePrice']) lhb_rank.up_down = lhb_rank.to_null(data['Chgradio']) lhb_rank.buy_amount = lhb_rank.to_null(data['JmMoney']) lhb_rank.change_rate = lhb_rank.to_null(data['Dchratio']) lhb_rank.currency_market = lhb_rank.to_null(data['Ltsz']) lhb_rank.data_save() print( f'当日龙虎榜涨跌幅排名:{lhb_rank.up_date}-{lhb_rank.code}-{lhb_rank.name}-导入完成' ) page += 1 lhb_rank.spider_end() print('end:龙虎榜当日跌幅排名')
def department_track_spider(): department_track = SuperSpider( host='47.102.40.81', passwd='Abc12345', db='bryframe', table_name='department_track', field_list=('spider_date', 'up_date', 'code', 'name', 'list_time', 'buy_sum', 'buy_time', 'sell_time', 'buy_amount', 'up_down')) month_ago = department_track.date_ago(30) page = 1 while True: try: json_data = department_track.use_requests_to_html( f'http://data.eastmoney.com/DataCenter_V3/stock2016/JgStatistic/pagesize=50,page={page},sortRule=-1,sortType=,startDate={month_ago},endDate={department_track.spider_date},gpfw=0,js=var%20data_tab_3.html?rt=25754592', 'GB2312') data_list = department_track.json_to_py(json_data, deal=True)['data'] except: print(f'第{page}页获取失败') page += 1 continue if not data_list or page == 500: break print(f'第{page}页') for data in data_list: department_track.up_date = department_track.spider_date department_track.code = data['SCode'] department_track.name = data['SName'] department_track.list_time = department_track.to_null( data['UPCount']) department_track.buy_sum = department_track.to_null( data['JGBMoney']) department_track.buy_time = department_track.to_null( data['JGBCount']) department_track.sell_time = department_track.to_null( data['JGSCount']) department_track.buy_amount = department_track.to_null( data['JGPBuy']) department_track.up_down = department_track.to_null( data['RChange1M']) department_track.data_save() print( f'机构席位买卖追踪:{department_track.up_date}-{department_track.code}-{department_track.name}-导入完成' ) page += 1 department_track.spider_end() print('end:机构席位买卖追踪')
def institution_business_spider(): institution_business = SuperSpider( host='47.102.40.81', passwd='Abc12345', db='bryframe', table_name='institution_business', field_list=('spider_date', 'up_date', 'code', 'name', 'buy_number', 'sell_number', 'buy_sum', 'sell_sum', 'buy_amount')) page = 1 while True: try: json_data = institution_business.use_requests_to_html( f'http://data.eastmoney.com/DataCenter_V3/stock2016/DailyStockListStatistics/pagesize=50,page={page},sortRule=-1,sortType=PBuy,startDate={institution_business.spider_date},endDate={institution_business.spider_date},gpfw=0,js=var%20data_tab_1.html?rt=25754580', 'GB2312') data_list = institution_business.json_to_py(json_data, deal=True)['data'] except: print(f'第{page}页获取失败') page += 1 continue if not data_list or page == 500: break print(f'第{page}页') for data in data_list: institution_business.up_date = institution_business.spider_date institution_business.code = data['SCode'] institution_business.name = data['SName'] institution_business.buy_number = institution_business.to_null( data['BSL']) institution_business.sell_number = institution_business.to_null( data['SSL']) institution_business.buy_sum = institution_business.to_null( data['BMoney']) institution_business.sell_sum = institution_business.to_null( data['SMoney']) institution_business.buy_amount = institution_business.to_null( data['PBuy']) institution_business.data_save() print( f'机构买卖情况:{institution_business.up_date}-{institution_business.code}-{institution_business.name}-导入完成' ) page += 1 institution_business.spider_end() print('end:机构买卖情况')