def department_count_spider(): department_count_list = [] department_count = SuperSpider( host='47.102.40.81', passwd='Abc12345', db='bryframe', table_name='department_count', field_list=('spider_date', 'up_date', 'name', 'list_time', 'buy_time', 'buy_sum', 'sell_time', 'sell_sum')) month_ago = department_count.date_ago(30) page = 1 while True: try: json_data = department_count.get_html( f'http://data.eastmoney.com/DataCenter_V3/stock2016/TraderStatistic/pagesize=50,page={page},sortRule=-1,sortType=,startDate={month_ago},endDate={department_count.spider_date},gpfw=0,js=var%20data_tab_1.html?rt=25754789', 'GB2312') data_list = department_count.json_to_py(json_data, deal=True)['data'] except: print(f'第{page}页获取失败') page += 1 continue if not data_list or page == 500: break print(f'第{page}页') for data in data_list: department_count.up_date = department_count.spider_date department_count.name = data['SalesName'] if department_count.name not in department_count_list: department_count_list.append(department_count.name) else: print(f'{department_count.name}-数据重复') continue sql = f'select name from department_count where name="{department_count.name}" and spider_date="{department_count.spider_date}"' same_data = department_count.sql_search(sql) if same_data: department_count.sql_search( f'delete from department_count where name="{department_count.name}" and spider_date="{department_count.spider_date}"' ) print( f'重新爬取-{department_count.spider_date}-{department_count.name}' ) department_count.list_time = department_count.to_null( data['UpCount']) department_count.buy_time = department_count.to_null( data['BCount']) department_count.buy_sum = department_count.to_null( data['SumActBMoney']) department_count.sell_time = department_count.to_null( data['SCount']) department_count.sell_sum = department_count.to_null( data['SumActSMoney']) department_count.data_save() print( f'证券营业部上榜统计:{department_count.up_date}-{department_count.name}-导入完成' ) page += 1 department_count.spider_end() print('end:证券营业部上榜统计')
def department_track_spider(): department_track = SuperSpider( host='47.102.40.81', passwd='Abc12345', db='bryframe', table_name='department_track', field_list=('spider_date', 'up_date', 'code', 'name', 'list_time', 'buy_sum', 'buy_time', 'sell_time', 'buy_amount', 'up_down')) month_ago = department_track.date_ago(30) page = 1 while True: try: json_data = department_track.use_requests_to_html( f'http://data.eastmoney.com/DataCenter_V3/stock2016/JgStatistic/pagesize=50,page={page},sortRule=-1,sortType=,startDate={month_ago},endDate={department_track.spider_date},gpfw=0,js=var%20data_tab_3.html?rt=25754592', 'GB2312') data_list = department_track.json_to_py(json_data, deal=True)['data'] except: print(f'第{page}页获取失败') page += 1 continue if not data_list or page == 500: break print(f'第{page}页') for data in data_list: department_track.up_date = department_track.spider_date department_track.code = data['SCode'] department_track.name = data['SName'] department_track.list_time = department_track.to_null( data['UPCount']) department_track.buy_sum = department_track.to_null( data['JGBMoney']) department_track.buy_time = department_track.to_null( data['JGBCount']) department_track.sell_time = department_track.to_null( data['JGSCount']) department_track.buy_amount = department_track.to_null( data['JGPBuy']) department_track.up_down = department_track.to_null( data['RChange1M']) department_track.data_save() print( f'机构席位买卖追踪:{department_track.up_date}-{department_track.code}-{department_track.name}-导入完成' ) page += 1 department_track.spider_end() print('end:机构席位买卖追踪')