def update(self): self._setup_dir_() self.prepare() if self.should_update(): start_date: date = max( self.start_date(), StockData().stock_basic.list_date_of(self.stock_code)) end_date: date = start_date last_trade_day = StockData().trade_calendar.latest_trade_day() df_list: List[pd.DataFrame] = [self.dataframe] step_days = timedelta(days=3650) while start_date <= last_trade_day: end_date = start_date + step_days end_date = min(end_date, last_trade_day) df = self.ts_top10_holders(start_date=start_date, end_date=end_date) df_list.append(df) start_date = end_date + timedelta(days=1) self.dataframe = pd.concat(df_list).drop_duplicates() self.dataframe.sort_values(by='ann_date', inplace=True) self.dataframe.to_csv(path_or_buf=self.file_path(), index=False) logging.info(colorama.Fore.YELLOW + '%s [股东增减持] 数据更新到: %s path: %s' % (self.stock_code, str(end_date), self.file_path())) else: logging.info(colorama.Fore.BLUE + '%s [股东增减持] 数据无须更新' % self.stock_code)
def update(self): self._setup_dir_() self.prepare() if self.should_update(): start_date: date = max( self.start_date(), StockData().stock_basic.list_date_of(self.stock_code)) end_date: date = start_date last_trade_day = StockData().trade_calendar.latest_trade_day() df_list: List[pd.DataFrame] = [self.dataframe] step_days = timedelta(days=50) while start_date <= last_trade_day: end_date = start_date + step_days end_date = min(end_date, last_trade_day) rs = bao.query_history_k_data_plus( code=self.stock_code, start_date=str(start_date), end_date=str(end_date), frequency='5', fields= 'date,time,code,open,high,low,close,volume,amount,adjustflag', adjustflag='3') df_5min = rs.get_data() if not df_5min.empty: df_5min['date'] = pd.to_datetime(df_5min['date'], format='%Y-%m-%d') df_5min['time'] = df_5min['time'].apply( lambda x: pd.to_datetime(x[:-3], format='%Y%m%d%H%M%S' )) df_5min['code'] = df_5min['code'].apply( lambda x: ts_code(x)) df_5min['open'] = df_5min['open'].astype(np.float64) df_5min['high'] = df_5min['high'].astype(np.float64) df_5min['low'] = df_5min['low'].astype(np.float64) df_5min['close'] = df_5min['close'].astype(np.float64) df_5min['volume'] = df_5min['volume'].astype(np.float64) df_5min['amount'] = df_5min['amount'].astype(np.float64) df_5min.set_index(keys='time', drop=False, inplace=True) logging.debug(colorama.Fore.YELLOW + '下载 %s 5min 线数据, 从 %s 到 %s 共 %s 条' % (self.stock_code, start_date, end_date, df_5min.shape[0])) df_list.append(df_5min) start_date = end_date + timedelta(days=1) self.dataframe = pd.concat(df_list).drop_duplicates() self.dataframe.sort_index(inplace=True) self.dataframe.to_csv(path_or_buf=self.file_path(), index=False) logging.info(colorama.Fore.YELLOW + '%s 5min 线数据更新到: %s path: %s' % (self.stock_code, str(end_date), self.file_path())) else: logging.info(colorama.Fore.BLUE + '%s 5min 线数据无须更新' % self.stock_code)
def update(self): self._setup_dir_() self.prepare() if self.should_update(): start_date: date = self.start_date() end_date: date = start_date last_trade_day = StockData().trade_calendar.latest_trade_day() df_list: List[pd.DataFrame] = [self.dataframe] step_days = timedelta(days = 10) try: while start_date <= last_trade_day: end_date = start_date + step_days end_date = min(end_date, last_trade_day) df = self.ts_block_trade(start_date = start_date, end_date = end_date) df_list.append(df) start_date = end_date + timedelta(days = 1) except Exception as ex: logging.warning('更新 [大宗交易] 发生异常中断: %s' % ex) raise ex finally: self.dataframe = pd.concat(df_list).drop_duplicates() self.dataframe.sort_values(by = 'trade_date', inplace = True) self.dataframe.to_csv( path_or_buf = self.file_path(), index = False ) logging.info( colorama.Fore.YELLOW + '[大宗交易] 数据更新到: %s path: %s' % (end_date, self.file_path())) else: logging.info(colorama.Fore.BLUE + '%s [大宗交易] 数据无须更新')
def start_date(self) -> date: """ 计算本次更新的起始日期 :return: """ self.prepare() if self.dataframe.empty: return StockData().stock_basic.list_date_of(self.stock_code) else: return self.dataframe.iloc[-1].loc['trade_date'].date() + timedelta(days = 1)
def should_update(self) -> bool: """ 如果数据文件的最后修改日期, 早于最近的一个交易日, 则需要更新数据 如果文件不存在, 直接返回 True :return: """ if not os.path.exists(self.file_path()): return True mtime = mtime_of_file(self.file_path()) if mtime < StockData().trade_calendar.latest_trade_day(): return True else: return False
def need_update_by_trade_date(df: pd.DataFrame, column_name: str) -> bool: """ 根据DataFrame中指定的交易日期的最后记录的值, 如果在最近的交易日之前, 则说明需要更新 :param df: :param column_name: 交易日期对应的字段名称 :return: """ if df.empty: # 如果DataFrame为空, 说明没有数据在本地, 需要更新 return True else: from sz.stock_data.stock_data import StockData return df.iloc[-1].loc[column_name].date() < StockData( ).trade_calendar.latest_trade_day()
def update(self): self._setup_dir_() self.prepare() latest_trade_day = StockData().trade_calendar.latest_trade_day() trad_date_list = StockData().trade_calendar.trade_day_between( from_date=self.start_date(), to_date=StockData().trade_calendar.latest_trade_day()) df_list: List[pd.DataFrame] = [self.dataframe] try: for trade_date in trad_date_list: df = self.ts_top_inst(trade_date) if not df.empty: df_list.append(df) latest_trade_day = trade_date else: logging.warning(colorama.Fore.RED + '没有 %s 的 [龙虎榜机构明细] 数据' % trade_date) except Exception as ex: logging.warning('更新 [龙虎榜机构明细] 发送异常中断: %s' % ex) raise ex finally: if len(df_list) > 1: self.dataframe = pd.concat(df_list).drop_duplicates() self.dataframe.sort_values(by='trade_date', inplace=True) self.dataframe.to_csv(path_or_buf=self.file_path(), index=False) logging.info(colorama.Fore.YELLOW + '[龙虎榜机构明细] 数据更新到: %s path: %s' % (latest_trade_day, self.file_path())) else: logging.info(colorama.Fore.BLUE + '[龙虎榜机构明细] 数据无须更新')
def update(self): self._setup_dir_() self.prepare() if self.should_update(): last_trade_day = StockData().trade_calendar.latest_trade_day() df_list: List[pd.DataFrame] = [self.dataframe] trad_date_list = StockData().trade_calendar.trade_day_between( from_date=self.start_date(), to_date=last_trade_day) try: for trade_date in trad_date_list: df = self.ts_margin_detail(trade_date) if not df.shape: df_list.append(df) except Exception as ex: logging.warning('更新 [融资融券交易明细] 发生异常中断: %s' % ex) raise ex finally: if len(df_list) > 1: self.dataframe = pd.concat(df_list).drop_duplicates() self.dataframe.sort_values(by='trade_date', inplace=True) self.dataframe.to_csv(path_or_buf=self.file_path(), index=False) logging.info(colorama.Fore.YELLOW + '[融资融券交易明细] 数据更新到: %s path: %s' % (last_trade_day, self.file_path())) else: logging.info(colorama.Fore.BLUE + '[融资融券交易明细] 数据无须更新') else: logging.info(colorama.Fore.BLUE + '[融资融券交易明细] 数据无须更新')
def update(self): self._setup_dir_() self.prepare() if self.should_update(): start_date: date = self.start_date() end_date: date = start_date last_trade_day = StockData().trade_calendar.latest_trade_day() df_list: List[pd.DataFrame] = [self.dataframe] step_days = timedelta(days=1000) while start_date <= last_trade_day: end_date = start_date + step_days end_date = min(end_date, last_trade_day) rs = bao.query_history_k_data_plus( code=self.index_code, start_date=str(start_date), end_date=str(end_date), frequency='d', fields= 'date,code,open,high,low,close,preclose,volume,amount,adjustflag,turn,tradestatus,pctChg,peTTM,psTTM,pcfNcfTTM,pbMRQ,isST', adjustflag='3') df = rs.get_data() if not df.empty: df['date'] = pd.to_datetime(df['date'], format='%Y-%m-%d') df['is_open'] = df['isST'].apply(lambda x: str(x) == '1') df['code'] = df['code'].apply(lambda x: ts_code(x)) df.set_index(keys='date', drop=False, inplace=True) logging.debug( colorama.Fore.YELLOW + '下载 [%s 日线] 数据, 从 %s 到 %s' % (self.index_name, str(start_date), str(end_date))) df_list.append(df) start_date = end_date + timedelta(days=1) self.dataframe = pd.concat(df_list).drop_duplicates() self.dataframe.sort_index(inplace=True) self.dataframe.to_csv(path_or_buf=self.file_path(), index=False) logging.info(colorama.Fore.YELLOW + '[%s 日线] 数据更新到: %s path: %s' % (self.index_name, str(end_date), self.file_path())) else: logging.info(colorama.Fore.BLUE + '[%s 日线] 数据无须更新' % self.index_name)
def update_for_stock(stock_code: str): StockDaily(data_dir=StockData().data_dir, stock_code=stock_code).update() Stock5min(data_dir=StockData().data_dir, stock_code=stock_code).update() AdjFactor(data_dir=StockData().data_dir, stock_code=stock_code).update() MoneyFlow(data_dir=StockData().data_dir, stock_code=stock_code).update() Top10Holders(data_dir=StockData().data_dir, stock_code=stock_code).update() Top10FloatHolders(data_dir=StockData().data_dir, stock_code=stock_code).update() StkHolderNumber(data_dir=StockData().data_dir, stock_code=stock_code).update() StkHolderTrade(data_dir=StockData().data_dir, stock_code=stock_code).update() PledgeStat(data_dir=StockData().data_dir, stock_code=stock_code).update() PledgeDetail(data_dir=StockData().data_dir, stock_code=stock_code).update() Suspend(data_dir=StockData().data_dir, stock_code=stock_code).update()
def test(): start_time = datetime.now() StockData().stock_basic.update() StockData().stock_company.update() StockData().zz500.update() StockData().hs300.update() StockTopList(data_dir=StockData().data_dir).update() StockTopInst(data_dir=StockData().data_dir).update() BlockTrade(data_dir=StockData().data_dir).update() StockConcept(data_dir=StockData().data_dir).update() StockMargin(data_dir=StockData().data_dir).update() StockMarginDetail(data_dir=StockData().data_dir).update() StockIndustry(data_dir=StockData().data_dir).update() IndexBasic(data_dir=StockData().data_dir).update() for index_code in StockData().index_basic.default_index_pool(): IndexDaily(data_dir=StockData().data_dir, index_code=index_code).update() stock_list = [] stock_list.extend(StockData().hs300.stock_codes()) stock_list.extend(StockData().zz500.stock_codes()) total_count = len(stock_list) finished_count = 0 for stock_code in stock_list: update_for_stock(stock_code) finished_count += 1 logging.debug( colorama.Fore.LIGHTGREEN_EX + '股票 %s 更新完毕, 进度: (%s/%s) %.2f %%' % (StockData().stock_basic.name_of(ts_code=stock_code), finished_count, total_count, finished_count / total_count * 100)) # for stock_code in StockData().hs300.stock_codes(): # update_for_stock(stock_code) # # for stock_code in StockData().zz500.stock_codes(): # update_for_stock(stock_code) logging.info(colorama.Fore.YELLOW + '更新完毕') end_time = datetime.now() logging.info(colorama.Fore.YELLOW + '本次更新总共耗时: %s' % (end_time - start_time))
logging.info(colorama.Fore.YELLOW + '更新完毕') end_time = datetime.now() logging.info(colorama.Fore.YELLOW + '本次更新总共耗时: %s' % (end_time - start_time)) def update_for_stock(stock_code: str): StockDaily(data_dir=StockData().data_dir, stock_code=stock_code).update() Stock5min(data_dir=StockData().data_dir, stock_code=stock_code).update() AdjFactor(data_dir=StockData().data_dir, stock_code=stock_code).update() MoneyFlow(data_dir=StockData().data_dir, stock_code=stock_code).update() Top10Holders(data_dir=StockData().data_dir, stock_code=stock_code).update() Top10FloatHolders(data_dir=StockData().data_dir, stock_code=stock_code).update() StkHolderNumber(data_dir=StockData().data_dir, stock_code=stock_code).update() StkHolderTrade(data_dir=StockData().data_dir, stock_code=stock_code).update() PledgeStat(data_dir=StockData().data_dir, stock_code=stock_code).update() PledgeDetail(data_dir=StockData().data_dir, stock_code=stock_code).update() Suspend(data_dir=StockData().data_dir, stock_code=stock_code).update() if __name__ == '__main__': bao_login() StockData().setup(data_dir='/Volumes/USBDATA/stock_data') test() bao_logout()
def __init__(self, data_dir: str, index_code: str): self.data_dir = data_dir self.index_code = ts_code(index_code) self.dataframe: Union[pd.DataFrame, None] = None self.index_name = StockData().index_basic.name_of_index( self.index_code)