def update(self):
self._setup_dir_()
self.prepare()
if self.should_update():
start_date: date = max(
self.start_date(),
StockData().stock_basic.list_date_of(self.stock_code))
end_date: date = start_date
last_trade_day = StockData().trade_calendar.latest_trade_day()
df_list: List[pd.DataFrame] = [self.dataframe]
step_days = timedelta(days=3650)
while start_date <= last_trade_day:
end_date = start_date + step_days
end_date = min(end_date, last_trade_day)
df = self.ts_top10_holders(start_date=start_date,
end_date=end_date)
df_list.append(df)
start_date = end_date + timedelta(days=1)
self.dataframe = pd.concat(df_list).drop_duplicates()
self.dataframe.sort_values(by='ann_date', inplace=True)
self.dataframe.to_csv(path_or_buf=self.file_path(), index=False)
logging.info(colorama.Fore.YELLOW +
'%s [股东增减持] 数据更新到: %s path: %s' %
(self.stock_code, str(end_date), self.file_path()))
else:
logging.info(colorama.Fore.BLUE +
'%s [股东增减持] 数据无须更新' % self.stock_code)
def update(self):
self._setup_dir_()
self.prepare()
if self.should_update():
start_date: date = max(
self.start_date(),
StockData().stock_basic.list_date_of(self.stock_code))
end_date: date = start_date
last_trade_day = StockData().trade_calendar.latest_trade_day()
df_list: List[pd.DataFrame] = [self.dataframe]
step_days = timedelta(days=50)
while start_date <= last_trade_day:
end_date = start_date + step_days
end_date = min(end_date, last_trade_day)
rs = bao.query_history_k_data_plus(
code=self.stock_code,
start_date=str(start_date),
end_date=str(end_date),
frequency='5',
fields=
'date,time,code,open,high,low,close,volume,amount,adjustflag',
adjustflag='3')
df_5min = rs.get_data()
if not df_5min.empty:
df_5min['date'] = pd.to_datetime(df_5min['date'],
format='%Y-%m-%d')
df_5min['time'] = df_5min['time'].apply(
lambda x: pd.to_datetime(x[:-3], format='%Y%m%d%H%M%S'
))
df_5min['code'] = df_5min['code'].apply(
lambda x: ts_code(x))
df_5min['open'] = df_5min['open'].astype(np.float64)
df_5min['high'] = df_5min['high'].astype(np.float64)
df_5min['low'] = df_5min['low'].astype(np.float64)
df_5min['close'] = df_5min['close'].astype(np.float64)
df_5min['volume'] = df_5min['volume'].astype(np.float64)
df_5min['amount'] = df_5min['amount'].astype(np.float64)
df_5min.set_index(keys='time', drop=False, inplace=True)
logging.debug(colorama.Fore.YELLOW +
'下载 %s 5min 线数据, 从 %s 到 %s 共 %s 条' %
(self.stock_code, start_date, end_date,
df_5min.shape[0]))
df_list.append(df_5min)
start_date = end_date + timedelta(days=1)
self.dataframe = pd.concat(df_list).drop_duplicates()
self.dataframe.sort_index(inplace=True)
self.dataframe.to_csv(path_or_buf=self.file_path(), index=False)
logging.info(colorama.Fore.YELLOW + '%s 5min 线数据更新到: %s path: %s' %
(self.stock_code, str(end_date), self.file_path()))
else:
logging.info(colorama.Fore.BLUE +
'%s 5min 线数据无须更新' % self.stock_code)
def update(self):
self._setup_dir_()
self.prepare()
if self.should_update():
start_date: date = self.start_date()
end_date: date = start_date
last_trade_day = StockData().trade_calendar.latest_trade_day()
df_list: List[pd.DataFrame] = [self.dataframe]
step_days = timedelta(days = 10)
try:
while start_date <= last_trade_day:
end_date = start_date + step_days
end_date = min(end_date, last_trade_day)
df = self.ts_block_trade(start_date = start_date, end_date = end_date)
df_list.append(df)
start_date = end_date + timedelta(days = 1)
except Exception as ex:
logging.warning('更新 [大宗交易] 发生异常中断: %s' % ex)
raise ex
finally:
self.dataframe = pd.concat(df_list).drop_duplicates()
self.dataframe.sort_values(by = 'trade_date', inplace = True)
self.dataframe.to_csv(
path_or_buf = self.file_path(),
index = False
)
logging.info(
colorama.Fore.YELLOW + '[大宗交易] 数据更新到: %s path: %s' % (end_date, self.file_path()))
else:
logging.info(colorama.Fore.BLUE + '%s [大宗交易] 数据无须更新')
def start_date(self) -> date:
"""
计算本次更新的起始日期
:return:
"""
self.prepare()
if self.dataframe.empty:
return StockData().stock_basic.list_date_of(self.stock_code)
else:
return self.dataframe.iloc[-1].loc['trade_date'].date() + timedelta(days = 1)
def should_update(self) -> bool:
"""
如果数据文件的最后修改日期, 早于最近的一个交易日, 则需要更新数据
如果文件不存在, 直接返回 True
:return:
"""
if not os.path.exists(self.file_path()):
return True
mtime = mtime_of_file(self.file_path())
if mtime < StockData().trade_calendar.latest_trade_day():
return True
else:
return False
def need_update_by_trade_date(df: pd.DataFrame, column_name: str) -> bool:
"""
根据DataFrame中指定的交易日期的最后记录的值, 如果在最近的交易日之前, 则说明需要更新
:param df:
:param column_name: 交易日期对应的字段名称
:return:
"""
if df.empty:
# 如果DataFrame为空, 说明没有数据在本地, 需要更新
return True
else:
from sz.stock_data.stock_data import StockData
return df.iloc[-1].loc[column_name].date() < StockData(
).trade_calendar.latest_trade_day()
def update(self):
self._setup_dir_()
self.prepare()
latest_trade_day = StockData().trade_calendar.latest_trade_day()
trad_date_list = StockData().trade_calendar.trade_day_between(
from_date=self.start_date(),
to_date=StockData().trade_calendar.latest_trade_day())
df_list: List[pd.DataFrame] = [self.dataframe]
try:
for trade_date in trad_date_list:
df = self.ts_top_inst(trade_date)
if not df.empty:
df_list.append(df)
latest_trade_day = trade_date
else:
logging.warning(colorama.Fore.RED +
'没有 %s 的 [龙虎榜机构明细] 数据' % trade_date)
except Exception as ex:
logging.warning('更新 [龙虎榜机构明细] 发送异常中断: %s' % ex)
raise ex
finally:
if len(df_list) > 1:
self.dataframe = pd.concat(df_list).drop_duplicates()
self.dataframe.sort_values(by='trade_date', inplace=True)
self.dataframe.to_csv(path_or_buf=self.file_path(),
index=False)
logging.info(colorama.Fore.YELLOW +
'[龙虎榜机构明细] 数据更新到: %s path: %s' %
(latest_trade_day, self.file_path()))
else:
logging.info(colorama.Fore.BLUE + '[龙虎榜机构明细] 数据无须更新')
def update(self):
self._setup_dir_()
self.prepare()
if self.should_update():
last_trade_day = StockData().trade_calendar.latest_trade_day()
df_list: List[pd.DataFrame] = [self.dataframe]
trad_date_list = StockData().trade_calendar.trade_day_between(
from_date=self.start_date(), to_date=last_trade_day)
try:
for trade_date in trad_date_list:
df = self.ts_margin_detail(trade_date)
if not df.shape:
df_list.append(df)
except Exception as ex:
logging.warning('更新 [融资融券交易明细] 发生异常中断: %s' % ex)
raise ex
finally:
if len(df_list) > 1:
self.dataframe = pd.concat(df_list).drop_duplicates()
self.dataframe.sort_values(by='trade_date', inplace=True)
self.dataframe.to_csv(path_or_buf=self.file_path(),
index=False)
logging.info(colorama.Fore.YELLOW +
'[融资融券交易明细] 数据更新到: %s path: %s' %
(last_trade_day, self.file_path()))
else:
logging.info(colorama.Fore.BLUE + '[融资融券交易明细] 数据无须更新')
else:
logging.info(colorama.Fore.BLUE + '[融资融券交易明细] 数据无须更新')
def update(self):
self._setup_dir_()
self.prepare()
if self.should_update():
start_date: date = self.start_date()
end_date: date = start_date
last_trade_day = StockData().trade_calendar.latest_trade_day()
df_list: List[pd.DataFrame] = [self.dataframe]
step_days = timedelta(days=1000)
while start_date <= last_trade_day:
end_date = start_date + step_days
end_date = min(end_date, last_trade_day)
rs = bao.query_history_k_data_plus(
code=self.index_code,
start_date=str(start_date),
end_date=str(end_date),
frequency='d',
fields=
'date,code,open,high,low,close,preclose,volume,amount,adjustflag,turn,tradestatus,pctChg,peTTM,psTTM,pcfNcfTTM,pbMRQ,isST',
adjustflag='3')
df = rs.get_data()
if not df.empty:
df['date'] = pd.to_datetime(df['date'], format='%Y-%m-%d')
df['is_open'] = df['isST'].apply(lambda x: str(x) == '1')
df['code'] = df['code'].apply(lambda x: ts_code(x))
df.set_index(keys='date', drop=False, inplace=True)
logging.debug(
colorama.Fore.YELLOW + '下载 [%s 日线] 数据, 从 %s 到 %s' %
(self.index_name, str(start_date), str(end_date)))
df_list.append(df)
start_date = end_date + timedelta(days=1)
self.dataframe = pd.concat(df_list).drop_duplicates()
self.dataframe.sort_index(inplace=True)
self.dataframe.to_csv(path_or_buf=self.file_path(), index=False)
logging.info(colorama.Fore.YELLOW + '[%s 日线] 数据更新到: %s path: %s' %
(self.index_name, str(end_date), self.file_path()))
else:
logging.info(colorama.Fore.BLUE +
'[%s 日线] 数据无须更新' % self.index_name)
def update_for_stock(stock_code: str):
StockDaily(data_dir=StockData().data_dir, stock_code=stock_code).update()
Stock5min(data_dir=StockData().data_dir, stock_code=stock_code).update()
AdjFactor(data_dir=StockData().data_dir, stock_code=stock_code).update()
MoneyFlow(data_dir=StockData().data_dir, stock_code=stock_code).update()
Top10Holders(data_dir=StockData().data_dir, stock_code=stock_code).update()
Top10FloatHolders(data_dir=StockData().data_dir,
stock_code=stock_code).update()
StkHolderNumber(data_dir=StockData().data_dir,
stock_code=stock_code).update()
StkHolderTrade(data_dir=StockData().data_dir,
stock_code=stock_code).update()
PledgeStat(data_dir=StockData().data_dir, stock_code=stock_code).update()
PledgeDetail(data_dir=StockData().data_dir, stock_code=stock_code).update()
Suspend(data_dir=StockData().data_dir, stock_code=stock_code).update()
def test():
start_time = datetime.now()
StockData().stock_basic.update()
StockData().stock_company.update()
StockData().zz500.update()
StockData().hs300.update()
StockTopList(data_dir=StockData().data_dir).update()
StockTopInst(data_dir=StockData().data_dir).update()
BlockTrade(data_dir=StockData().data_dir).update()
StockConcept(data_dir=StockData().data_dir).update()
StockMargin(data_dir=StockData().data_dir).update()
StockMarginDetail(data_dir=StockData().data_dir).update()
StockIndustry(data_dir=StockData().data_dir).update()
IndexBasic(data_dir=StockData().data_dir).update()
for index_code in StockData().index_basic.default_index_pool():
IndexDaily(data_dir=StockData().data_dir,
index_code=index_code).update()
stock_list = []
stock_list.extend(StockData().hs300.stock_codes())
stock_list.extend(StockData().zz500.stock_codes())
total_count = len(stock_list)
finished_count = 0
for stock_code in stock_list:
update_for_stock(stock_code)
finished_count += 1
logging.debug(
colorama.Fore.LIGHTGREEN_EX + '股票 %s 更新完毕, 进度: (%s/%s) %.2f %%' %
(StockData().stock_basic.name_of(ts_code=stock_code),
finished_count, total_count, finished_count / total_count * 100))
# for stock_code in StockData().hs300.stock_codes():
# update_for_stock(stock_code)
#
# for stock_code in StockData().zz500.stock_codes():
# update_for_stock(stock_code)
logging.info(colorama.Fore.YELLOW + '更新完毕')
end_time = datetime.now()
logging.info(colorama.Fore.YELLOW + '本次更新总共耗时: %s' %
(end_time - start_time))
logging.info(colorama.Fore.YELLOW + '更新完毕')
end_time = datetime.now()
logging.info(colorama.Fore.YELLOW + '本次更新总共耗时: %s' %
(end_time - start_time))
def update_for_stock(stock_code: str):
StockDaily(data_dir=StockData().data_dir, stock_code=stock_code).update()
Stock5min(data_dir=StockData().data_dir, stock_code=stock_code).update()
AdjFactor(data_dir=StockData().data_dir, stock_code=stock_code).update()
MoneyFlow(data_dir=StockData().data_dir, stock_code=stock_code).update()
Top10Holders(data_dir=StockData().data_dir, stock_code=stock_code).update()
Top10FloatHolders(data_dir=StockData().data_dir,
stock_code=stock_code).update()
StkHolderNumber(data_dir=StockData().data_dir,
stock_code=stock_code).update()
StkHolderTrade(data_dir=StockData().data_dir,
stock_code=stock_code).update()
PledgeStat(data_dir=StockData().data_dir, stock_code=stock_code).update()
PledgeDetail(data_dir=StockData().data_dir, stock_code=stock_code).update()
Suspend(data_dir=StockData().data_dir, stock_code=stock_code).update()
if __name__ == '__main__':
bao_login()
StockData().setup(data_dir='/Volumes/USBDATA/stock_data')
test()
bao_logout()
def __init__(self, data_dir: str, index_code: str):
self.data_dir = data_dir
self.index_code = ts_code(index_code)
self.dataframe: Union[pd.DataFrame, None] = None
self.index_name = StockData().index_basic.name_of_index(
self.index_code)