def main():
df_stocklist = pd.read_csv(
DataLocationConfig.STOCKLIST_FILE
)
Logger.d(TAG, df_stocklist['銘柄コード'].unique())
codes = df_stocklist['銘柄コード'].unique()
STOCKPRICE_FILEPATH_FMT = 's3://fin-app/stockprice_concat/{code}.csv'
for code in codes[100:101]:
code = 1382
try:
df = pd.read_csv(
STOCKPRICE_FILEPATH_FMT.format(code=code)
)
except Exception as e:
Logger.e(TAG, f'failed to load csv file from s3 : {e}')
continue
df['日付'] = pd.to_datetime(df['日付'])
df = df.set_index('日付')
df = df.rename(columns={
'始値': 'open',
'高値': 'high',
'安値': 'low',
'終値': 'close'
})
df.sort_index(inplace=True)
df['last_close'] = df['close'].shift(1)
df.dropna(inplace=True)
print(df.tail())
df['stop_high_low'] = df.apply(lambda x: check_stop_high_low(
x['last_close'],
x['high'],
x['low']
), axis=1)
print(df[df['stop_high_low']==1])
def on_failed(self, e, args):
Logger.e(TAG, f'on_failed : {e}')
Logger.e(TAG, '=' * 100)
def on_failed(
self,
e: Exception,
args: Dict,
) -> None:
Logger.e(TAG, f'on_failed : {args["code"]} : {e}')
def on_failed(self, e, args):
Logger.e(TAG, f'on_failed : {e} : {args}')
def main():
df_stocklist = pd.read_csv(
DataLocationConfig.STOCKLIST_FILE
)
Logger.d(TAG, df_stocklist['銘柄コード'].unique())
codes = df_stocklist['銘柄コード'].unique()
STOCKPRICE_FILEPATH_FMT = 's3://fin-app/stockprice_concat/{code}.csv'
METADATA_LOCAL_FILEPATH = '/tmp/DAILY_WINDOW-120d_STRIDE-30d_WIDTH-0.5_stockprice_metadata.csv'
METADATA_S3_FILEPATH = os.path.join(
DataLocationConfig.STOCKPRICE_CANDLECHART_BASEDIR.replace('s3://fin-app/', ''),
f'metadata/DAILY_WINDOW-120d_STRIDE-30d_WIDTH-0.5/stockprice_metadata.csv'
)
s3_filepath_list = []
start_dt_str_list = []
end_dt_str_list = []
code_list = []
change_rate_list = []
for code in tqdm(codes[:]):
Logger.i(TAG, code)
files = S3.get_filelist(
basedir=os.path.join(
DataLocationConfig.STOCKPRICE_CANDLECHART_BASEDIR.replace('s3://fin-app/', ''),
# 'DAILY_WINDOW-120d_STRIDE-30d_WIDTH-0.5/1301'
f'DAILY_WINDOW-120d_STRIDE-30d_WIDTH-0.5/{code}'
)
)
start_dt_str = [file.split('/')[-1].replace('.png', '').split('_')[0] for file in files]
end_dt_str = [file.split('/')[-1].replace('.png', '').split('_')[1] for file in files]
s3_filepath_list += files
start_dt_str_list += start_dt_str
end_dt_str_list += end_dt_str
code_list += [code]*len(files)
Logger.i(TAG, f'len(files) : {len(files)}')
Logger.i(TAG, f'len(s3_filepath_list) : {len(s3_filepath_list)}')
try:
df = pd.read_csv(
STOCKPRICE_FILEPATH_FMT.format(code=code)
)
except Exception as e:
Logger.e(TAG, f'failed to load csv file from s3 : {e}')
change_rate_list += [None]*len(files)
continue
df['日付'] = pd.to_datetime(df['日付'])
df = df.set_index('日付')
df = df.rename(columns={
'始値': 'open',
'高値': 'high',
'安値': 'low',
'終値': 'close'
})
MAX_DT = df.index.max()
for sds, eds in zip(start_dt_str, end_dt_str):
if len(df[sds:eds]) == 0:
change_rate_list.append(None)
continue
edt = datetime.strptime(eds, '%Y-%m-%d')
for i in range(119):
try:
df.loc[edt]
break
except Exception:
edt -= timedelta(days=1)
continue
#raise Exception('')
change_rate_start_dt = edt + timedelta(days=1)
change_rate_end_dt = change_rate_start_dt + timedelta(days=30)
if change_rate_end_dt > MAX_DT or len(df[change_rate_start_dt:change_rate_end_dt]) == 0:
change_rate_list.append(None)
continue
change_rate = \
(df[change_rate_start_dt:change_rate_end_dt]['close'] - df.loc[edt]['close']).mean() / \
df.loc[edt]['close']
change_rate_list.append(change_rate)
if code % 10 == 0:
df_meta = pd.DataFrame({
's3_filepath': s3_filepath_list,
'code': code_list,
'start_dt': start_dt_str_list,
'end_dt': end_dt_str_list,
'change_rate_30d': change_rate_list,
})
df_meta.to_csv(
METADATA_LOCAL_FILEPATH,
index=None
)
Logger.i(TAG, f'len(df_meta) : {len(df_meta)}')
df_meta = pd.DataFrame({
's3_filepath': s3_filepath_list,
'code': code_list,
'start_dt': start_dt_str_list,
'end_dt': end_dt_str_list,
'change_rate_30d': change_rate_list,
})
df_meta.to_csv(
METADATA_LOCAL_FILEPATH,
index=None
)
S3.save_file(
local_filepath=METADATA_LOCAL_FILEPATH,
s3_filepath=METADATA_S3_FILEPATH,
)
def main():
df_stocklist = pd.read_csv(
DataLocationConfig.STOCKLIST_FILE
)
Logger.d(TAG, df_stocklist['銘柄コード'].unique())
codes = df_stocklist['銘柄コード'].unique()
STOCKPRICE_FILEPATH_FMT = 's3://fin-app/stockprice_concat/{code}.csv'
STRIDE_DAYS = 30
WINDOW_DAYS = 30*4
STRIDE_D_TD = timedelta(days=STRIDE_DAYS)
WINDOW_D_TD = timedelta(days=WINDOW_DAYS)
WIDTH = 0.5
S3_CANDLECHART_FILEPATH_FMT = os.path.join(
DataLocationConfig.STOCKPRICE_CANDLECHART_BASEDIR.replace('s3://fin-app/', ''),
f'DAILY_WINDOW-{WINDOW_DAYS}d_STRIDE-{STRIDE_DAYS}d_WIDTH-{WIDTH}',
'{code}',
'{start_dt}_{end_dt}.png'
)
LOCAL_CANDLECHART_FILEPATH_FMT = os.path.join(
'/tmp',
f'WINDOW-{WINDOW_DAYS}d_STRIDE-{STRIDE_DAYS}d',
'{code}',
'{start_dt}_{end_dt}.png'
)
for code in codes[2:]:
# if code < 1515:
# continue
try:
df = pd.read_csv(
STOCKPRICE_FILEPATH_FMT.format(code=code)
)
except Exception as e:
Logger.e(TAG, f'failed to load csv file from s3 : {e}')
continue
df['日付'] = pd.to_datetime(df['日付'])
df = df.set_index('日付')
df = df.rename(columns={
'始値': 'open',
'高値': 'high',
'安値': 'low',
'終値': 'close'
})
MIN_DT = df.index.min()
MAX_DT = df.index.max()
start_dt = MIN_DT
end_dt = MIN_DT + WINDOW_D_TD
try:
while end_dt <= MAX_DT:
start_dt_str = start_dt.strftime('%Y-%m-%d')
end_dt_str = end_dt.strftime('%Y-%m-%d')
df_sliced = df[start_dt_str:end_dt_str]
s3_filepath = S3_CANDLECHART_FILEPATH_FMT.format(
code=code,
start_dt=start_dt_str,
end_dt=end_dt_str,
)
local_filepath = LOCAL_CANDLECHART_FILEPATH_FMT.format(
code=code,
start_dt=start_dt_str,
end_dt=end_dt_str
)
if not os.path.exists(os.path.dirname(local_filepath)):
os.makedirs(os.path.dirname(local_filepath))
local_filepath = creaet_candle_chart(
opens=df_sliced.open,
closes=df_sliced.close,
highs=df_sliced.high,
lows=df_sliced.low,
width=WIDTH,
filepath=local_filepath
)
S3.save_file(
local_filepath=local_filepath,
s3_filepath=s3_filepath,
)
Logger.i(TAG, f'Saved candle chart image to {s3_filepath}')
os.remove(local_filepath)
start_dt += STRIDE_D_TD
end_dt += STRIDE_D_TD
except Exception as e:
Logger.e(TAG, f'{e}')
continue