def update_marketdata_from_crawler():
m_type_dict = scdw.get_type_dict('A', is_name_index=True)
def file_processing(csv_file_name):
trading_df = pd.read_csv(os.path.join(CRAWLING_TARGET_PATH, csv_file_name),
delimiter=',', encoding='CP949', names=COLUMN_NAMES,
skiprows=[0])
trading_df = trading_df.fillna(0)
trading_df['date'] = parse(str(re.findall('\d{8}', csv_file_name)[0])).date()
trading_df['m_type'] = trading_df['m_type']\
.apply(lambda m_type: m_type_dict[str(m_type)])
trading_df = trading_df.drop(['m_dept'], axis=1)
smdw.insert(trading_df)
shutil.move(os.path.join(CRAWLING_TARGET_PATH, csv_file_name),
os.path.join(CRAWLING_BACKUP_PATH, csv_file_name))
se = StartEndLogging()
try:
for file_name in tqdm(sorted(os.listdir(CRAWLING_TARGET_PATH))):
file_processing(file_name)
se.mid(file_name)
except Exception as e:
log.error(e)
sys.exit()
se.end()
def lstm_test():
se = StartEndLogging()
modeling_target_qs = scw.gets_modeling_target()
log.info(len(modeling_target_qs))
cnt_skip_trend, cnt_skip_accuracy = 0, 0
for modeling_company in modeling_target_qs[:15]:
model = LstmTraining(modeling_company.com_code, kwargs)
is_skip = model.modeling()
se.mid(f'{modeling_company.com_code}')
if is_skip['trend']:
cnt_skip_trend += 1
if is_skip['accuracy']:
cnt_skip_accuracy += 1
log.info(
f'modeling total count: {len(modeling_target_qs)}, '
f'trend skip: {cnt_skip_trend}, accuracy skip: {cnt_skip_accuracy}')
se.end()
def today_modeling():
se = StartEndLogging()
modeling_target_qs = scw.gets_modeling_target()
modeling_size = len(modeling_target_qs)
cnt_processing = 0
cnt_skip_trend, cnt_skip_accuracy = 0, 0
for modeling_company in modeling_target_qs:
model = LstmTraining(modeling_company.com_code, LSTM_KWARGS)
is_skip = model.modeling2()
cnt_processing += 1
se.mid(f'{modeling_company.com_code}, {cnt_processing}/{modeling_size}')
if is_skip['trend']:
cnt_skip_trend += 1
if is_skip['accuracy']:
cnt_skip_accuracy += 1
log.info(f'modeling total count: {len(modeling_target_qs)}, '
f'trend skip: {cnt_skip_trend}, accuracy skip: {cnt_skip_accuracy}')
se.end()