def get_normal_marketdata(com_code):
company_market_qs = smdw.gets(com_code=com_code)
first_data = company_market_qs.first()
yesterday_data, first_normal_data = first_data, first_data
if first_data.t_volume != 0:
diff_ratio = company_market_qs.last(
).t_volume / first_data.t_volume
else:
# 첫번째 t_volume값이 0이기 때문에 전체 변동량 체크가 불가능함.
# 따라서 세부 체크가 수행될 수 있도록 diff_ratio를 설정함
diff_ratio = TOTAL_CHECK_MAX_RATIO + 1.
if diff_ratio > TOTAL_CHECK_MAX_RATIO or diff_ratio < TOTAL_CHECK_MIN_RATIO:
for market_data in company_market_qs[
1:]: # 첫번째 값은 위에서 first()을 이용해 이미 사용
if yesterday_data.t_volume != 0 and market_data.t_volume != 0:
diff_ratio = market_data.t_volume / yesterday_data.t_volume
if diff_ratio > DAY_CHECK_MAX_RATIO or diff_ratio < DAY_CHECK_MIN_RATIO:
first_normal_data = market_data
# 당일 거래량이 없는 종목은 감자/증자 대상으로 간주한다.
if market_data.volume == 0:
first_normal_data = market_data
yesterday_data = market_data
normal_qs = company_market_qs.filter(date__gte=first_normal_data.date)
log.debug(f'com_code: {com_code}, modeling data size: '
f'total({len(company_market_qs)}), normal({len(normal_qs)})')
return normal_qs
def make_object(data, update_data=None):
company_dt = {}
try:
source = dc.other_to_dict(data)
company_dt['com_code'] = source['com_code']
company_dt['data_size'] = source.get('data_size', 0)
if update_data is None:
company_dt['id'] = None
else:
# 갱신용 데이터가 있는 경우에는 기존 ID를 저장하고,
# 갱신용 데이터를 소스 데이터로 치환해 소스코드를 간략화 함.
company_dt['id'] = source['id']
source = dc.other_to_dict(update_data)
company_dt['com_name'] = source['com_name']
company_dt['m_type'] = source['m_type']
company_dt['t_volume'] = source['t_volume']
new_object = Company(**company_dt)
except Exception as e:
log.error(e)
log.error(f'error data: data={data}, update_data={update_data}, '
f'company_df={company_dt}')
new_object = None
return new_object
def update_marketdata_from_crawler():
m_type_dict = scdw.get_type_dict('A', is_name_index=True)
def file_processing(csv_file_name):
trading_df = pd.read_csv(os.path.join(CRAWLING_TARGET_PATH, csv_file_name),
delimiter=',', encoding='CP949', names=COLUMN_NAMES,
skiprows=[0])
trading_df = trading_df.fillna(0)
trading_df['date'] = parse(str(re.findall('\d{8}', csv_file_name)[0])).date()
trading_df['m_type'] = trading_df['m_type']\
.apply(lambda m_type: m_type_dict[str(m_type)])
trading_df = trading_df.drop(['m_dept'], axis=1)
smdw.insert(trading_df)
shutil.move(os.path.join(CRAWLING_TARGET_PATH, csv_file_name),
os.path.join(CRAWLING_BACKUP_PATH, csv_file_name))
se = StartEndLogging()
try:
for file_name in tqdm(sorted(os.listdir(CRAWLING_TARGET_PATH))):
file_processing(file_name)
se.mid(file_name)
except Exception as e:
log.error(e)
sys.exit()
se.end()
def gets(*args, **kwargs):
try:
code_qs = cdw.gets(*args, **kwargs)
for code in code_qs:
log.debug(code)
except Exception as e:
log.error(e)
sys.exit()
def get_datas_test():
try:
code_qs = sbw.gets(Code, '-code', id=100)
except Exception as e:
log.error(e)
else:
for code in code_qs:
log.debug(code)
def get_and_update():
data_qs = saw.get_datas()
# 1개, dict, Account
# n개, QuerySet, list, DataFrame
log.info(f'get account name: {data_qs[0].acc_name}')
log.debug(f'get account data: {data_qs}')
data_qs.base_money = 3000000
saw.insert(data_qs)
def save_company():
company = {
'id': 1,
'com_code': '1aaaaaa',
'com_name': '1bbbbbbb',
'm_type': 'ccc',
't_volume': 1.,
'data_size': 11
}
log.info(company)
Company(**company).save()
def code_init():
se = StartEndLogging()
try:
cdw.delete()
code_df = pd.read_csv(CODE_FILE_PATH, delimiter=',', encoding='utf-8')
code_df = code_df.fillna('')
cdw.insert(code_df)
except Exception as e:
log.error(e)
sys.exit()
se.end(f'{len(code_df)} codes insert!')
def insert_err():
code_dt = {
'err': 'error',
'c_type': 'B',
'code': 'B003',
'name': 'Next Close vs Next Open',
'memo': 'Next day predict close price vs next day predict open price'
}
try:
cdw.insert(code_dt)
except Exception as e:
log.error(e)
sys.exit()
def insert_company_from_market():
se = StartEndLogging()
scw.delete()
market_qs = smdw.gets(date=smdw.get_date(is_add_one=False))
log.debug(f'market data size: {len(market_qs)}')
company_objects = []
for market_data in market_qs:
company_object = scw.make_object(market_data)
if company_object is not None:
company_objects.append(company_object)
scw.insert(company_objects)
se.end()
def update_modelingdata_from_market():
"""
daily 작업에서는 추가되는 회사 데이터의 건수에 대한 카운팅은 무시하고
업데이트 되지 않은 마켓 데이터를 일괄로 추가한다.
- 이 작업은 시스템 초기화나 주간 작업에서만 수행한다.
:return: 없음
"""
se = StartEndLogging()
try:
data_df = read_frame(smdw.gets('date', date__gt=smlw.get_date()))
smlw.insert(data_df[['date','com_code']+MODELING_COLUMNS])
except Exception as e:
log.error(e)
sys.exit()
se.end()
def insert_modelingdata_from_market():
se = StartEndLogging()
company_qs = scw.gets('id')
company_size = len(company_qs)
log.debug(f'company size: {company_size}')
smlw.delete()
def get_normal_marketdata(com_code):
company_market_qs = smdw.gets(com_code=com_code)
first_data = company_market_qs.first()
yesterday_data, first_normal_data = first_data, first_data
if first_data.t_volume != 0:
diff_ratio = company_market_qs.last(
).t_volume / first_data.t_volume
else:
# 첫번째 t_volume값이 0이기 때문에 전체 변동량 체크가 불가능함.
# 따라서 세부 체크가 수행될 수 있도록 diff_ratio를 설정함
diff_ratio = TOTAL_CHECK_MAX_RATIO + 1.
if diff_ratio > TOTAL_CHECK_MAX_RATIO or diff_ratio < TOTAL_CHECK_MIN_RATIO:
for market_data in company_market_qs[
1:]: # 첫번째 값은 위에서 first()을 이용해 이미 사용
if yesterday_data.t_volume != 0 and market_data.t_volume != 0:
diff_ratio = market_data.t_volume / yesterday_data.t_volume
if diff_ratio > DAY_CHECK_MAX_RATIO or diff_ratio < DAY_CHECK_MIN_RATIO:
first_normal_data = market_data
# 당일 거래량이 없는 종목은 감자/증자 대상으로 간주한다.
if market_data.volume == 0:
first_normal_data = market_data
yesterday_data = market_data
normal_qs = company_market_qs.filter(date__gte=first_normal_data.date)
log.debug(f'com_code: {com_code}, modeling data size: '
f'total({len(company_market_qs)}), normal({len(normal_qs)})')
return normal_qs
for company in tqdm(company_qs):
market_df = read_frame(get_normal_marketdata(company.com_code))
company.data_size = len(market_df)
company.save()
market_df = market_df[['date', 'com_code'] + MODELING_COLUMNS]
smlw.insert(market_df)
se.end()
def predict():
try:
model_info_qs = smw.get_models('LSTM', '2021-04-28')
for model_info in model_info_qs[:1]:
data_lst = smlw.get_prediction_datas(
model_info.com_code, model_info.date,
model_info.info['window_size'])
if data_lst is None:
continue
log.warning(f'data_lst type: {type(data_lst)}\n{data_lst}')
pred_data_lst = data_lst / model_info.max_value
model = lstm(model_info.info)
model.load_weights(model_info.model_path)
pred = model.predict(pred_data_lst)
log.warning(pred * model_info.max_value)
except Exception as e:
raise Exception(e)
def lstm_test():
se = StartEndLogging()
modeling_target_qs = scw.gets_modeling_target()
log.info(len(modeling_target_qs))
cnt_skip_trend, cnt_skip_accuracy = 0, 0
for modeling_company in modeling_target_qs[:15]:
model = LstmTraining(modeling_company.com_code, kwargs)
is_skip = model.modeling()
se.mid(f'{modeling_company.com_code}')
if is_skip['trend']:
cnt_skip_trend += 1
if is_skip['accuracy']:
cnt_skip_accuracy += 1
log.info(
f'modeling total count: {len(modeling_target_qs)}, '
f'trend skip: {cnt_skip_trend}, accuracy skip: {cnt_skip_accuracy}')
se.end()
def update_company_from_market():
se = StartEndLogging()
market_qs = smdw.gets(date=smdw.get_date(is_min=False, is_add_one=False))
company_df = read_frame(scw.gets('id'))
log.debug(f'market data size: {len(market_qs)}, company data size: {len(company_df)}')
company_objects = []
for market_data in market_qs:
company = company_df[company_df['com_code']==market_data.com_code]
if company.empty:
company_object = scw.make_object(market_data)
else:
company_object = scw.make_object(company, market_data)
if company_object is not None:
company_objects.append(company_object)
scw.insert(company_objects)
se.end()
def modeling3(self):
try:
train, test = self.preprocessing()
model = self.training([train['x'], train['yc']],
[test['x'], test['yc']])
model.load_weights(self.etc_info['model_file_name'])
pred_yc = model.predict(test['x'])
log.debug(self.kwargs['modeling_info'])
model_info = {
'model_name': self.kwargs['model_name'],
'com_code': self.model_info['com_code'],
'date': self.model_info['date'],
'info': self.kwargs['modeling_info'],
'model_path': self.etc_info['model_file_name'],
'max_value': self.etc_info['max_price'],
'accuracy': ml.accuracy_trend(test['yc'], pred_yc)
}
smw.insert(model_info)
except Exception as e:
raise Exception(e)
def today_modeling():
se = StartEndLogging()
modeling_target_qs = scw.gets_modeling_target()
modeling_size = len(modeling_target_qs)
cnt_processing = 0
cnt_skip_trend, cnt_skip_accuracy = 0, 0
for modeling_company in modeling_target_qs:
model = LstmTraining(modeling_company.com_code, LSTM_KWARGS)
is_skip = model.modeling2()
cnt_processing += 1
se.mid(f'{modeling_company.com_code}, {cnt_processing}/{modeling_size}')
if is_skip['trend']:
cnt_skip_trend += 1
if is_skip['accuracy']:
cnt_skip_accuracy += 1
log.info(f'modeling total count: {len(modeling_target_qs)}, '
f'trend skip: {cnt_skip_trend}, accuracy skip: {cnt_skip_accuracy}')
se.end()
def start_crawler(self, is_disp_processing_step=False):
se_check = StartEndLogging()
about_period_months = int((self._e_date - self._s_date).days / 30) + 2
log.info(f'Input days: {self._s_date} ~ {self._e_date} '
f'({(self._e_date - self._s_date).days + 1} days)')
if is_disp_processing_step:
lists = tqdm(range(about_period_months))
else:
lists = range(about_period_months)
try:
for _ in lists:
cnt_work_days, cnt_skip_days = self._go_end_trading_day(not self._is_start) \
if self._is_start else \
self._change_calendar_month()
if cnt_work_days == 0: break
prev_work_date = None
for day_idx in reversed(range(cnt_work_days)):
curr_work_day = self._get_day_data(day_idx)
self._cnt_work_days += 1
if self._we_date is None:
self._we_date = parse(curr_work_day).date()
self._ws_date = parse(curr_work_day).date()
if self._ws_date == self._s_date:
self._is_stop = True
break
if self._ws_date < self._s_date:
self._is_stop = True
os.remove(
os.path.join(CRAWLING_TARGET_PATH,
f'{curr_work_day}.csv'))
self._ws_date = prev_work_date
self._cnt_work_days -= 1
break
prev_work_date = self._ws_date
if str(self._ws_date)[:7] == str(self._s_date)[:7]:
self._is_stop = True
if self._is_stop:
break
except Exception as e:
log.error(e)
log.info(
f'Working days: '
f'{self._ws_date} ~ {self._we_date} ({self._cnt_work_days} days)')
self._web_driver.quit()
se_check.end()
def get_date():
log.debug(f'max date: {smdw.get_date()}')
log.debug(f'max data + 1days: {smdw.get_date(is_add_one=True)}')
log.debug(f'min date: {smdw.get_date(is_min=True)}')
def tensorflow_init():
gpu = tf.config.experimental.list_physical_devices('GPU')
try:
tf.config.experimental.set_memory_growth(gpu[0], True)
except RuntimeError as e:
log.error(e)
def get():
code = cdw.get(code='B002')
log.debug(code)
code = cdw.get(code='B002', is_name=True)
log.debug(code)
import os
import sys
project_path = os.path.abspath(__file__+'/../../..')
if project_path not in sys.path:
sys.path.append(project_path)
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')
import django
django.setup()
from config.sysfiles.parameter import *
from trading.utils import Logger as log
import time
log.debug(f'*****************************')
log.info(f'*****************************')
log.warning(f'*****************************')
log.error(f'*****************************')
log.critical(f'*****************************')
def save_from_marketdata():
market_qs = scw.save_from_marketdata()
log.debug(f'market_qs size: {len(market_qs)}')
def get_data():
company = scw.get('060310')
log.debug(company)
company_qs = scw.gets('id')
for com in company_qs[:2]:
log.debug(com)
buy_money = y_trading.buy_price * y_trading.volume
diff_money = base_money_for_cnt - buy_money
sell_money = y_trading.sell_price * y_trading.volume
result_money = diff_money + sell_money
total_money += result_money
y_trading.save()
account.balance = total_money - TRADING_BASE_MONEY
account.ratio = total_money / TRADING_BASE_MONEY
account.base_money = total_money
account.save()
except Exception as e:
raise Exception(e)
if __name__ == '__main__':
tse = StartEndLogging('daily processing')
try:
start_krx_crawling()
update_marketdata_from_crawler()
update_company_from_market()
update_modelingdata_from_market()
yesterday_trading_result()
# today_modeling()
# today_trading()
except Exception as err:
log.error(err)
tse.end('daily processing')
def insert_and_update():
adt1 = {
'acc_name' : 'ccccc',
't_type' : 'aaaa',
't_count' : 3,
'base_money': 2000000
}
adt2 = {
'acc_name' : 'ddddd',
't_type' : 'aaaa',
't_count' : 4,
'base_money': 3000000
}
adt3 = {
'id' : [5],
'acc_name' : ['ccccc'],
't_type' : ['aaaa'],
't_count' : ['5'],
'base_money': [4000000]
}
# saw.insert(adt1)
# log.debug('\n\n')
# saw.insert(pd.Series(adt2))
# log.debug('\n\n')
# saw.insert(pd.DataFrame.from_dict(adt3))
# log.debug('\n\n')
# acc1 = saw.get_datas(acc_name='bbbbb')
# acc1.base_money = 4000000
# saw.insert(acc1)
# log.debug('\n\n')
# QuerySet은 직접 수정이 안되기 때문에 insert에서 제외
# acc2 = saw.get_datas(t_type='aaaa')
# acc2[0].balance = 10 # 직접 수정 안됨
# saw.insert(acc2)
# log.debug('\n\n')
# acc3 = saw.get_datas(t_type='aaaa')
# acc_list = []
# for acc in acc3:
# acc.balance = 20.
# acc_list.append(acc)
# saw.insert(acc_list)
# log.debug('\n\n')
# acc4 = saw.get_datas(t_type='aaaa')
# acc_list1 = []
# for acc in acc4:
# acc.balance = 10.
# tmp_acc = dc.other_to_dict(acc)
# log.info(tmp_acc)
# acc_list1.append(tmp_acc)
# saw.insert(acc_list1)
# log.debug('\n\n')
acc3 = saw.get_datas(t_type='aaaa')
acc_df = read_frame(acc3)
saw.insert(acc_df)
log.debug('\n\n')
def get_test():
try:
code = sbw.get(Code, c_type='A', code='A005')
log.debug(code)
except Exception as e:
log.error(e)
