def get_normal_marketdata(com_code): company_market_qs = smdw.gets(com_code=com_code) first_data = company_market_qs.first() yesterday_data, first_normal_data = first_data, first_data if first_data.t_volume != 0: diff_ratio = company_market_qs.last( ).t_volume / first_data.t_volume else: # 첫번째 t_volume값이 0이기 때문에 전체 변동량 체크가 불가능함. # 따라서 세부 체크가 수행될 수 있도록 diff_ratio를 설정함 diff_ratio = TOTAL_CHECK_MAX_RATIO + 1. if diff_ratio > TOTAL_CHECK_MAX_RATIO or diff_ratio < TOTAL_CHECK_MIN_RATIO: for market_data in company_market_qs[ 1:]: # 첫번째 값은 위에서 first()을 이용해 이미 사용 if yesterday_data.t_volume != 0 and market_data.t_volume != 0: diff_ratio = market_data.t_volume / yesterday_data.t_volume if diff_ratio > DAY_CHECK_MAX_RATIO or diff_ratio < DAY_CHECK_MIN_RATIO: first_normal_data = market_data # 당일 거래량이 없는 종목은 감자/증자 대상으로 간주한다. if market_data.volume == 0: first_normal_data = market_data yesterday_data = market_data normal_qs = company_market_qs.filter(date__gte=first_normal_data.date) log.debug(f'com_code: {com_code}, modeling data size: ' f'total({len(company_market_qs)}), normal({len(normal_qs)})') return normal_qs
def make_object(data, update_data=None): company_dt = {} try: source = dc.other_to_dict(data) company_dt['com_code'] = source['com_code'] company_dt['data_size'] = source.get('data_size', 0) if update_data is None: company_dt['id'] = None else: # 갱신용 데이터가 있는 경우에는 기존 ID를 저장하고, # 갱신용 데이터를 소스 데이터로 치환해 소스코드를 간략화 함. company_dt['id'] = source['id'] source = dc.other_to_dict(update_data) company_dt['com_name'] = source['com_name'] company_dt['m_type'] = source['m_type'] company_dt['t_volume'] = source['t_volume'] new_object = Company(**company_dt) except Exception as e: log.error(e) log.error(f'error data: data={data}, update_data={update_data}, ' f'company_df={company_dt}') new_object = None return new_object
def update_marketdata_from_crawler(): m_type_dict = scdw.get_type_dict('A', is_name_index=True) def file_processing(csv_file_name): trading_df = pd.read_csv(os.path.join(CRAWLING_TARGET_PATH, csv_file_name), delimiter=',', encoding='CP949', names=COLUMN_NAMES, skiprows=[0]) trading_df = trading_df.fillna(0) trading_df['date'] = parse(str(re.findall('\d{8}', csv_file_name)[0])).date() trading_df['m_type'] = trading_df['m_type']\ .apply(lambda m_type: m_type_dict[str(m_type)]) trading_df = trading_df.drop(['m_dept'], axis=1) smdw.insert(trading_df) shutil.move(os.path.join(CRAWLING_TARGET_PATH, csv_file_name), os.path.join(CRAWLING_BACKUP_PATH, csv_file_name)) se = StartEndLogging() try: for file_name in tqdm(sorted(os.listdir(CRAWLING_TARGET_PATH))): file_processing(file_name) se.mid(file_name) except Exception as e: log.error(e) sys.exit() se.end()
def gets(*args, **kwargs): try: code_qs = cdw.gets(*args, **kwargs) for code in code_qs: log.debug(code) except Exception as e: log.error(e) sys.exit()
def get_datas_test(): try: code_qs = sbw.gets(Code, '-code', id=100) except Exception as e: log.error(e) else: for code in code_qs: log.debug(code)
def get_and_update(): data_qs = saw.get_datas() # 1개, dict, Account # n개, QuerySet, list, DataFrame log.info(f'get account name: {data_qs[0].acc_name}') log.debug(f'get account data: {data_qs}') data_qs.base_money = 3000000 saw.insert(data_qs)
def save_company(): company = { 'id': 1, 'com_code': '1aaaaaa', 'com_name': '1bbbbbbb', 'm_type': 'ccc', 't_volume': 1., 'data_size': 11 } log.info(company) Company(**company).save()
def code_init(): se = StartEndLogging() try: cdw.delete() code_df = pd.read_csv(CODE_FILE_PATH, delimiter=',', encoding='utf-8') code_df = code_df.fillna('') cdw.insert(code_df) except Exception as e: log.error(e) sys.exit() se.end(f'{len(code_df)} codes insert!')
def insert_err(): code_dt = { 'err': 'error', 'c_type': 'B', 'code': 'B003', 'name': 'Next Close vs Next Open', 'memo': 'Next day predict close price vs next day predict open price' } try: cdw.insert(code_dt) except Exception as e: log.error(e) sys.exit()
def insert_company_from_market(): se = StartEndLogging() scw.delete() market_qs = smdw.gets(date=smdw.get_date(is_add_one=False)) log.debug(f'market data size: {len(market_qs)}') company_objects = [] for market_data in market_qs: company_object = scw.make_object(market_data) if company_object is not None: company_objects.append(company_object) scw.insert(company_objects) se.end()
def update_modelingdata_from_market(): """ daily 작업에서는 추가되는 회사 데이터의 건수에 대한 카운팅은 무시하고 업데이트 되지 않은 마켓 데이터를 일괄로 추가한다. - 이 작업은 시스템 초기화나 주간 작업에서만 수행한다. :return: 없음 """ se = StartEndLogging() try: data_df = read_frame(smdw.gets('date', date__gt=smlw.get_date())) smlw.insert(data_df[['date','com_code']+MODELING_COLUMNS]) except Exception as e: log.error(e) sys.exit() se.end()
def insert_modelingdata_from_market(): se = StartEndLogging() company_qs = scw.gets('id') company_size = len(company_qs) log.debug(f'company size: {company_size}') smlw.delete() def get_normal_marketdata(com_code): company_market_qs = smdw.gets(com_code=com_code) first_data = company_market_qs.first() yesterday_data, first_normal_data = first_data, first_data if first_data.t_volume != 0: diff_ratio = company_market_qs.last( ).t_volume / first_data.t_volume else: # 첫번째 t_volume값이 0이기 때문에 전체 변동량 체크가 불가능함. # 따라서 세부 체크가 수행될 수 있도록 diff_ratio를 설정함 diff_ratio = TOTAL_CHECK_MAX_RATIO + 1. if diff_ratio > TOTAL_CHECK_MAX_RATIO or diff_ratio < TOTAL_CHECK_MIN_RATIO: for market_data in company_market_qs[ 1:]: # 첫번째 값은 위에서 first()을 이용해 이미 사용 if yesterday_data.t_volume != 0 and market_data.t_volume != 0: diff_ratio = market_data.t_volume / yesterday_data.t_volume if diff_ratio > DAY_CHECK_MAX_RATIO or diff_ratio < DAY_CHECK_MIN_RATIO: first_normal_data = market_data # 당일 거래량이 없는 종목은 감자/증자 대상으로 간주한다. if market_data.volume == 0: first_normal_data = market_data yesterday_data = market_data normal_qs = company_market_qs.filter(date__gte=first_normal_data.date) log.debug(f'com_code: {com_code}, modeling data size: ' f'total({len(company_market_qs)}), normal({len(normal_qs)})') return normal_qs for company in tqdm(company_qs): market_df = read_frame(get_normal_marketdata(company.com_code)) company.data_size = len(market_df) company.save() market_df = market_df[['date', 'com_code'] + MODELING_COLUMNS] smlw.insert(market_df) se.end()
def predict(): try: model_info_qs = smw.get_models('LSTM', '2021-04-28') for model_info in model_info_qs[:1]: data_lst = smlw.get_prediction_datas( model_info.com_code, model_info.date, model_info.info['window_size']) if data_lst is None: continue log.warning(f'data_lst type: {type(data_lst)}\n{data_lst}') pred_data_lst = data_lst / model_info.max_value model = lstm(model_info.info) model.load_weights(model_info.model_path) pred = model.predict(pred_data_lst) log.warning(pred * model_info.max_value) except Exception as e: raise Exception(e)
def lstm_test(): se = StartEndLogging() modeling_target_qs = scw.gets_modeling_target() log.info(len(modeling_target_qs)) cnt_skip_trend, cnt_skip_accuracy = 0, 0 for modeling_company in modeling_target_qs[:15]: model = LstmTraining(modeling_company.com_code, kwargs) is_skip = model.modeling() se.mid(f'{modeling_company.com_code}') if is_skip['trend']: cnt_skip_trend += 1 if is_skip['accuracy']: cnt_skip_accuracy += 1 log.info( f'modeling total count: {len(modeling_target_qs)}, ' f'trend skip: {cnt_skip_trend}, accuracy skip: {cnt_skip_accuracy}') se.end()
def update_company_from_market(): se = StartEndLogging() market_qs = smdw.gets(date=smdw.get_date(is_min=False, is_add_one=False)) company_df = read_frame(scw.gets('id')) log.debug(f'market data size: {len(market_qs)}, company data size: {len(company_df)}') company_objects = [] for market_data in market_qs: company = company_df[company_df['com_code']==market_data.com_code] if company.empty: company_object = scw.make_object(market_data) else: company_object = scw.make_object(company, market_data) if company_object is not None: company_objects.append(company_object) scw.insert(company_objects) se.end()
def modeling3(self): try: train, test = self.preprocessing() model = self.training([train['x'], train['yc']], [test['x'], test['yc']]) model.load_weights(self.etc_info['model_file_name']) pred_yc = model.predict(test['x']) log.debug(self.kwargs['modeling_info']) model_info = { 'model_name': self.kwargs['model_name'], 'com_code': self.model_info['com_code'], 'date': self.model_info['date'], 'info': self.kwargs['modeling_info'], 'model_path': self.etc_info['model_file_name'], 'max_value': self.etc_info['max_price'], 'accuracy': ml.accuracy_trend(test['yc'], pred_yc) } smw.insert(model_info) except Exception as e: raise Exception(e)
def today_modeling(): se = StartEndLogging() modeling_target_qs = scw.gets_modeling_target() modeling_size = len(modeling_target_qs) cnt_processing = 0 cnt_skip_trend, cnt_skip_accuracy = 0, 0 for modeling_company in modeling_target_qs: model = LstmTraining(modeling_company.com_code, LSTM_KWARGS) is_skip = model.modeling2() cnt_processing += 1 se.mid(f'{modeling_company.com_code}, {cnt_processing}/{modeling_size}') if is_skip['trend']: cnt_skip_trend += 1 if is_skip['accuracy']: cnt_skip_accuracy += 1 log.info(f'modeling total count: {len(modeling_target_qs)}, ' f'trend skip: {cnt_skip_trend}, accuracy skip: {cnt_skip_accuracy}') se.end()
def start_crawler(self, is_disp_processing_step=False): se_check = StartEndLogging() about_period_months = int((self._e_date - self._s_date).days / 30) + 2 log.info(f'Input days: {self._s_date} ~ {self._e_date} ' f'({(self._e_date - self._s_date).days + 1} days)') if is_disp_processing_step: lists = tqdm(range(about_period_months)) else: lists = range(about_period_months) try: for _ in lists: cnt_work_days, cnt_skip_days = self._go_end_trading_day(not self._is_start) \ if self._is_start else \ self._change_calendar_month() if cnt_work_days == 0: break prev_work_date = None for day_idx in reversed(range(cnt_work_days)): curr_work_day = self._get_day_data(day_idx) self._cnt_work_days += 1 if self._we_date is None: self._we_date = parse(curr_work_day).date() self._ws_date = parse(curr_work_day).date() if self._ws_date == self._s_date: self._is_stop = True break if self._ws_date < self._s_date: self._is_stop = True os.remove( os.path.join(CRAWLING_TARGET_PATH, f'{curr_work_day}.csv')) self._ws_date = prev_work_date self._cnt_work_days -= 1 break prev_work_date = self._ws_date if str(self._ws_date)[:7] == str(self._s_date)[:7]: self._is_stop = True if self._is_stop: break except Exception as e: log.error(e) log.info( f'Working days: ' f'{self._ws_date} ~ {self._we_date} ({self._cnt_work_days} days)') self._web_driver.quit() se_check.end()
def get_date(): log.debug(f'max date: {smdw.get_date()}') log.debug(f'max data + 1days: {smdw.get_date(is_add_one=True)}') log.debug(f'min date: {smdw.get_date(is_min=True)}')
def tensorflow_init(): gpu = tf.config.experimental.list_physical_devices('GPU') try: tf.config.experimental.set_memory_growth(gpu[0], True) except RuntimeError as e: log.error(e)
def get(): code = cdw.get(code='B002') log.debug(code) code = cdw.get(code='B002', is_name=True) log.debug(code)
import os import sys project_path = os.path.abspath(__file__+'/../../..') if project_path not in sys.path: sys.path.append(project_path) os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') import django django.setup() from config.sysfiles.parameter import * from trading.utils import Logger as log import time log.debug(f'*****************************') log.info(f'*****************************') log.warning(f'*****************************') log.error(f'*****************************') log.critical(f'*****************************')
def save_from_marketdata(): market_qs = scw.save_from_marketdata() log.debug(f'market_qs size: {len(market_qs)}')
def get_data(): company = scw.get('060310') log.debug(company) company_qs = scw.gets('id') for com in company_qs[:2]: log.debug(com)
buy_money = y_trading.buy_price * y_trading.volume diff_money = base_money_for_cnt - buy_money sell_money = y_trading.sell_price * y_trading.volume result_money = diff_money + sell_money total_money += result_money y_trading.save() account.balance = total_money - TRADING_BASE_MONEY account.ratio = total_money / TRADING_BASE_MONEY account.base_money = total_money account.save() except Exception as e: raise Exception(e) if __name__ == '__main__': tse = StartEndLogging('daily processing') try: start_krx_crawling() update_marketdata_from_crawler() update_company_from_market() update_modelingdata_from_market() yesterday_trading_result() # today_modeling() # today_trading() except Exception as err: log.error(err) tse.end('daily processing')
def insert_and_update(): adt1 = { 'acc_name' : 'ccccc', 't_type' : 'aaaa', 't_count' : 3, 'base_money': 2000000 } adt2 = { 'acc_name' : 'ddddd', 't_type' : 'aaaa', 't_count' : 4, 'base_money': 3000000 } adt3 = { 'id' : [5], 'acc_name' : ['ccccc'], 't_type' : ['aaaa'], 't_count' : ['5'], 'base_money': [4000000] } # saw.insert(adt1) # log.debug('\n\n') # saw.insert(pd.Series(adt2)) # log.debug('\n\n') # saw.insert(pd.DataFrame.from_dict(adt3)) # log.debug('\n\n') # acc1 = saw.get_datas(acc_name='bbbbb') # acc1.base_money = 4000000 # saw.insert(acc1) # log.debug('\n\n') # QuerySet은 직접 수정이 안되기 때문에 insert에서 제외 # acc2 = saw.get_datas(t_type='aaaa') # acc2[0].balance = 10 # 직접 수정 안됨 # saw.insert(acc2) # log.debug('\n\n') # acc3 = saw.get_datas(t_type='aaaa') # acc_list = [] # for acc in acc3: # acc.balance = 20. # acc_list.append(acc) # saw.insert(acc_list) # log.debug('\n\n') # acc4 = saw.get_datas(t_type='aaaa') # acc_list1 = [] # for acc in acc4: # acc.balance = 10. # tmp_acc = dc.other_to_dict(acc) # log.info(tmp_acc) # acc_list1.append(tmp_acc) # saw.insert(acc_list1) # log.debug('\n\n') acc3 = saw.get_datas(t_type='aaaa') acc_df = read_frame(acc3) saw.insert(acc_df) log.debug('\n\n')
def get_test(): try: code = sbw.get(Code, c_type='A', code='A005') log.debug(code) except Exception as e: log.error(e)