def work(self, **args):
"""
tasks : ['products','day','minute', 'backup']
"""
if not self.tasks:
self.logger.info("No more work to do!")
return
elif not self.session.is_connected():
self.login(key=args['key'] if 'key' in args else None)
else:
#날짜
self.today = ezdate('today')
self.timer = time.time() #조회 제한 용 타이머
task = self.tasks.pop(0)
# 백업
if task == 'backup':
self.backup()
# 상품정보 업데이트
elif task == "products":
self.request_productsinfo()
elif task == "ohlc" or task == "density":
#open DB
db_dir = os.path.join(DATA_DIR, 'market.hdf5')
filters = tb.Filters(complib='blosc', complevel=9)
self.h5file = tb.open_file(db_dir, mode="a", filters=filters)
self.products_l = list(load_products().values())
self.codelength = len(self.products_l)
self.message = [] #중요 메시지 마지막에 보여주는 용도
# 신규 종목 db 생성
for product in self.products_l:
if not hasattr(self.h5file.root, product['symbol']):
node = self.h5file.create_group('/', product['symbol'] , product['name'])
ohlc = self.h5file.create_table(node, "OHLC", OHLC, "Daily OHLC Data")
#ohlc.cols.date.create_csindex()
self.h5file.create_table(node, "Density", Density, "Minutely Density Data")
self.h5file.flush()
if task == "ohlc":
self.logger.info("TASK : Updating Daily OHLC Data")
self.get_ohlc_data()
elif task == 'density':
self.logger.info("TASK : Updating Density Data")
self.get_density_data()
def get_ohlc_data(self):
#tr 정보
self.tr = TR.o3108()
self.query = Query(self, self.tr.CODE)
#db 정보
self.product = self.products_l.pop()
symbol = self.product['symbol']
self.cursor = getattr(self.h5file.root, symbol).OHLC #db 커서
self.lastday = ezdate(max(self.cursor.cols.date, default=self.today.delta(-3).stamp())) #최근 저장된 날짜
startday = self.lastday.delta(1) #시작일
# db에 액티브월물 저장 안되어있으면 저장하기
if not hasattr(self.cursor.attrs, 'active'):
self.cursor.attrs.active = self.product['active']['code']
active = self.cursor.attrs.active #db에 저장된 액티브 월물코드
# 액티브 월물이 변경된 경우: 선 갭보정 후 다운
if self.product['active']['code'] != active:
#digit = self.active['decimal_places'] #소숫점자릿수(반올림용)
price_gap = self.product['active']['price_gap']
self.cursor.cols.open[:] = self.cursor.cols.open[:] + price_gap
self.cursor.cols.high[:] = self.cursor.cols.high[:] + price_gap
self.cursor.cols.low[:] = self.cursor.cols.low[:] + price_gap
self.cursor.cols.close[:] = self.cursor.cols.close[:] + price_gap
self.cursor.attrs.active = self.product['active']['code'] # db에 새로운 액티브 코드 저장
self.cursor.flush()
self.message.append("!!! %s Data Has been changed up by %s from %s"%\
(self.product['name'], price_gap, self.lastday.str()))
self.fields = dict(
shcode=self.product['active']['code'],
gubun=0, #일별
qrycnt=500,
sdate=startday.str(),
edate=self.today.delta(-1).str(),
cts_date='',
)
#조회 요청
errcode = self.query.request(self.tr.INBLOCK, self.fields)
if errcode < 0:
self.parse_err_code(self.tr.CODE, errcode)
def get_density_data(self):
""" 분봉 데이터 받기 """
#tr 정보
self.tr = TR.o3103()
self.query = Query(self, self.tr.CODE)
#db 정보
self.product = self.products_l.pop()
symbol = self.product['symbol']
self.cursor = getattr(self.h5file.root, symbol).Density
self.lastdate = ezdate(max(self.cursor.cols.date, default=self.today.delta(-2).stamp())) #최근 저장된 날짜
self.flag = False #last date 매칭 되었을때 사용
if not hasattr(self.cursor.attrs, 'active'):
self.cursor.attrs.active = self.product['active']['code']
active = self.cursor.attrs.active #db에 저장된 종목 코드
# 액티브 월물이 변경된 경우: 선 갭보정 후 다운
if self.product['active']['code'] != active:
price_gap = self.product['active']['price_gap'] #가격 차이
#데이터 변환
self.cursor.cols.price[:] = self.cursor.cols.price[:] + price_gap
self.cursor.attrs.active = self.product['active']['code'] #새로운 액티브 코드 저장
self.cursor.flush()
self.message.append("!!!CASE1: %s Data Has been changed up by %s from %s"%\
(self.product['name'], price_gap, self.lastdate.str('s')))
self.fields = dict(
shcode=self.product['active']['code'],
ncnt=1, #분단위
readcnt=500,
cts_date='',
cts_time='',
)
# 로깅
self.logger.info("Started to get MINUTE data : %s upto %s",
self.product['name'], self.lastdate.str('s'))
# 조회 요청
errcode = self.query.request(self.tr.INBLOCK, self.fields)
if errcode < 0:
self.parse_err_code(self.tr.CODE, errcode)
def _on_get_density_data(self, code):
if self.tr.methodname != 'get_density_data':
return
shcode = self.query.get_field_data(self.tr.OUTBLOCK, 'shcode', 0) #종목코드
cts_date = self.query.get_field_data(self.tr.OUTBLOCK, 'cts_date', 0) #연속일자
cts_time = self.query.get_field_data(self.tr.OUTBLOCK, 'cts_time', 0) #연속시간
timediff = int(self.query.get_field_data(self.tr.OUTBLOCK, 'timediff', 0)) * (-1) #시차
cnt = self.query.get_block_count(self.tr.OUTBLOCK1)
for i in range(cnt):
date = self.query.get_field_data(self.tr.OUTBLOCK1, 'date', i) #날짜
dtime = self.query.get_field_data(self.tr.OUTBLOCK1, 'time', i) #시간
high = float(self.query.get_field_data(self.tr.OUTBLOCK1, 'high', i)) #고가
low = float(self.query.get_field_data(self.tr.OUTBLOCK1, 'low', i)) #저가
volume = int(self.query.get_field_data(self.tr.OUTBLOCK1, 'volume', i)) #거래량
items = []
#날짜가 이상할때가 있음.
try:
ndate = np.datetime64(datetime.strptime(date+dtime, '%Y%m%d%H%M%S')) \
+ np.timedelta64(timediff, 'h')
date = ezdate(ndate)
#ndate = ndate.astype('uint64')/1000000
#sdate = datetime.strptime(date+dtime, '%Y%m%d%H%M%S') + timedelta(hours=timediff)
#sdate = sdate.strftime('%Y-%m-%dT%H:%M:%S')
except:
self.logger.warning("%s has a missing DATE or something is wrong %s", shcode, date.str('s'))
self.logger.error(traceback.format_exc())
continue
#거래량이 1 미만이면 버림
if int(volume) < 1:
self.logger.info("%s with volume %s will be passed at %s", shcode, volume, sdate)
continue
#db에 저장된 최근 날짜보다 이전이면 끝냄
if np.rint(date.stamp()) <= np.rint(self.lastdate.stamp()):
self.flag = True
self.logger.warning("Last date of %s in DB matched at %s", shcode, date.str('s'))
break
# 날짜 겹치면 버림
ndate = date.stamp()
if self.cursor.read_where('date==ndate').size:
self.logger.info("duplicated date: %s", date.str('s'))
continue
else:
digit = self.product['decimals']
tickunit = float(self.product['tick_unit'])
if round(low, digit) == round(high, digit):
item = (date.stamp(), round(low, digit), volume)
items.append(item)
else:
length = (high-low)/tickunit + 1
length = np.rint(length)
value = volume/length
if np.isinf(value) or (value < 0.1): #inf value 종종 생겨서..
self.logger.warning("wrong volume: %s, length: %s at %s",
volume, length, date.str('s'))
continue
for price in np.arange(round(low, digit), high - tickunit/2, tickunit):
item = (date.stamp(), round(price, digit), value)
items.append(item)
if items:
self.cursor.append(items)
self.cursor.flush()
count = self.query.get_tr_count_request(self.tr.CODE)
if 'items' in locals() and items:
msg = "Updating Minute data: %s, TR: %s, (%s/%s)"\
%(self.product['name'], count, len(self.products_l), self.codelength)
self.logger.info(msg)
else:
msg = "Nothing to update: %s, TR: %s, (%s/%s)"\
%(self.product['name'], count, len(self.products_l), self.codelength)
self.logger.info(msg)
# 10분당 조회 tr 200회 제한
self.check_req_limit(self.tr)
if (cts_date == '00000000') or self.flag:
if 'date' in locals():
self.logger.info("Reached last date at %s", date.str('s'))
if self.products_l:
self.get_density_data()
else:
for msg in self.message:
self.logger.info(msg)
self.logger.info("** Minute Data updated completely **")
self.h5file.close()
self.flush()
self.work()
elif cts_date != '00000000':
self.fields['cts_date'] = cts_date
self.fields['cts_time'] = cts_time
errcode = self.query.request(self.tr.INBLOCK, self.fields, bnext=True)
if errcode < 0:
self.parse_err_code(self.tr.CODE, errcode)
def _on_get_ohlc_data(self, code):
if self.tr.methodname != 'get_ohlc_data':
return
data = []
shcode = self.query.get_field_data(self.tr.OUTBLOCK, 'shcode', 0) #종목코드
cts_date = self.query.get_field_data(self.tr.OUTBLOCK, 'cts_date', 0) #연속일자
cnt = self.query.get_block_count(self.tr.OUTBLOCK1)
for i in range(cnt):
date = self.query.get_field_data(self.tr.OUTBLOCK1, 'date', i) #날짜
open = self.query.get_field_data(self.tr.OUTBLOCK1, 'open', i) #시가
high = self.query.get_field_data(self.tr.OUTBLOCK1, 'high', i) #고가
low = self.query.get_field_data(self.tr.OUTBLOCK1, 'low', i) #저가
close = self.query.get_field_data(self.tr.OUTBLOCK1, 'close', i) #종가
volume = self.query.get_field_data(self.tr.OUTBLOCK1, 'volume', i) #거래량
#날짜가 이상할때가 있음.
try:
date = ezdate(date)
#ndate = np.datetime64(datetime.strptime(date, '%Y%m%d')).astype('uint64')/1000000
#sdate = datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')
except:
self.logger.warning("%s has a missing DATE or something is wrong", shcode)
self.logger.error(traceback.format_exc())
continue
#거래량이 1 미만이면 버림
if int(volume) < 1:
self.logger.info("%s with volume %s will be passed at %s", shcode, volume, date.str())
continue
if np.rint(date.stamp()) <= np.rint(self.lastday.stamp()):
self.logger.warning("Last date of %s in DB matched at %s", shcode, date.str())
continue
ndate = date.stamp()
if self.cursor.read_where('date==ndate').size:
self.logger.info("duplicated date: %s", date.str())
continue
datum = (date.stamp(), open, high, low, close, volume)
data.append(datum)
count = self.query.get_tr_count_request(self.tr.CODE)
if data:
msg = "Updating daily: %s at %s, TR: %s, (%s/%s)"\
%(self.product['name'], date.str(), count, len(self.products_l), self.codelength)
self.logger.info(msg)
self.cursor.append(data)
self.cursor.flush()
else:
msg = "Nothing to update: %s , TR: %s (%s/%s)"\
%(self.product['name'], count, len(self.products_l), self.codelength)
self.logger.info(msg)
# 10분당 조회 tr 200회 제한
self.check_req_limit(self.tr)
if cts_date != '00000000':
self.fields['cts_date'] = cts_date
errcode = self.query.request(self.tr.INBLOCK, self.fields, bnext=True)
if errcode < 0:
self.parse_err_code(self.tr.CODE, errcode)
else:
if self.products_l:
self.get_ohlc_data()
else:
for msg in self.message:
self.logger.info(msg)
self.logger.info("** Daily Data updated completely **")
self.h5file.close()
self.flush()
self.work()