def append_one_stock(service, code, dtype, listing_date):
'''
:param worker:
:param code:
:param dtype:
:param last_fetchdate:
:param listing_date:
:return:
'''
global is_closing
todayStr = DateUtil.getTodayStr()
last_fetchdate = DateUtil.string_toDate('2018-08-02')
start = None
ld, tindex = service.storeservice.find_lastdate_and_tindex(code, dtype)
lastdate = last_fetchdate if ld == None else ld
if lastdate is not None and lastdate.date() > listing_date:
start = DateUtil.getDatetimeFutureStr(lastdate.date(), 1)
else:
start = DateUtil.date_toString(last_fetchdate)
end = todayStr
gen = DateUtil.getNextHalfYear(DateUtil.string_toDate(start),
DateUtil.string_toDate(end),
ndays=360)
while True:
try:
end = next(gen)
if is_closing is True:
break
b2 = time.time()
days = DateUtil.diff(start, end)
lastest_date = service.get_hisft_klinetory_data(
code[3:], start, days)
if lastest_date is not None:
service.storeservice.update_lastdate(
code, dtype,
DateUtil.string_toDatetime(
DateUtil.datetime_toString(lastest_date)))
logging.info('********I')
e2 = time.time()
logging.info(
"fetching {} dtype {} listing_date: {} start: {} end:{} cost time {}"
.format(code, dtype, listing_date, start, end, e2 - b2))
start = DateUtil.getDatetimeFutureStr(DateUtil.string_toDate(end),
1)
except StopIteration as e:
print(e)
break
def append_one_stock(worker, code, dtype, ktype, listing_date):
'''
:param worker:
:param code:
:param dtype:
:param last_fetchdate:
:param listing_date:
:return:
'''
global is_closing
todayStr = DateUtil.getTodayStr()
# last_fetchdate = DateUtil.string_toDate( DateUtil.getDatetimePastStr( DateUtil.string_toDate(todayStr),30) )
last_fetchdate = DateUtil.string_toDate('2018-08-02')
start = None
ld, tindex = worker.storeservice.find_lastdate_and_tindex(code, dtype)
lastdate = last_fetchdate if ld == None else ld
if lastdate is not None and lastdate.date() > listing_date:
start = DateUtil.getDatetimeFutureStr(lastdate.date(), 1)
else:
# if listing_date.year == 1970:
# listing_date = listing_date.replace(year=1997)
# start = DateUtil.date_toString(listing_date)
start = DateUtil.date_toString(last_fetchdate)
end = todayStr
gen = DateUtil.getNextHalfYear(DateUtil.string_toDate(start),
DateUtil.string_toDate(end))
while True:
try:
end = next(gen)
if is_closing is True:
break
b2 = time.time()
_, _, lastest_date = worker.get_history_kline(
code, tindex, start, end, ktype)
if lastest_date is not None:
worker.storeservice.update_lastdate(
code, dtype, DateUtil.string_toDatetime(lastest_date))
e2 = time.time()
logging.info(
"fetching {} dtype {} ktype {} listing_date: {} start: {} end:{} cost time {}"
.format(code, dtype, ktype, listing_date, start, end, e2 - b2))
start = DateUtil.getDatetimeFutureStr(DateUtil.string_toDate(end),
1)
except StopIteration as e:
print(e)
break
def job_once_global_m5_append(*_args):
'''
线程工作：低频数据接口
:return:
'''
global is_closing
worker = _args[0][0]
arr = _args[0][1]
while not is_closing:
begin = time.time()
ret_arr = arr
todayStr = DateUtil.getTodayStr()
#last_fetchdate = DateUtil.string_toDate( DateUtil.getDatetimePastStr( DateUtil.string_toDate(todayStr),30) )
last_fetchdate = DateUtil.string_toDate('2018-08-02')
total = len(ret_arr)
curr = 0
for code, listing_date in ret_arr:
curr += 1
logging.info("current fetching progress {}/{} code:{} ".format(curr,total,code))
if curr < 1:
continue
# KLType.K_DAY
start = None
ld = worker.storeservice.find_lastdate2(code,'hk')
lastdate = last_fetchdate if ld == None else ld
if lastdate is not None and lastdate.date() > listing_date:
start = DateUtil.getDatetimeFutureStr( lastdate.date(),1 )
else:
# if listing_date.year == 1970:
# listing_date = listing_date.replace(year=1997)
# start = DateUtil.date_toString(listing_date)
start = DateUtil.date_toString(last_fetchdate)
end = todayStr
gen = DateUtil.getNextHalfYear(DateUtil.string_toDate(start), DateUtil.string_toDate(end))
b = time.time()
while True:
try:
end = next(gen)
if is_closing is True:
break
b2 = time.time()
_, _, lastest_date = worker.get_history_kline(code, start, end, ktype=KLType.K_DAY)
if lastest_date is not None:
worker.storeservice.update_lastdate(code, 'hk', DateUtil.string_toDatetime(lastest_date))
e2 = time.time()
logging.info(
"fetching {} K_DAY listing_date: {} start: {} end:{} cost time {}".format(code, listing_date, start, end, e2-b2))
start = DateUtil.getDatetimeFutureStr(DateUtil.string_toDate(end),1)
except StopIteration as e:
print(e)
break
# KLType.K_5M
start = None
ld = worker.storeservice.find_lastdate2(code,'hk_5m')
lastdate = last_fetchdate if ld == None else ld
if lastdate is not None and lastdate.date() > listing_date:
start = DateUtil.getDatetimeFutureStr(lastdate.date(), 1)
else:
# if listing_date.year == 1970:
# listing_date = listing_date.replace(year=1997)
# start = DateUtil.date_toString(listing_date)
start = DateUtil.date_toString(last_fetchdate)
end = todayStr
gen = DateUtil.getNextHalfYear(DateUtil.string_toDate(start), DateUtil.string_toDate(end))
b = time.time()
while True:
try:
end = next(gen)
if is_closing is True:
break
b1 = time.time()
_,_,lastest_date = worker.get_history_kline(code, start, end, ktype=KLType.K_5M)
if lastest_date is not None:
worker.storeservice.update_lastdate(code, 'hk_5m',lastest_date)
e1 = time.time()
logging.info(
"fetching {} K_5M_LINE listing_date:{} start: {} end:{} cost time {}".format(code,
listing_date,
start, end,
e1 - b1))
start = DateUtil.getDatetimeFutureStr(DateUtil.string_toDate(end), 1)
except StopIteration as e:
print(e)
break
e = time.time()
logging.info("position {} fetching {} const time {}".format(curr, code, e - b))
if is_closing is True:
break
end = time.time()
logging.info("fetching for one period , cost time: {}".format((end - begin)))
break
def job_once_global_m5_append_multithread(*_args):
'''
线程工作：低频数据接口
:return:
'''
global is_closing
worker = _args[0][0]
arr = _args[0][1]
while not is_closing:
begin = time.time()
ret_arr = arr
todayStr = DateUtil.getTodayStr()
#last_fetchdate = DateUtil.string_toDate(DateUtil.getDatetimePastStr(DateUtil.string_toDate(todayStr), 30))
last_fetchdate = DateUtil.string_toDate('2018-08-02')
total = len(ret_arr)
curr = 0
for code, listing_date in ret_arr:
curr += 1
#1 - (1~2998包含)
#2 - (2999~15918不含）
#3 - （15918~18986不含）
#4 - （18986~19430不含）default InnoDB,
#5 - (19430~21898不含) MyISAM engine,ft_history_kline_5
#6 - (21898~24768不含) MyISAM engine,ft_history_kline_6
#7 - (24768~26347不含） MyISAM engine, ft_history_kline_7
#8 - (26347~27096不含) MyISAM engine, ft_history_kline_8， trigged by docker upgrade
#9 - (27096~28123不含) MyISAM engine, ft_history_kline_9
#10 - (28123~31918) MyISAM engine, ft_history_kline_10
# ft_history_kline tale as the mrg_myisam
logging.info("current fetching progress {}/{} ".format(curr,total))
if curr < 1:
continue
# KLType.K_DAY
start = None
ld = worker.storeservice.find_lastdate2(code, 'hk')
lastdate = last_fetchdate if ld == None else ld
if lastdate is not None and lastdate.date() > listing_date:
start = DateUtil.getDatetimeFutureStr( lastdate.date(),1 )
else:
# if listing_date.year == 1970:
# listing_date = listing_date.replace(year=1997)
# start = DateUtil.date_toString(listing_date)
start = DateUtil.date_toString(last_fetchdate)
end = todayStr
gen = DateUtil.getNextHalfYear(DateUtil.string_toDate(start), DateUtil.string_toDate(end))
b = time.time()
while True:
try:
end = next(gen)
if is_closing is True:
break
b2 = time.time()
_, _, lastest_date = worker.get_history_kline(code, start, end, ktype=KLType.K_DAY)
if lastest_date is not None:
worker.storeservice.update_lastdate(code, 'hk', DateUtil.string_toDatetime(lastest_date))
e2 = time.time()
logging.info(
"fetching {} K_DAY listing_date: {} start: {} end:{} cost time {}".format(code, listing_date, start, end, e2-b2))
start = DateUtil.getDatetimeFutureStr(DateUtil.string_toDate(end),1)
except StopIteration as e:
print(e)
break
# KLType.K_5M
start = None
ld = worker.storeservice.find_lastdate2(code, 'hk_5m')
lastdate = last_fetchdate if ld == None else ld
if lastdate is not None and lastdate.date() > listing_date:
start = DateUtil.getDatetimeFutureStr(lastdate.date(), 1)
else:
# if listing_date.year == 1970:
# listing_date = listing_date.replace(year=1997)
# start = DateUtil.date_toString(listing_date)
start = DateUtil.date_toString(last_fetchdate)
end = todayStr
gen = DateUtil.getNextHalfYear(DateUtil.string_toDate(start), DateUtil.string_toDate(end))
b = time.time()
while True:
try:
end = next(gen)
if is_closing is True:
break
b1 = time.time()
_, _, lastest_date = worker.get_history_kline(code, start, end, ktype=KLType.K_5M)
if lastest_date is not None:
worker.storeservice.update_lastdate(code, 'hk_5m', lastest_date)
e1 = time.time()
logging.info(
"fetching {} K_5M_LINE listing_date:{} start: {} end:{} cost time {}".format(code,
listing_date,
start, end,
e1 - b1))
start = DateUtil.getDatetimeFutureStr(DateUtil.string_toDate(end), 1)
except StopIteration as e:
print(e)
break
e = time.time()
logging.info("position {} fetching {} const time {}".format(curr, code, e - b))
if is_closing is True:
break
end = time.time()
logging.info("fetching for one period , cost time: {}".format((end - begin)))
break
# -*- coding: UTF-8 -*-