def get_cols(bar, start_day, day_cnt, bar_sec, end_day='', cols=[utcc,lrc,volc,vbsc,vol1c]) : """ This will get the cols w.r.t. time for the period given. It is similar with get_weekly() but doesn't have to be in weekly and therefore return ca, i.e. two dim array indexed by bar number with each row represent cols specified. bar: a DailyBar object loaded with repo data, i.e. bar.set_repo(b) has been called start_day, end_day is in yyyymmdd string day_cnt is integer, including the start_day. if end_day is specified day_cnt is ignored b is a loaded object from npz object. Otherwise return: ca, dt0 ca: two dim array indexed by bar number with each row represent cols specified. Note, the first cols[0] has to be utcc dt0: a one dimensional array of datetime object for each bar's ending time Internally, it calls the bar's get_bar function. """ assert cols[0] == utcc, 'first column has to be utcc' if len(end_day) == 0 : # get the end day ti=l1.TradingDayIterator(start_day) ti.next_n_trade_day(day_cnt) end_day=ti.yyyymmdd() ca_arr=[] ti=l1.TradingDayIterator(start_day) day=ti.yyyymmdd() while day <= end_day : ca_arr.append(bar.get_bar(day, 1, bar_sec, cols=cols)) ti.next() day=ti.yyyymmdd() ca=np.vstack(ca_arr) dt0=[] for utc0 in ca[:, 0] : dt0.append(datetime.datetime.fromtimestamp(utc0)) return ca, dt0
def get_bar(self, start_day, day_cnt, bar_sec, end_day='', cols=[utcc,lrc]) : """ return bars for specified period, with bar period return index of multiday bar starting from start_day, running for day_cnt """ if end_day!='' : print 'end_day not null, ignoring day_cnt: ', day_cnt ti=l1.TradingDayIterator(start_day) day_cnt=0 day=ti.yyyymmdd() while day <= end_day : day_cnt+=1 ti.next() day=ti.yyyymmdd() print 'got ', day_cnt, ' days from ', start_day, ' to ', end_day ti=l1.TradingDayIterator(start_day) day=ti.yyyymmdd() dc=0 ixa=[] while dc < day_cnt : try : ixa.append(self.daily_idx(day)) except Exception as e : print str(e) print 'problem getting for the day ', day, ', continue' ixa.append((-1,0,0,day)) ti.next() day=ti.yyyymmdd() dc+=1 return self.bar_by_idx(ixa,bar_sec,cols)
def get_weekly(dbar, symbol, start_day, end_day, bar_sec, cols=[utcc,lrc,volc,vbsc,vol1c]) : """ dbar is the DailyBar object wbdict is a basic object holding weekly data as 3 dimensional array wbdict['wbar'] the 3D array of [week, bar_of_week, cols] where first 5 columns in cols has to be utcc, lrc, volc, vbsc, vol1c """ ti=l1.TradingDayIterator(start_day) while ti.dt.weekday() != 0 : ti.next() day=ti.yyyymmdd() ti2=l1.TradingDayIterator(end_day) while ti2.dt.weekday() != 4 : ti2.prev() end_day=ti2.yyyymmdd() print 'getting from ', day , ' to ', end_day wbar=[] while day <= end_day : while ti.dt.weekday() != 4 : ti.next() eday=ti.yyyymmdd() wbar.append(dbar.get_bar(day,0,bar_sec,end_day=eday,cols=cols)) while ti.dt.weekday() != 0 : ti.next() day=ti.yyyymmdd() wbar=np.array(wbar) wbdict={'wbar':wbar,'cols':cols} return wbdict
def ingest_ib(sday, eday, get_missing=True, ingest_hist=True, ingest_l1=True): """ sday is the first day to write to repo, usually a monday eday is the last day to write to repo, usually a friday get_missing if true, will try to get a missing day from IB, so it assumes the IB connectivity. ingest_hist if true, will read qt/trd files from hist/ and overwrite repo ingest_l1 if true, will read live *B1S.csv files from bar/ and update repo In normal case, this is supposed to be run on a ib machine, on a weekly basis. Note, it is not recommended to run while the normal history download is running, since there is some problem with duplicate session id that needs to be fixed. This first gets hist/ to populate the repo, then reads IB_L1 *B1S*.csv to update the trades and other bid/ask columns. """ if ingest_hist: print "ingesting history!" import IB_hist as ibh ibh.ingest_all_symb(sday, eday, get_missing=get_missing) if ingest_l1: print "ingesting l1!" import IB_L1_Bar as ibl1 # figure out the bar_dir_list as fridays during sday to eday # since the bar_dir is the friday containing previous 5 trading days tdi = l1.TradingDayIterator(sday) bar_list = [] while tdi.yyyymmdd() <= eday: if tdi.weekday() == 4: bar_list.append(tdi.yyyymmdd()) tdi.next() ibl1.ingest_all_l1(bar_list)
def getFday(cl_bar_file, fday = None) : if fday is None : fday = datetime.datetime.now().strftime('%Y%m%d') ti = l1.TradingDayIterator(fday) assert ti.weekday() == 4, 'not a friday '+ fday ti.prev() thuday = ti.yyyymmdd() cl=np.genfromtxt(cl_bar_file,delimiter=',',usecols=[0,1,2,3,4]) t0=l1.TradingDayIterator.local_ymd_to_utc(thuday,4,45) t1=l1.TradingDayIterator.local_ymd_to_utc(thuday,6,15) t2=l1.TradingDayIterator.local_ymd_to_utc(thuday,16,55) t3=l1.TradingDayIterator.local_ymd_to_utc(thuday,18,35) i0 = np.nonzero(cl[:,0]==t0)[0] i1 = np.nonzero(cl[:,0]==t1)[0] i2 = np.nonzero(cl[:,0]==t2)[0] i3 = np.nonzero(cl[:,0]==t3)[0] mid0=cl[i0,2] mid1=cl[i1,2] mid2=cl[i2,2] mid3=cl[i3,2] lr1 = np.log(mid1)-np.log(mid0) lr2 = np.log(mid3)-np.log(mid2) print lr1, lr2 xmu = [0.000498121848, 0.000272078733] xstd = [0.00485678068, 0.00245295458] fcst = (lr1-xmu[0])/xstd[0] * 0.00064095 + (lr2-xmu[1])/xstd[1] *0.00117629 + 0.00107083 print fcst
def remove_outlier_lr(dbar, sday, eday, outlier_mul=500): tdi = l1.TradingDayIterator(sday) d = tdi.yyyymmdd() while d <= eday: b, c, bs = dbar.load_day(d) if len(b) > 0 and bs == 1: lr = b[:, ci(c, lrc)] vol = b[:, ci(c, volc)] lrmax = max(np.std(lr) * outlier_mul, 0.0001) volm = np.mean(vol) ix = np.nonzero(np.abs(lr) > lrmax)[0] if len(ix) > 0: ix0 = np.nonzero(vol[ix] < volm)[0] if len(ix0) > 0: print('outlier ', len(ix0), ' ticks!') ix0 = ix[ix0] t = b[:, ci(c, utcc)] ix1 = [] for ix0_ in ix0: dt = datetime.datetime.fromtimestamp(t[ix0_]) if not l1.is_pre_market_hour(dbar.symbol, dt): ix1.append(ix0_) else: print('NOT removing 1 tick (pre_market=True: ', dbar.symbol, ', ', dt) dbar._delete_rows(b, c, ix1) # remove lpx and overwrite the day # to be reconstructed from lr if lpxc in c: b = np.delete(b, ci(c, lpxc), axis=1) c.remove(lpxc) dbar.overwrite([b], [d], [c], 1) tdi.next() d = tdi.yyyymmdd()
def get_future_trade_front(symbol_list, start_date, end_date, kdb_util='bin/get_trade', mock_run=False) : for symbol in symbol_list : bar_dir = symbol os.system(' mkdir -p ' + bar_dir) ti = l1.TradingDayIterator(start_date) day=ti.yyyymmdd() while day <= end_date : fc=l1.FC(symbol, day) # for each day, get trades for FC, FC+, FC/FC+, FC+/FC++ #fc_next, roll_day=FC_next(symbol, day) #fc_next_next, roll_day=FC_next(symbol, roll_day) for c in [fc ] : fn=bar_dir+'/'+c+'_trd_'+day+'.csv' print 'checking ', c, fn # check if the file exists and the size is small if l1.get_file_size(fn) < 10000 and l1.get_file_size(fn+'.gz') < 10000 : os.system('rm -f ' + fn + ' > /dev/null 2>&1') os.system('rm -f ' + fn + '.gz' + ' > /dev/null 2>&1') cmdline=kdb_util + ' ' + c + ' ' + day + ' > ' + fn print 'running ', cmdline if not mock_run : os.system( cmdline ) os.system( 'gzip ' + fn ) os.system( 'sleep 5' ) ti.next() day=ti.yyyymmdd()
def weekly_get_ingest(start_end_days=None, repo_path='repo_hist', rsync_dir_list=None): """ This is supposed to be run on IB machine at EoD Friday. It first gets all the history of this week, and then ingest into a hist_repo. The need for ingestion, is to correct on any missing data. After this run, the files in the hist dir is copied to data machine """ import ibbar if start_end_days is None: cdt = datetime.datetime.now() if cdt.weekday() != 4: raise ValueError('sday not set while running on non-friday!') eday = cdt.strftime('%Y%m%d') tdi = l1.TradingDayIterator(eday) sday = tdi.prev_n_trade_day(5).yyyymmdd() else: sday, eday = start_end_days print 'Got start/end day: ', sday, eday ibbar.weekly_get_hist(sday, eday) #No need to do this, unless the previous get failed. But #then it should be tried again. #ingest_all_symb(sday, eday, repo_path=repo_path) hist_path = ibbar.read_cfg('HistPath') if rsync_dir_list is not None: for rsync_dir in rsync_dir_list: if len(rsync_dir) > 0: os.system('rsync -avz ' + hist_path + '/ ' + rsync_dir)
def _get_overnight_lr(self, day, firstpx) : """ This goes to self.dbar to try to get the lastpx of day-1 and calculate lr. Note 1: Only do it if the day's contract is consistent. Note 2: The overnight lr also calculated by IB_hist. This calculation is meant to be used as a back up in case IB_hist is not available on that day. Otherwise, always use IB_hist's overnight lr. The reason is because 'firstpx' here could be delayed but is more reliable in IB_hist. Note 3: The IB_L1 ingestions is meant to be run after IB_hist ingestion. And is supposedly to run in order of days. So dbar should be populated on previous day. """ # check the contract tdi=l1.TradingDayIterator(day) tdi.prev() day_prev=tdi.yyyymmdd() if l1.is_future(self.symbol) : ct_today=l1.FC(self.symbol, day) ct_prev = l1.FC(self.symbol,day_prev) if ct_prev != ct_today : print 'IB_L1 Failed Overnight ', self.symbol, day, ' due to contract roll from ', ct_prev, ' to ', ct_today return 0 # get the lastpx b,c,bs=self.dbar.load_day(day_prev) if len(b) > 0 : lastpx=b[-1,repo.ci(c,repo.lpxc)] else : print 'IB_L1 Failed Overnight ', self.symbol, day, ' no lastpx on previous day ', day_prev, self.dbar.path return 0 # return lr return np.log(firstpx)-np.log(lastpx)
def copy_from_repo(symarr, repo_path_write='./repo', repo_path_read_arr=['./repo_cme'], bar_sec=1, sday='20170601', eday='20171231', keep_overnight='onzero'): """ simply copy days from one repo to another, with overnight lr options. """ tdi = l1.TradingDayIterator(sday) d = tdi.yyyymmdd() while d <= eday: UpdateFromRepo(symarr, [d], repo_path_write, repo_path_read_arr, bar_sec, keep_overnight=keep_overnight) tdi.next() d = tdi.yyyymmdd()
def plot_dist_weekly_by_utc(wbdict, weekday, hhmmss, param_str='', if_plot_dist=False) : """ find the ix that corresponding to hhmmss of the weekday. wbdict: returned by bar.get_weekly() weekday: integer from datetime.weekday(), mon is 0, sun is 6, etc hhmmss: string of 103000, in '%H%M%S' format """ assert weekday < 7, 'weekday has to be less than 7' utc0=wbdict['wbar'][0,0,0] dt0=datetime.datetime.fromtimestamp(utc0) ti=l1.TradingDayIterator(dt0.strftime('%Y%m%d'), adj_start=False) dc=0 while ti.dt.weekday() != weekday and dc<7: ti.next() dc+=1 assert ti.dt.weekday() == weekday, 'weekday ' + str(weekday) + ' not found!' dtstr=ti.yyyymmdd()+hhmmss utc1=int(l1.TradingDayIterator.local_dt_to_utc(datetime.datetime.strptime(dtstr,'%Y%m%d%H%M%S'))) ix=np.searchsorted(wbdict['wbar'][0,:,0].astype(int),utc1) utc2=wbdict['wbar'][0,ix,0] dt2=datetime.datetime.fromtimestamp(utc2) assert dt2.strftime('%H%M%S')==hhmmss and dt2.weekday()==weekday, hhmmss+' not found on weekday ' + str(weekday) plot_dist_weekly_by_ix(wbdict,ix,if_plot_dist=if_plot_dist,param_str=param_str) return ix
def daily_idx(self,day) : """ return a daily bar index of day starting from 18:00:00+self.bs(5 sec) of previous day, ending at 17:00:00 of current day each index has two elements indexing into self.b[i][j] """ ti=l1.TradingDayIterator(day) if ti.yyyymmdd() != day : raise ValueError(day + ' not a trading day') # starting being 18:00:05 utc0=float(l1.TradingDayIterator.local_dt_to_utc(ti.dt)) utc_st=utc0-6*3600+self.bs utc_ed=utc0+17*3600 i=np.searchsorted(self.utc0[1:], utc_st+1) i0=np.searchsorted(self.utc0[1:], utc_ed+1) assert i==i0, 'repo error, daily bar accross array ' + str(day) + ': '+str( [i,i0] ) six=np.searchsorted(self.b[i][:,utcc],utc_st) eix=np.searchsorted(self.b[i][:,utcc],utc_ed) #eix is included assert eix > six, 'nothing found from repo on ' + day assert self.b[i][six,utcc]==utc_st, 'repo bar of start not found ' + day + ': ' + str(utc_st) assert self.b[i][eix,utcc]==utc_ed, 'repo bar of end not found ' + day + ': ' + str(utc_ed) return i, six, eix, day
def gen_bar(sym_array, sday, eday, repo_cme_path='./repo_cme', cme_path='./cme', bar_sec=1, nc=False): """ getting from the ts [utc, px, signed_vol] output format bt, lr, vl, vbs, lrhl, vwap, ltt, lpx repo_cme_path: repo to store the 1S trd bars return : None update (remove first) dbar with bar_arr, days, col_arr """ if nc: assert repo_cme_path[ -2:] == 'nc', 'repo_cme_path=' + repo_cme_path + ' not ending with nc' for symbol in sym_array: try: dbar = repo.RepoDailyBar(symbol, repo_path=repo_cme_path) except: print 'repo_trd_path failed, trying to create' dbar = repo.RepoDailyBar(symbol, repo_path=repo_cme_path, create=True) start_hour, end_hour = l1.get_start_end_hour(symbol) TRADING_HOURS = end_hour - start_hour # sday has to be a trading day it = l1.TradingDayIterator(sday) tday = it.yyyymmdd() if tday != sday: raise ValueError('sday has to be a trading day! sday: ' + sday + ' trd_day: ' + tday) lastpx = 0 prev_con = '' while tday <= eday: eutc = it.local_ymd_to_utc(tday, h_ofst=end_hour) sutc = eutc - (TRADING_HOURS) * 3600 if nc: con = l1.FC_next(symbol, tday)[0] else: con = l1.FC(symbol, tday) con = symbol + con[-2:] try: bar = bar_by_file(get_fn(cme_path, symbol, tday, con)) except (KeyboardInterrupt): print 'interrupt!' return except: print 'problem getting ', symbol, tday bar = [] if len(bar) == 0: lastpx = 0 prev_con = '' else: # this is the good case, prepare for the bar # 1) get bar with start/stop, 2) contract updated 3) lastpx # need to allow for entire content being in one ta, i.e. some # days having tds==2 but all contents in one ta, due to gmt_offset # have everything, need to get to # output format bt, lr, vl, vbs, lrhl, vwap, ltt, lp if lastpx == 0 or prev_con != con: lastpx = bar[0, 1] bt = np.arange(sutc + bar_sec, eutc + bar_sec, bar_sec) tts = np.r_[sutc, bar[:, 0]] pts = np.r_[bar[0, 1], bar[:, 1]] vts = np.r_[0, bar[:, 2]] pvts = np.abs(vts) * pts pxix = np.clip(np.searchsorted(tts[1:], bt + 1e-6), 0, len(tts) - 1) lpx = pts[pxix] lr = np.log(np.r_[lastpx, lpx]) lr = lr[1:] - lr[:-1] # tricky way to get index right on volumes btdc = np.r_[0, np.cumsum(vts)[pxix]] vbs = btdc[1:] - btdc[:-1] btdc = np.r_[0, np.cumsum(np.abs(vts))[pxix]] vol = btdc[1:] - btdc[:-1] # even tickier way to get vwap/ltt right ixg = np.nonzero(vol)[0] btdc = np.r_[0, np.cumsum(pvts)[pxix]] vwap = lpx.copy() #when there is no vol vwap[ixg] = (btdc[1:] - btdc[:-1])[ixg] / vol[ixg] ltt = np.zeros(len(bt)) ltt[ixg] = tts[pxix][ixg] repo.fwd_bck_fill(ltt, v=0) # give up, ignore the lrhl for trd bars lrhl = np.zeros(len(bt)) b = np.vstack((bt, lr, vol, vbs, lrhl, vwap, ltt, lpx)).T d = tday c = repo.kdb_ib_col dbar.remove_day(d) dbar.update([b], [d], [c], bar_sec) lastpx = lpx[-1] prev_con = con it.next() tday = it.yyyymmdd()
def write_daily_bar(bar,bar_sec=5,last_close_px=None) : import pandas as pd dt=datetime.datetime.fromtimestamp(bar[0,0]) # get the initial day, last price day_start=dt.strftime('%Y%m%d') utc_s = int(l1.TradingDayIterator.local_ymd_to_utc(day_start, 18, 0, 0)) if last_close_px is None : x=np.searchsorted(bar[1:,0], float(utc_s-3600+bar_sec)) last_close_px=bar[x,2] print 'last close price set to previous close at ', datetime.datetime.fromtimestamp(bar[x,0]), ' px: ', last_close_px else : print 'last close price set to ', last_close_px day_end=datetime.datetime.fromtimestamp(bar[-1,0]).strftime('%Y%m%d') # deciding on the trading days if dt.hour > 17 : ti=l1.TradingDayIterator(day_start,adj_start=False) ti.next() trd_day_start=ti.yyyymmdd() else : trd_day_start=day_start trd_day_end=day_end print 'preparing bar from ', day_start, ' to ', day_end, ' , trading days: ', trd_day_start, trd_day_end ti=l1.TradingDayIterator(day_start, adj_start=False) day=ti.yyyymmdd() # day is the start_day barr=[] TRADING_HOURS=23 while day < day_end: ti.next() day1=ti.yyyymmdd() utc_e = int(l1.TradingDayIterator.local_ymd_to_utc(day1, 17,0,0)) # get start backwards for starting on a Sunday utc_s = utc_e - TRADING_HOURS*3600 day=datetime.datetime.fromtimestamp(utc_s).strftime('%Y%m%d') i=np.searchsorted(bar[:, 0], float(utc_s)-1e-6) j=np.searchsorted(bar[:, 0], float(utc_e)-1e-6) bar0=bar[i:j,:] # take the bars in between the first occurance of 18:00:00 (or after) and the last occurance of 17:00:00 or before N = (utc_e-utc_s)/bar_sec # but we still fill in each bar ix_utc=((bar0[:,0]-float(utc_s))/bar_sec+1e-9).astype(int) bar_utc=np.arange(utc_s+bar_sec, utc_e+bar_sec, bar_sec) # bar time will be time of close price, as if in prod print 'getting bar ', day+'-18:00', day1+'-17:00', ' , got ', j-i, 'bars' # start to construct bar if j<=i : print ' NO bars found, skipping' else : bar_arr=[] bar_arr.append(bar_utc.astype(float)) # construct the log returns for each bar, fill in zeros for gap #lpx_open=np.log(bar0[:,2]) lpx_open=np.log(np.r_[last_close_px,bar0[:-1,5]]) lpx_hi=np.log(bar0[:,3]) lpx_lo=np.log(bar0[:,4]) lpx_close=np.log(bar0[:,5]) lpx_vwap=np.log(bar0[:,6]) lr=lpx_close-lpx_open lr_hi=lpx_hi-lpx_open lr_lo=lpx_lo-lpx_open lr_vw=lpx_vwap-lpx_open # remove bars having abnormal return, i.e. circuit break for ES # with 9999 prices MaxLR=0.2 ix1=np.nonzero(np.abs(lr)>=MaxLR)[0] ix1=np.union1d(ix1,np.nonzero(np.abs(lr_hi)>=MaxLR)[0]) ix1=np.union1d(ix1,np.nonzero(np.abs(lr_lo)>=MaxLR)[0]) ix1=np.union1d(ix1,np.nonzero(np.abs(lr_vw)>=MaxLR)[0]) if len(ix1) > 0 : print 'warning: removing ', len(ix1), 'ticks exceed MaxLR (lr/lo/hi/vw) ', zip(lr[ix1],lr_hi[ix1],lr_lo[ix1],lr_vw[ix1]) lr[ix1]=0 lr_hi[ix1]=0 lr_lo[ix1]=0 lr_vw[ix1]=0 # the trade volumes for each bar, fill in zeros for gap vlm=bar0[:,7] vb=bar0[:,8] vs=np.abs(bar0[:,9]) vbs=vb-vs for v0, vn in zip([lr,lr_hi,lr_lo,lr_vw,vlm,vbs], ['lr','lr_hi','lr_lo','lr_vw','vlm','vbs']) : nix=np.nonzero(np.isnan(v0))[0] nix=np.union1d(nix, np.nonzero(np.isinf(np.abs(v0)))[0]) if len(nix) > 0 : print 'warning: removing ', len(nix), ' nan/inf ticks for ', vn v0[nix]=0 b0=np.zeros(N).astype(float) b0[ix_utc]=v0 bar_arr.append(b0.copy()) # get the last trade time, this is needs to be ltt=np.empty(N)*np.nan ltt[ix_utc]=bar0[:,1] df=pd.DataFrame(ltt) df.fillna(method='ffill',inplace=True) if not np.isfinite(ltt[0]) : ptt=0 #no previous trading detectable if i > 0 : #make some effort here ptt=bar[i-1,1] if not np.isfinite(ptt) : ptt=0 df.fillna(ptt,inplace=True) bar_arr.append(ltt) # get the last price, as a debugging tool lpx=np.empty(N)*np.nan lpx[ix_utc]=bar0[:,5] df=pd.DataFrame(lpx) df.fillna(method='ffill',inplace=True) if not np.isfinite(lpx[0]) : df.fillna(last_close_px,inplace=True) bar_arr.append(lpx) barr.append(np.array(bar_arr).T.copy()) last_close_px=lpx[-1] day=day1 return np.vstack(barr), trd_day_start, trd_day_end
def daily_bar(self, start_day, day_cnt, bar_sec, end_day=None, cols=[utcc, lrc, volc, vbsc, lpxc], group_days=5, verbose=False): """ return 3-d array of bars for specified period, with bar period, column, grouped by days (i.e.e daily, weekly, etc) start_day: the first trading day to be returned day_cnt : number of trading days to be returned, can be None to use end_day bar_sec : bar period to be returned end_day : last trading day to be returned, can be None to use day_cnt cols : columns to be returned group_days: first dimension of the 3-d array, daily: 1, weekly=5, etc """ if end_day is not None: #print ('end_day not null, got ',) ti = l1.TradingDayIterator(start_day) day_cnt = 0 day = ti.yyyymmdd() while day <= end_day: day_cnt += 1 ti.next() day = ti.yyyymmdd() else: ti = l1.TradingDayIterator(start_day) ti.next_n_trade_day(day_cnt - 1) end_day = ti.yyyymmdd() # getting the day count, removing initial and final missing days ti = l1.TradingDayIterator(start_day) day = ti.yyyymmdd() darr = [] inarr = [] while day <= end_day: darr.append(day) if self.has_day(day): inarr.append(True) else: inarr.append(False) ti.next() day = ti.yyyymmdd() ix = np.nonzero(inarr)[0] if len(ix) == 0: raise ValueError('no bars found in repo! %s to %s!' % (start_day, end_day)) start_day = darr[ix[0]] end_day = darr[ix[-1]] day_cnt = ix[-1] - ix[0] + 1 if day_cnt / group_days * group_days != day_cnt: print( '( Warning! group_days ' + str(group_days) + ' not multiple of ' + str(day_cnt) + ' adjustint...)', ) day_cnt = day_cnt / group_days * group_days start_day = darr[ix[-1] - day_cnt + 1] print("got", day_cnt, 'days from', start_day, 'to', end_day) ti = l1.TradingDayIterator(start_day) day = ti.yyyymmdd() bar = [] day_arr = [] while day <= end_day: #print ("reading ", day, ) b, c, bs = self.load_day(day) if len(b) == 0: if verbose: print(" missing, filling zeros on ", day) bar.append(self._fill_daily_bar_col(day, bar_sec, cols)) day_arr.append(day) else: bar.append(self._scale(day, b, c, bs, cols, bar_sec)) day_arr.append(day) #print (" scale bar_sec from ", bs, " to ", bar_sec) ti.next() day = ti.yyyymmdd() bar = np.vstack(bar) # process missing days if any for c in [lpxc, lttc] + col_idx(['ism1']): if c in cols: self._fill_last(bar[:, ci(cols, c)]) d1 = day_cnt / group_days bar = bar.reshape((d1, bar.shape[0] / d1, bar.shape[1])) return bar
def get_ib_future(symbol_list, start_date, end_date, barsec, ibclient=IB_CLIENT, clp='IB', mock_run=False, getqt=True, gettrd=False, cid=100, start_end_hour=[], next_contract=False, reuse_exist_file=False, verbose=False, num_threads=None, wait_thread=True): bar_path = read_cfg('HistPath') if num_threads is not None: import _strptime n = len(symbol_list) k = np.linspace(0, n, num=num_threads + 1).astype(int) pool = mp.Pool(processes=num_threads) res = [] for i0, i1 in zip(k[:-1], k[1:]): if i1 == i0: continue res.append( pool.apply_async(get_ib_future, args=(symbol_list[i0:i1], start_date, end_date, barsec, ibclient, clp, mock_run, getqt, gettrd, cid, start_end_hour, next_contract, reuse_exist_file, verbose, None, True))) cid += 1 fnarr = [] if wait_thread: for r in res: fnarr += r.get() return fnarr step_sec = barsec_dur[barsec] fnarr = [] for symbol in symbol_list: venue = ibvenue(symbol) if venue == 'FX': bar_dir = bar_path + '/FX' elif venue == 'ETF': bar_dir = bar_path + '/ETF' elif venue == 'IDX': bar_dir = bar_path + '/IDX' else: bar_dir = bar_path + '/' + symbol if next_contract: bar_dir += '/nc' os.system(' mkdir -p ' + bar_dir) if len(start_end_hour) != 2: start_hour, end_hour = l1.get_start_end_hour(symbol) else: start_hour, end_hour = start_end_hour ti = l1.TradingDayIterator(start_date) day = ti.yyyymmdd() eday = day while day <= end_date: sday = eday fc = l1fc(symbol, day) fcn = l1fc(symbol, day, next_contract=True) while day <= end_date: ti.next() day = ti.yyyymmdd() fc0 = l1fc(symbol, day) if fc != fc0: break eday = day # make sure eday is not more than end_date # if end_date was given as a weekend dates if (eday > end_date): print 'ending to ', end_date, ' adjust to ', ti0 = l1.TradingDayIterator(eday) eday = ti0.prev().yyyymmdd() print eday if next_contract: fc = fcn fn = bar_dir + '/' + ibfn(fc, barsec, sday, eday) fnarr.append(fn) fext = [] cext = [] for gt, ext, ext_str, etp in zip([getqt, gettrd], ['_qt.csv', '_trd.csv'], ['quote', 'trade'], ['0', '1']): if not gt: continue fn0 = fn + ext # reuse_exist_file try: found = 0 assert reuse_exist_file for ext0 in ['', '.gz']: try: if os.stat(fn0 + ext0).st_size > 1024: found += 1 print 'found existing file: ', fn0 + ext0, ' count = ', found except: continue assert found == 1 print 'reusing ', fn0, ' for ', ext_str except: print 'getting ', ext_str, ' FILE: ', fn0, ' (found = %d)' % ( found) fext.append(ext) cext.append(etp) if len(fext) == 0: print 'Nothing to get from %s to %s!' % (sday, eday) continue if len(fext) == 1 and fext[ 0] == '_trd.csv' and next_contract and getqt: print '!! Next Contract using existing quote only' continue if ibclient is None: # here if ibclient is None then # don't run it (save time) # the caller should except file # not found and handle it with zero bar print 'Not running ibclient (None)!' fnarr.remove(fn) continue # clean up the existing files for ext in fext: fn0 = fn + ext if not mock_run: os.system('rm -f ' + fn0 + ' > /dev/null 2>&1') os.system('rm -f ' + fn0 + '.gz' + ' > /dev/null 2>&1') if symbol in ib_sym_special: fc = symbol + fc[-2:] sym = venue + '/' + fc # get all days with the same contract, saving to the same file tic = l1.TradingDayIterator(sday) d0 = tic.yyyymmdd() try: while d0 <= eday and d0 <= end_date: # get for day d0 utc1 = tic.to_local_utc(end_hour, 0, 0) utc0 = utc1 - (end_hour - start_hour) * 3600 while utc0 < utc1: # get for quote and trade for end_time as utc utc0 += step_sec eday_str = datetime.datetime.fromtimestamp( utc0).strftime('%Y%m%d %H:%M:%S') #for ist, ext in zip (['0', '1'], ['_qt.csv','_trd.csv']): for ist, ext in zip(cext, fext): fn0 = fn + ext cmdline = ibclient + ' ' + str( cid ) + ' ' + sym + ' ' + '\"' + eday_str + '\"' + ' ' + str( barsec) + ' ' + fn0 + ' ' + ist + ' ' + clp print 'running ', cmdline if not mock_run: os.system(cmdline) time.sleep(2) #os.system( 'sleep 2' tic.next() d0 = tic.yyyymmdd() except (KeyboardInterrupt, SystemExit): print 'stop ...' return [] except: traceback.print_exc() for fn in fnarr: for ext in fext: fn0 = fn + ext if not mock_run: print 'gzip ', fn0 os.system('gzip ' + fn0) """ if upd_repo : repo_path = read_cfg('RepoPath') future_inclusion = ['back' if next_contract else 'front'] from IB_hist import ingest_all_symbol ingest_all_symbol(start_date, end_date, repo_path=repo_path, get_missing=True, sym_list=sym_list, future_inclusion=future_inclusion) """ return fnarr
def write_daily_bar(symbol, bar, bar_sec=5, is_front=True, last_close_px=None, get_missing=True): """ bar: all bars from a hist file having the format of [utc, utc_ltt, open_px, hi_px, lo_px, close_px, vwap, vol, vb, vs] These bars have the same contract. The bar is in increasing utc, but may have gaps, or other invalid values The first day of that contract bar, due to prev_close_px unknown, it is usually covered by having the previous contract day. Note there is a limitation that the start end time has to be on a whole hour i.e. cannot stop on 4:30, just make it 5, which will write some zero bars. However, it can handle 24 hour trading, i.e. start/end at 18:00, for fx venues. Note 2, the first bar of a day should be 1 bar_sec after the starting utc and the last bar of a day should be at the ending utc. if get_missing is set to true, then try to get the bar on a bad day Output: array of daily_bar for each day covered in the bar (hist file) Each daily_bar have the following format: [obs_utc, lr, trd_vol, vbs, lrhl, lrvwap, ltt, lpx] where: obs_utc is the checking time stamp lr is the log return between this checking price and last checking price i.e. the lr of the previous bar that ended at this checking time (obs_utc) (May extend in the future) Note that the Trading Hours set to 24 for ICE hours In addition, it does the following: 1. loop the close px to the first open px, 2. convert the price to lr, removing bars with maxlr more than 0.2 (CME circuit breaker) 3. replace all inf/nan values with zero 4. cacluate the ltt and lpx """ import pandas as pd dt = datetime.datetime.fromtimestamp(bar[ 0, 0]) # fromtimestamp is safe for getting local representation of utc start_hour, end_hour = l1.get_start_end_hour(symbol) TRADING_HOURS = end_hour - start_hour start_hour = start_hour % 24 # get the initial day, last price day_start = dt.strftime('%Y%m%d') utc_s = int( l1.TradingDayIterator.local_ymd_to_utc(day_start, start_hour, 0, 0)) if last_close_px is None: x = np.searchsorted(bar[1:, 0], float(utc_s) - 1e-6) # only take the last price within 5 minutes of utc_s if x + 1 >= bar.shape[0] or bar[x + 1, 0] - utc_s > 300: if x + 1 >= bar.shape[0]: print 'no bars found after the start utc of ', day_start else: print 'start up utc (%d) more than 5 minutes later than start utc (%d) on %s' % ( bar[x + 1, 0], utc_s, day_start) print 'initializing start up last_close_px deferred' else: if x == 0: #last_close_px = bar[0, 2] #print 'last close price set as the first bar open px, this should use previous contract', datetime.datetime.fromtimestamp(bar[0,0]), datetime.datetime.fromtimestamp(bar[1,0]) last_close_px = bar[0, 5] print 'lost last close price, set as the first bar close px' else: last_close_px = bar[x, 5] print 'last close price set to close px of bar ', datetime.datetime.fromtimestamp( bar[x, 0]), ' px: ', last_close_px print 'GOT last close px ', last_close_px else: print 'GIVEN last close price ', last_close_px day_end = datetime.datetime.fromtimestamp(bar[-1, 0]).strftime('%Y%m%d') # deciding on the trading days if dt.hour > end_hour or (start_hour == end_hour and dt.hour >= end_hour): # CME 17, ICE 18, # the second rule is for 24 hour trading, note start/end has to be on a whole hour ti = l1.TradingDayIterator(day_start, adj_start=False) ti.next() trd_day_start = ti.yyyymmdd() else: trd_day_start = day_start trd_day_end = day_end print 'preparing bar from ', day_start, ' to ', day_end, ' , trading days: ', trd_day_start, trd_day_end ti = l1.TradingDayIterator(trd_day_start, adj_start=False) # day maybe a sunday day1 = ti.yyyymmdd() # first trading day barr = [] trade_days = [] col_arr = [] bad_trade_days = [] while day1 <= trd_day_end: utc_e = int( l1.TradingDayIterator.local_ymd_to_utc(day1, end_hour, 0, 0)) # get start backwards for starting on a Sunday utc_s = utc_e - TRADING_HOURS * 3600 # LIMITATION: start/stop has to be on a whole hour day = datetime.datetime.fromtimestamp(utc_s).strftime('%Y%m%d') i = np.searchsorted(bar[:, 0], float(utc_s) - 1e-6) j = np.searchsorted(bar[:, 0], float(utc_e) - 1e-6) bar0 = bar[ i: j, :] # take the bars in between the first occurance of start_hour (or after) and the last occurance of end_hour or before print 'getting bar ', day + '-' + str( start_hour) + ':00', day1 + '-' + str( end_hour) + ':00', ' , got ', j - i, 'bars' N = ( utc_e - utc_s ) / bar_sec # but we still fill in each bar, so N should be fixed for a given symbol/venue pair # here N*0.90, is to account for some closing hours during half hour ib retrieval time # The problem with using histclient.exe to retrieve IB history data for ES is # set end time is 4:30pm, will retreve 3:45 to 4:15. Because 4:15-4:30pm doesn't # have data. This is only true for ES so far # another consideration is that IB Hist client usually won't be off too much, so 90% is # a good threshold for missing/bad day bar_good = True if j - i < N * 0.90: if symbol in ['LE', 'HE'] or l1.venue_by_symbol(symbol) == 'IDX': bar_good = (j - i) > N * 0.75 elif not is_front: bar_good = (j - i) > N * 0.5 else: bar_good = False if not bar_good: print 'fewer bars for trading day %s: %d < %d * 0.9' % (day1, j - i, N) if day1 not in l1.bad_days and get_missing: # recurse with the current last price and get the updated last price print 'getting missing day %s' % (day1) from ibbar import get_missing_day fn = get_missing_day(symbol, [day1], bar_sec=bar_sec, is_front=is_front, reuse_exist_file=True) try: _, _, b0 = bar_by_file_ib(fn[0], symbol, start_day=day1, end_day=day1) except Exception as e: print e b0 = [] if len(b0) > j - i: print 'Getting more bars %d > %d on %s for %s, take it!' % ( len(b0), j - i, day1, symbol) barr0, trade_days0, col_arr0, bad_trade_days0, last_close_px0 = write_daily_bar( symbol, b0, bar_sec=bar_sec, is_front=is_front, last_close_px=last_close_px, get_missing=False) # taken as done barr += barr0 trade_days += trade_days0 col_arr += col_arr0 bad_trade_days += bad_trade_days0 last_close_px = last_close_px0 ti.next() day1 = ti.yyyymmdd() continue print 'Got %d bars on %s, had %d bars (%s), use previous!' % ( len(b0), day1, j - i, symbol) if len(bar0) < 1: print 'Bad Day! Too fewer bars in trading day %s: %d, should have %d ' % ( day1, j - i, N) bad_trade_days.append(day1) else: ix_utc = ((bar0[:, 0] - float(utc_s)) / bar_sec + 1e-9).astype( int) # lr(close_px-open_px) of a bar0 has bar_utc bar_utc = np.arange( utc_s + bar_sec, utc_e + bar_sec, bar_sec) # bar time will be time of close price, as if in prod if N != j - i: print 'fill missing for only ', j - i, ' bars (should be ', N, ')' bar1 = np.empty((N, bar0.shape[1])) bar1[:, 0] = np.arange(utc_s, utc_e, bar_sec) # filling all missing for [utc, utc_ltt, open_px, hi_px, lo_px, close_px, vwap, vol, vb, vs] # fillforward for utc_ltt, close_px, vwap for col in [1, 5, 6]: bar1[:, col] = np.nan bar1[ix_utc, col] = bar0[:, col] df = pd.DataFrame(bar1[:, col]) df.fillna(method='ffill', inplace=True) df.fillna(method='bfill', inplace=True) # fill zero for vol, vb, bs for col in [7, 8, 9]: bar1[:, col] = 0 bar1[ix_utc, col] = bar0[:, col] # copy value of close_px for open_px, hi_px, lo_px for col in [2, 3, 4]: bar1[:, col] = bar1[:, 5] bar1[ix_utc, col] = bar0[:, col] bar_arr = [] bar_arr.append(bar_utc.astype(float)) # construct the log returns for each bar, fill in zeros for gap #lpx_open=np.log(bar0[:,2]) if last_close_px is None: print 'setting last_close_px to ', bar0[0, 2] last_close_px = bar0[0, 2] lpx_open = np.log(np.r_[last_close_px, bar0[:-1, 5]]) lpx_hi = np.log(bar0[:, 3]) lpx_lo = np.log(bar0[:, 4]) lpx_close = np.log(bar0[:, 5]) lpx_vwap = np.log(bar0[:, 6]) lr = lpx_close - lpx_open lr_hi = lpx_hi - lpx_open lr_lo = lpx_lo - lpx_open lr_vw = lpx_vwap - lpx_open # remove bars having abnormal return, i.e. circuit break for ES # with 9999 prices MaxLR = 0.5 if l1.is_holiday(day) or l1.is_fx_future( symbol) or l1.venue_by_symbol(symbol) == 'FX': MaxLR = 5 ix1 = np.nonzero(np.abs(lr) >= MaxLR)[0] ix1 = np.union1d(ix1, np.nonzero(np.abs(lr_hi) >= MaxLR)[0]) ix1 = np.union1d(ix1, np.nonzero(np.abs(lr_lo) >= MaxLR)[0]) ix1 = np.union1d(ix1, np.nonzero(np.abs(lr_vw) >= MaxLR)[0]) if len(ix1) > 0: print 'MaxLR (', MaxLR, ') exceeded: ', len(ix1), ' ticks!' # removing one-by-one for ix1_ in ix1: dt = datetime.datetime.fromtimestamp(bar_utc[ix1_]) if not l1.is_pre_market_hour(symbol, dt): print 'warning: removing 1 tick lr/lo/hi/vw: ', lr[ ix1_], lr_hi[ix1_], lr_lo[ix1_], lr_vw[ix1_] lr[ix1_] = 0 lr_hi[ix1_] = 0 lr_lo[ix1_] = 0 lr_vw[ix1_] = 0 else: print 'NOT removing 1 tick (pre_market=True: ', symbol, ', ', dt, ') lr/lo/hi/vw: ', lr[ ix1_], lr_hi[ix1_], lr_lo[ix1_], lr_vw[ix1_] # the trade volumes for each bar, fill in zeros for gap vlm = bar0[:, 7] vb = bar0[:, 8] vs = np.abs(bar0[:, 9]) vbs = vb - vs for v0, vn in zip([lr, lr_hi, lr_lo, lr_vw, vlm, vbs], ['lr', 'lr_hi', 'lr_lo', 'lr_vw', 'vlm', 'vbs']): nix = np.nonzero(np.isnan(v0))[0] nix = np.union1d(nix, np.nonzero(np.isinf(np.abs(v0)))[0]) if len(nix) > 0: print 'warning: removing ', len( nix), ' nan/inf ticks for ', vn v0[nix] = 0 b0 = np.zeros(N).astype(float) b0[ix_utc] = v0 bar_arr.append(b0.copy()) # get the last trade time, this is needs to be ltt = np.empty(N) * np.nan ltt[ix_utc] = bar0[:, 1] df = pd.DataFrame(ltt) df.fillna(method='ffill', inplace=True) if not np.isfinite(ltt[0]): ptt = 0 #no previous trading detectable if i > 0: #make some effort here ptt = bar[i - 1, 1] if not np.isfinite(ptt): ptt = 0 df.fillna(ptt, inplace=True) bar_arr.append(ltt) # get the last price, as a debugging tool # close price lpx = np.empty(N) * np.nan lpx[ix_utc] = bar0[:, 5] df = pd.DataFrame(lpx) df.fillna(method='ffill', inplace=True) if not np.isfinite(lpx[0]): df.fillna(last_close_px, inplace=True) bar_arr.append(lpx) ba = np.array(bar_arr).T bt0 = ba[:, 0] lr0 = ba[:, 1] vl0 = ba[:, 5] vbs0 = ba[:, 6] # add a volatility measure here lrhl0 = ba[:, 2] - ba[:, 3] vwap0 = ba[:, 4] ltt0 = ba[:, 7] lpx0 = ba[:, 8] barr.append( np.vstack((bt0, lr0, vl0, vbs0, lrhl0, vwap0, ltt0, lpx0)).T) last_close_px = lpx[-1] trade_days.append(day1) col_arr.append(repo.kdb_ib_col) ti.next() day1 = ti.yyyymmdd() # filling in missing days if not included in the bad_trade_days bad_trade_days = [] good_trade_days = [] it = l1.TradingDayIterator(trd_day_start) while True: day = it.yyyymmdd() if day > trd_day_end: break if day not in trade_days: bad_trade_days.append(day) else: good_trade_days.append(day) it.next() print 'got bad trade days ', bad_trade_days return barr, good_trade_days, col_arr, bad_trade_days, last_close_px
def gen_daily_bar_ib(symbol, sday, eday, default_barsec, dbar_repo, is_front_future=True, get_missing=True, barsec_from_file=True, overwrite_dbar=False, EarliestMissingDay='19980101'): """ generate IB dily bars from sday to eday. It is intended to be used to add too the daily bar repo manually NOTE 1: bar_sec from file name is used to read/write the day. default_barsec given is taken as a default when getting missing days. When barsec_from_file is not True, the bar_sec from file name is checked against the default bar_sec given and raises on mismatch. NOTE 2: barsec_from_file being False enforces all day's barsec has to agree with default_barsec NOTE 3: The flexibility on barsec from file name is to entertain IB's rule for half year history on 1S, 1 year history on 30S bar, etc, enforced differently on asset classes. Inconsistencies on weekly operations NOTE 4: if overwrite_dbar is True, then the existing repo content on the day will be deleted before ingestion """ fn, is_fx, is_etf, is_idx = fn_from_dates(symbol, sday, eday, is_front_future) spread = get_future_spread(symbol) print 'Got ', len(fn), ' files: ', fn, ' spread: ', spread num_col = 8 # adding spd vol, last_trd_time, last_close_pxa tda = [] tda_bad = [] for f in fn: bar_sec = get_barsec_from_file(f) if bar_sec != default_barsec: if not barsec_from_file: raise ValueError( 'Bar second mismatch for file %s with barsec %d' % (f, default_barsec)) else: print 'Set barsec to ', bar_sec, ' from ', default_barsec try: d0, d1 = get_days_from_file(f) _, _, b = bar_by_file_ib(f, symbol, start_day=max(sday, d0), end_day=min(eday, d1)) except KeyboardInterrupt as e: raise e except Exception as e: print e b = [] if len(b) > 0: ba, td, col, bad_days, last_px = write_daily_bar( symbol, b, bar_sec=bar_sec, is_front=is_front_future, get_missing=get_missing) if overwrite_dbar: for td0 in td: # assuming days in increasing order: don't delete days # just written # don't delete if the barsec does not match # Because I don't want the IB_hist (barsec=1) to # overwrite the KDB days, which has barsec=5 if td0 not in tda: dbar_repo.remove_day(td0, match_barsec=bar_sec) dbar_repo.update(ba, td, col, bar_sec) tda += td tda_bad += bad_days else: print '!!! No bars was read from ', f tda = list(set(tda)) tda.sort() tda_bad = list(set(tda_bad)) tda_bad.sort() # The following gets the days that are either in tda nor in # tda_bad, i.e. some missing days not found in any history files # todo - this shouldn't happen and most probably due to the # half day/holidays, should remove if len(tda) == 0: print 'NOTHING found! Not getting any missing days!' # in case there are some entirely missed days elif get_missing: # there could be some duplication in files, so # so some files has bad days but otherwise already in other files. missday = [] d0 = max(sday, EarliestMissingDay) print ' checking on the missing days from %s to %s' % (d0, eday) diter = l1.TradingDayIterator(d0) while d0 <= eday: if d0 not in tda and d0 not in tda_bad and d0 not in l1.bad_days: missday.append(d0) diter.next() d0 = diter.yyyymmdd() if len(missday) > 0: print 'getting the missing days ', missday from ibbar import get_missing_day fn = [] mdays = [] for md in missday: fn0 = get_missing_day(symbol, [md], bar_sec, is_front_future, reuse_exist_file=True) if len(fn0) > 0: fn += fn0 mdays.append(md) else: print 'nothing on missing day: ', md for f, d in zip(fn, mdays): try: _, _, b = bar_by_file_ib(f, symbol, start_day=d, end_day=d) if len(b) > 0: print 'got ', len( b), ' bars from ', f, ' on missing day', d ba, td, col, bad_days, lastpx0 = write_daily_bar( symbol, b, bar_sec=bar_sec, is_front=is_front_future, get_missing=False) tda += td tda_bad += bad_days if len(td) > 0: if overwrite_dbar: for td0 in td: dbar_repo.remove_day(td0, match_barsec=bar_sec) dbar_repo.update(ba, td, col, bar_sec) else: print 'no trading day is found from ', f, ' on missing day ', d else: print 'nothing got for missing day: ', d except KeyboardInterrupt as e: raise e except: traceback.print_exc() print 'problem processing file ', f tda.sort() tda_bad.sort() print 'Done! Bad Days: ', tda_bad return tda, tda_bad
def launch_sustain(): alive = False dtnow = datetime.datetime.now() while not should_run() and dtnow.weekday() != 6: print 'wait for Sunday open...' #reset_network() bounce_ibg() time.sleep(RESET_WAIT_SECOND) dtnow = datetime.datetime.now() while dtnow.weekday() == 6 and not should_run(): utcnow = l1.TradingDayIterator.local_dt_to_utc(dtnow) utcstart = get_utcstart() while utcnow < utcstart - RESET_WAIT_SECOND - 10: print 'wait for Sunday open...', utcnow, utcstart, utcstart - utcnow #reset_network() bounce_ibg() time.sleep(RESET_WAIT_SECOND) utcnow = l1.TradingDayIterator.cur_utc() print 'getting on-line, updating roll ', datetime.datetime.now() ibbar.update_ib_config(cfg_file=cfg) utcnow = l1.TradingDayIterator.cur_utc() if utcstart > utcnow and not is_in_daily_trading(): time.sleep(utcstart - utcnow) utcnow = l1.TradingDayIterator.cur_utc() print 'spining for start', utcnow while not is_in_daily_trading(): utcnow = l1.TradingDayIterator.cur_utc() #time.sleep( float((1000000-utcnow.microsecond)/1000)/1000.0 ) print 'starting on', utcnow alive = True tpm = TPMon() while should_run(): if is_in_daily_trading(): if not alive: print 'getting on-line, updating roll ', datetime.datetime.now( ) ibbar.update_ib_config(cfg_file=cfg) alive = True # poll and sustain for p in procs: if (p not in proc_map.keys()) or (not is_proc_alive( proc_map[p])): launch(p) time.sleep(1) if not tpm.check(): # All L2 repo hasn't been updated for 1 min # exit the process and retry in outer (while [ 1 ]) loop print 'stale detected, exit!' _should_run = False kill_all() alive = False sys.exit(1) continue else: if alive: print 'getting off-line, killing all ', datetime.datetime.now() kill_all() alive = False # do one hour of reset dtnow = datetime.datetime.now() utcstart = get_utcstart() cur_utc = l1.TradingDayIterator.cur_utc() while cur_utc <= utcstart - RESET_WAIT_SECOND - 10: print 'reset network', cur_utc, utcstart #reset_network() bounce_ibg() time.sleep(RESET_WAIT_SECOND) cur_utc = l1.TradingDayIterator.cur_utc() print 'getting on-line, updating roll ', datetime.datetime.now() ibbar.update_ib_config(cfg_file=cfg) cur_utc = l1.TradingDayIterator.cur_utc() if utcstart > cur_utc: time.sleep(utcstart - cur_utc) cur_utc = l1.TradingDayIterator.cur_utc() print 'spinning for start', cur_utc while cur_utc <= utcstart: cur_utc = l1.TradingDayIterator.cur_utc() alive = True tpm = TPMon() print 'stopped ', datetime.datetime.now() kill_all() if is_weekend(): # only do it on friday close dt = datetime.datetime.now() wd = dt.weekday() if wd == 4: remove_logs() # edrive prev_wk, this_wk = ibbar.move_bar( rsync_dir_list=['/cygdrive/e/ib/kisco/bar']) bar_path = ibbar.read_cfg('BarPath') #os.system('scp -r ' + bar_path + '/'+this_wk + ' ' + USER+'@'+DATA_MACHINE+':'+BAR_PATH) print 'moving bar files to ', this_wk print 'previous week was ', prev_wk #import IB_hist #IB_hist.weekly_get_ingest(rsync_dir_list=['/cygdrive/e/ib/kisco/hist']) eday = dt.strftime('%Y%m%d') tdi = l1.TradingDayIterator(eday) sday = tdi.prev_n_trade_day(5).yyyymmdd() #ibbar.weekly_get_hist(sday, eday) os.system("nohup python/ibbar.py " + sday + " " + eday + " 2>&1 >> ./gethist.log &") print "started nohup python/ibbar.py " + sday + " " + eday + " 2>&1 >> ./gethist.log &", datetime.datetime.now( ) time.sleep(30)
def get_missing_day(symbol, trd_day_arr, bar_sec, is_front, cid=None, reuse_exist_file=True, reuse_exist_only=False): """ Couple of options: reuse_exist_file: will take the previous daily file and try to reuse it reuse_exist_only: will only try to reuse the existing daily file. If not found, then don't run the ibclient. This is usually the case for unnecessary days (such as outside of sday/eday of file name). Note: if IB_CLIENT is not found, i.e. on the hp notebook, reuse_exist_only is set to true """ import copy ibclient = copy.deepcopy(IB_CLIENT) try: os.stat(ibclient) except: reuse_exist_only = True if reuse_exist_only: ibclient = None if cid is None: dt = datetime.datetime.now() cid = dt.month * 31 + dt.day + 300 + dt.second fnarr = [] for day in trd_day_arr: if day in l1.bad_days or l1.is_holiday(day): print 'not getting holiday ', day continue cur_day = datetime.datetime.now().strftime('%Y%m%d') tdi = l1.TradingDayIterator(cur_day) tdi.prev_n_trade_day(260) # IB allow 1 year 1S bar if day <= tdi.yyyymmdd(): print 'older than a year, IB not allowed to get ', day continue if l1.venue_by_symbol(symbol) == 'FX': fnarr += get_ib(day, day, cid=cid + 3, sym_list=[symbol], reuse_exist_file=reuse_exist_file, verbose=False, ibclient=ibclient) else: # future or etf next_contract = not is_front fnarr += get_ib_future([symbol], day, day, bar_sec, mock_run=False, cid=cid + 1, getqt=True, gettrd=True, next_contract=next_contract, reuse_exist_file=reuse_exist_file, verbose=False, ibclient=ibclient) return fnarr