def cnv2minute1(tk,library=mds.assetTS): dti = library.min_date(tk+'_1Second') dtf = library.max_date(tk+'_1Second') dts = pd.date_range(dti,dtf,freq='6MS').normalize() lst1=list(pd.to_datetime(dts)) lst1 = [l.to_pydatetime() for l in lst1] dts=[dti]+lst1+[dtf] try: mds.delete(tk,ds.freqHelper.minute, library=library) except Exception as e: print('UNABLE TO DELETE:') uu.printException(e) for t,dt0 in enumerate(dts[:-1]): dt1 = dts[t+1] if dt0>= dt1: continue try: df1 = mds.read(tk, ds.freqHelper.second, library=library, date_range=[dt0, dt1]) except Exception as e: print('Error reading {}'.format(tk)) uu.printException(e) for fld in ['close','yield_close','ask','yield_ask','bid','yield_bid']: if fld in df1: df1[fld][df1[fld]==0]=np.NaN df2 = convertFreq(df1,ds.freqHelper.minute) mds.append(df2,library=library,check_metadata=False)
def __init__(self, series, transformation='mom', frequency='monthly'): self.series = series self.transformation = transformation self.warnings = [] if frequency == 'monthly': self.freq = f.monthBegin elif frequency == 'quarterly': self.freq = f.quarterBegin self.name = series + '_' + transformation + '_' + frequency self.old_obj = mds.obj.load(self.name, path='economia') self.db = mds.read(name=series, freq=self.freq, library=mds.econVS) self.summary = {} self.accuracy = pd.DataFrame() self.country = self.db.md['country'] self.proj_ponta = None #### pensar nos casos em que: #### 1. pegamos variaveis com frequencia mensal e as transformamos para trimestral #### 2. pegamos variáveis com frequencia trimestral if self.old_obj == (None, None): self.update = False self.tested_lags_y = None self.tested_lags_x = None self.models_par = [] self.models_npar = [] self.prediction = {} self.avg_y = None self.y = {} self.x = {} self.selected_dict = {} self.n_start = None self.n_end = None self.config = {} self.mcs = [] self.models = {} self.mongo_list = [] self.colsorder = None else: self.update = True obj = self.old_obj[1]['obj'] self.models_par = obj.models_par self.models_npar = obj.models_npar self.prediction = obj.prediction self.avg_y = obj.avg_y self.y = obj.y self.x = obj.x self.selected_dict = obj.selected_dict self.n_start = obj.n_start self.n_end = obj.n_end self.config = obj.config self.mcs = obj.mcs self.models = obj.models self.mongo_list = obj.mongo_list self.colsorder = self.x['select'].columns.to_list()
def renameDIs(): from mDataStore.globalMongo import mds lst = mds.find(library=mds.assetTS2) for l in lst: if l[2].name.startswith('OD'): df1 = mds.read(l[2].name,l[2].freq,library=mds.assetTS2) df1.md.name =df1.md.name.replace('OD','DI1') mds.write(df1,library=mds.assetTS2) mds.delete(l[2].name,l[2].freq,library=mds.assetTS2)
def update(tt): global df1 dt0=df1.index[-1] xaxis_dt_format = '%d %b %Y, %H:%M:%S' print(' ### READ ####') df1_ = mds.read(TK, ds.freqHelper.minute, date_range=[dt(2010,1,1), dt(2035, 1, 1)], library=mds.mktBars).tz_convert('America/Sao_Paulo') if TEST: df1_ = df1_.iloc[:-T + tt] if tt%2==0: df1_.high=df1_.high*1.0001 else: df1_.high=df1_.high/1.0001 df1_['Date']=df1_.index df1_['idx'] = np.arange(df1_.shape[0]) df1_=df1_.set_index('idx') # t=df1_.index.get_loc(dt0,'nearest') t=len(inc_source.data['high1']) inc_source.data['high1']=df1_.high.iloc[:t].values.tolist() inc_source.data['low1'] = df1_.low.iloc[:t].values.tolist() inc_source.data['close1'] = df1_.close.iloc[:t].values.tolist() inc_source.data['open1'] = df1_.open.iloc[:t].values.tolist() inc_source.data['high1'] = df1_.high.iloc[:t].values.tolist() inc = df1_.close.iloc[:t] > df1_.open.iloc[:t] inc_source.data['color'] = np.where(inc,INCREASING_COLOR,DECREASING_COLOR).tolist() if df1_.index[-1] != df1.index[-1]: for t1 in range(t,df1_.shape[0]): inc = df1_.close.iloc[-1] > df1_.open.iloc[-1] color=INCREASING_COLOR if inc else DECREASING_COLOR new_data = dict( x1= [df1_.index[t1]], Date1=[df1_.Date.values[t1]], open1=[df1_.open.iloc[t1]], high1=[df1_.high.iloc[t1]], low1=[df1_.low.iloc[t1]], close1=[df1_.close.iloc[t1]], color=[color], ) inc_source.stream(new_data) dt1=pd.Index(df1_.Date).tz_localize(None).values fig.xaxis.major_label_overrides = { i: date.strftime(xaxis_dt_format) for i, date in enumerate(pd.to_datetime(dt1)) } a=1 df1=df1_ print(' ### DONE ####')
def correctGenFut(mds, code, freq): ''' Correct data for Gen Future from reuters. It sometimes use the second future instead of the first. just deleting this data. ''' symbol = code + '_' + freq dt0 = mds.assetTS.min_date(symbol) dt1 = mds.assetTS.max_date(symbol) dts = pd.date_range(dt0, dt1, freq='3MS').tolist() dts.append(dt1) print('analyzing {}'.format(code)) #last_underlying = [] for t, dt0_ in enumerate(dts[:-1]): dt1_ = dts[t + 1] # print('analyzing {} : {} - {}'.format(code,dt0_, dt1_)) df = mds.read(code, freq=freq, date_range=[dt0_, dt1_]) df1 = df.copy() #order unames according to time (so, invert year and month. #underlying will be replaced by integers reflecting order unames_ = pd.unique(df.underlying) unames=unames_.copy() for i in range(len(unames)): dt1=df.index[df.underlying==unames_[i]][-1] match = re.match(r"([a-z]+)([0-9]+)", unames_[i], re.I) assert(match) items = match.groups() dig=1*(dt1.year - 2000-int(items[1][-1])>2) unames[i]=items[0][:-1]+dig.__str__()+ items[1][-1]+items[0][-1] idx=np.argsort(unames) map1 = {k:v for v,k in enumerate(unames_[idx])} df1.underlying = df1.underlying.replace(map1) # df1['cnt']=cntrep(df1.underlying.values) # df1.underlying[df1['cnt']<1500] I = df1.underlying.values[1:] < df1.underlying.cummax().values[:-1] # I = df1.underlying.values[:-1] > df1.underlying.cummin().values[:0:-1] I1=I[1:]!=I[:-1] J = np.where(I1)[0] dt_rep = df1.index[J + 1] dt_rep=dt_rep.append(df1.index[-1:].shift(1,'S')) # dt_rep = df1.index[J] if len(dt_rep)>0: for j in range(len(dt_rep)-1): if I[J[j]+1]: dt_rg = DateRange(dt_rep[j], dt_rep[j+1]) print('correcting: {} - dates: {} - {}'.format(symbol,dt_rg.start,dt_rg.end)) a=1 mongoDS.TS_deleteDateRange(mds.assetTS, symbol, dt_rg, df)
def convertOne(i, assetBase, date_range, freq=ds.freqHelper.minute, freqBase=ds.freqHelper.second, firstRun=False): from mDataStore.globalMongo import mds name = assetBase.name[i] #################### REMOVE THIS ##########################3 # if firstRun: # try: # mds.delete(name, freq, mds.assetTS) # except: # pass ##########################################################3 print('starting {} . {}/{}'.format(name, i, len(assetBase.name))) try: df1 = mds.read(name, freq, mds.assetTS, date_range=date_range) print('Already Done!') return except: pass try: df = mds.read(name, freqBase, mds.assetTS, date_range=date_range, tz='GMT') except: print('unable to read {} - dt {}->{}'.format(name, date_range[0], date_range[1])) return df1 = convertFreq(df, freq) df = mds.write(df1, mds.assetTS)
def reutersSplitGenTickerIntoContractsRoutine( genTickers=['DIJc{}'.format(i + 1) for i in range(38)], dt0=dt(2000, 1, 1), freq='6MS', library=mds.assetTS): dts = pd.date_range(dt0, dt.now(), freq=freq, closed='left') dts = dts.append(pd.DatetimeIndex([dt.now()])) uCode0 = [] for t, dt1 in enumerate(dts[1:]): dt0_ = dts[t] dfs = mds.read(genTickers, f.second, library, date_range=[dt0_, dt1]) df1 = pd.concat(dfs, 0) if df1.shape[0] == 0: continue print('dt: {}'.format(dt0_)) df1 = df1.rename(columns={ 'close': 'yield_close', 'ask': 'yield_bid', 'bid': 'yield_ask' }) uCode = pd.unique(df1.underlying) #rename for 2 digits for i, code1 in enumerate(uCode): match = re.match(r"([a-z]+)([0-9]+)", code1, re.I) items = match.groups() if len(items[1]) == 1: #1 digit try: mds.delete(code1, f.second, library) except: pass dig = 1 * (dt0_.year - 2000 - int(items[1][-1]) > 2) newCode = items[0][:-1] + items[0][-1] + dig.__str__( ) + items[1][-1] df1.underlying.loc[df1.underlying == uCode[i]] = newCode uCode[i] = newCode #year1 = 2000 + int(match[2]) #delete previous data for nm1 in uCode: if nm1 not in uCode0: try: mds.delete(nm1, f.second, mds.assetTS) except: pass uCode0.append(nm1) updateAllContracts(df1)
def cp2onlineVS(): from mDataStore.globalMongo import mds ms = mds.find(library=mds.assetVS) for i,m in enumerate(ms): nm = m[2].name print('Copying {} - {}/{}'.format(nm,i,len(ms))) if m[2].freq=='1BDay': df = mds.read(nm,'1BDay',mds.assetVS,date_range=[dt(1990,1,1),dt(2035,1,1)]) try: mds.delete(nm,'1BDay',library=mds.onlineVS) except: pass mds.write(df,library=mds.onlineVS,check_metadata=False)
def __init__(self, y_all): self.y_all = y_all YX = pd.DataFrame() for el in y_all: X = mds.read(name=el, freq=f.monthBegin, library=mds.econVS) YX = pd.concat([YX, X.iloc[:, 0]], axis=1) YX.columns = y_all YX = YX.diff(periods=1).dropna() weights = LinearRegression(fit_intercept=False).fit( YX.iloc[:, 1:], YX.iloc[:, 0]).coef_ self.weights = weights / sum(weights) self.alpha = {} self.beta = {} self.gamma = {}
def getIntradayHistoricDataBA(self, feeder_id, interval, startDate, endDate, md, event='TRADE', mds=None, **kwargs): if mds is None: from mDataStore.globalMongo import mds from mDataStore.mongo import mDataFrame df = mDataFrame(self.getIntradayHistoricData(feeder_id,interval , startDate, endDate,event='TRADE', **kwargs)) df_bid = self.getIntradayHistoricData(feeder_id,interval , startDate, endDate,event='BID', **kwargs) df_ask = self.getIntradayHistoricData(feeder_id,interval , startDate, endDate,event='ASK', **kwargs) #desc1 = mds.blp.getRefData(feeder_id, desc) df.md=md df.rename(columns={'numEvents': 'trades'}, inplace=True) df.index=df.index.tz_localize('GMT') df_bid.index = df_bid.index.tz_localize('GMT') df_ask.index = df_ask.index.tz_localize('GMT') # overwrite=True if md.subtype in ['fut_nrol','fut_rol']: dts = mds.read(df.md.name.lower() + '_dates', library=mds.fundamentalVS) dts1=pd.Index(pd.to_datetime(dts.iloc[:,1])) dts1=dts1.tz_localize(df.index.tzinfo.zone) I = dts1.get_indexer(df.index,method='bfill') underlying=dts.iloc[I,0] df['underlying']=underlying.values df['bid'] =nan J = df_bid.index.get_indexer(df.index) df.loc[J>=0,'bid']=df_bid.close.iloc[J[J>=0]].values df['ask'] =nan J = df_ask.index.get_indexer(df.index) df.loc[J>=0,'ask']=df_ask.close.iloc[J[J>=0]].values # df['vwap'] = df=df.rename(columns={'trades':'gmt_off'}) df['vwap']= df['value']/df['volume'] del df['value'] return df
def cp2blp(librarySource=mds.assetTS,libraryDest=mds.assetTS2): # from mDataStore.bloomberg import get_srs_meta # if srsMeta is None: # srsMeta = get_srs_meta() nms,nmsBlp=cnvAllReutersTickers(library=librarySource) #k=[i for i,nm in enumerate(nms) if nm =='ESc1'][0] for i,nm in enumerate(nms): print('Copying {} - {}/{}'.format(nm,i,len(nms))) # if not nmsBlp[i].startswith('DI1'): # continue df1=mds.read(nm,ds.freqHelper.minute,librarySource,date_range=[dt(1990,1,1),dt(2030,1,1)]) df1.md.name=nmsBlp[i] try: mds.delete(df1.md.name,df1.md.freq,library=libraryDest) except: pass mds.write(df1,library=libraryDest)
def updateCDI(self): df = mds.read('CDI', '1BDay', library=mds.assetVS) dt0 = dt.now().replace(hour=0, minute=0, second=0, microsecond=0) cdi0 = self.blp.getRefData(['BZDIOVRA Index'], ['px_last']).values[0] / 100 cdi0 = (1 + cdi0)**(1 / 252) - 1 j = df.index.get_loc(dt0, method='ffill') if df.index[j] == dt0: cdi_tm1 = df.close[j - 1] else: cdi_tm1 = df.close[j] df.loc[dt0] = cdi_tm1 * (1 + cdi0) mds.delete(df.md.name, df.md.freq, library=mds.onlineVS) mds.write(df, library=mds.onlineVS, check_metadata=False)
def saveFuture2pickle(mds, code, freq,path): symbol = code + '_' + freq dt0 = mds.assetTS.min_date(symbol) dt1 = mds.assetTS.max_date(symbol) dts = pd.date_range(dt0, dt1, freq='6MS').tolist() dts.append(dt1) print('analyzing {}'.format(code)) #last_underlying = [] for t, dt0_ in enumerate(dts[:-1]): dt1_ = dts[t + 1] # print('analyzing {} : {} - {}'.format(code,dt0_, dt1_)) df = mds.read(code, freq=freq, date_range=[dt0_, dt1_]) if df.shape[0]>0: name=symbol+'_{:%Y-%m-%d}_{:%Y-%m-%d}'.format(dt0_, dt1_) uu.save_obj(df,name,path=path) return name
def clean(self, transf='mom', min_y_sample=.8, lags_y=1, lags_X=(0, 4)): if lags_y == 0 and lags_X == (0, 0): with_lags = False else: with_lags = True for el in self.y_all: db = mds.read(name=el, freq=f.monthBegin, library=mds.econVS) alpha = sl_clean(target=db) alpha.transform_y(transf) alpha.get_db(check_real=True, check_seas=True, min_y_sample=min_y_sample, mongo_list=None) alpha.make_lags(lags_y=lags_y, lags_X=lags_X) alpha.check_rel_date(with_lags=with_lags, new_output=False, update_mongo_list=True) self.alpha[el] = alpha self.with_lags = with_lags
def updateDB(): input = xw.Book.caller().sheets['db_input'] out = xw.Book.caller().sheets['db'] tb_blp = input.range('blp_hist_inp[#ALL]').options(pd.DataFrame, expand='table').value tb_db = input.range('db_inp[#ALL]').options(pd.DataFrame, expand='table').value out.clear_contents() k = 1 for i in range(tb_db.shape[0]): at = tb_db.index[i] freq = tb_db.freq[i] library = tb_db.library[i] dti = tb_db.dt_start[i] dtf = tb_db.dt_end[i] dfO = mds.read(at, freq, library=library, date_range=[dti, dtf]) out.cells(1, k).value = pd.DataFrame(dfO) out.cells(1, k).value = at + '_' + freq k += dfO.shape[1] + 2 for i in range(tb_blp.shape[0]): at = tb_blp.index[i] dti = tb_blp.dt_start[i] dtf = tb_blp.dt_end[i] fields = tb_blp.iloc[i, 2:] fields = fields[~fields.isna()].values.tolist() dfs = blp1.getHistoricData([at], fields, dti, dtf) dfs = dfs[0] out.cells(1, k).value = pd.DataFrame(dfs) out.cells(1, k).value = at k += dfs.shape[1] + 2
dbdi_du.index = uu.x2dti_date(dbdi_du.index) dbdi_tx.index = uu.x2dti_date(dbdi_tx.index) #fer=uu.m2pdate(fer[:,0]) #ferDT=fer.date #dbdi=Dict(dbdi) #DATA=Dict(DATA) out = Dict() out.cpom = cpom #pd.DataFrame(data=cpom[:,1],index=uu.x2pdate(cpom[:,0]),columns=['rate']) out.cpomAll = out.cpom.copy() out.du = dbdi_du out.tx = dbdi_tx / 100 #out.cdi=pd.DataFrame(data=DATA.INDEX.CDI[:,1],index=uu.x2pdate(DATA.INDEX.CDI[:,0]),columns=['rate']) out.cdi = mds.read('CDI') out.cdi.iloc[:, 0] = insert(diff(log(out.cdi.values), axis=0), 0, 0) out.cdi.iloc[:, 0] = exp(out.cdi)**252 - 1 out.cpom = out.cpom.reindex(out.du.index, method='ffill') / 100 out.cpom = out.cpom.fillna(method='ffill') out.cdi = out.cdi.reindex(out.du.index, method='ffill') delt = (out.cdi - out.cpom).rolling(30).mean() out.cdiProxy = out.cpom.shift(1) + delt #out.cpom=out.cpom[(~pd.isna(out.cpom)).values] # t=out.tx.shape[0]-1 # dt=out.tx.index[t] # t0=out.cpom.index.searchsorted(dt)
from py_init import * from mDataStore.globalMongo import mds df1 = pd.DataFrame(np.random.normal(300), index=pd.date_range(dt(2010, 1, 1), dt(2012, 1, 1), periods=300), columns=['close']) md1 = ds.metadataAsset(name='teste', type='index', subtype='index') df1.md = md1 mds.write(df1, library='testVS') dff = mds.read('teste', library='testVS')
[ds.freqHelper.minute15, ds.freqHelper.minute15, ds.freqHelper.minute15], 'library': [mds.mktBars, mds.mktBars, mds.mktBars] }, path='plot_data', saveStrategyAsAsset=False) # TK='ES1 index' N = 0 obj1, dobj = mds.obj.load('plot_cfg{}'.format(N), 'plot_data') TK = obj1['TK'] freq = obj1['freq'] library = obj1['library'] df = [ mds.read(TK1, freq[i], date_range=[dt(2010, 1, 1), dt(2020, 1, 1)], library=library[i]).tz_convert('America/Sao_Paulo') for i, TK1 in enumerate(TK) ] # for i,freq1 in enumerate(freq): # if freq1 != ds.freqHelper.minute: # df[i]=convertFreq(df[i],freq1) # df2 = mds.read('BZ1 index',ds.freqHelper.minute,date_range=[dt(2010,1,1),dt(2020,1,1)],library=mds.mktBars).tz_convert('America/Sao_Paulo') if TEST: T = 1000 for i in range(len(df)): df[i] = df[i].iloc[:-T] fig = []
def doBars(self): #for intraday bars from mDataStore.mongo import mDataFrame, metadataAsset from mDataStore import mongo if self.onlineTable is None: return t1 = time.time() # print('doBars1') if hasattr(self, 't0_bars') and (t1 - self.t0_bars < 1): return self.t0_bars = t1 dt_today = dt.today().replace(hour=0, minute=0, second=0, microsecond=0) dt_today_loc = pd.Timestamp(dt_today).tz_localize('America/Sao_Paulo') dt_max = dt.today().replace(year=dt_today.year + 1, hour=0, minute=0, second=0, microsecond=0) if not 'lastBlpIndex' in self.onlineTable: self.onlineTable['lastBlpIndex'] = dt_today_loc for i in range(self.onlineTable.shape[0]): a = Dict(self.onlineTable.iloc[i].to_dict()) if (not isinstance(a.INTRA_SHORT_NAME, str)) or (not isinstance( a.freq, str)): continue nfreq, sfreq = mongo.splitFreq(a.freq) #assume freq is minutes if (t1 - self.t0_intra[i] > a.updateInterval): # st1 = dt.now() - datetime.timedelta(seconds=a.updateInterval*5) #np.maximum(a.startDate,dt_today) # try: # df_ = mds.read(a.security,a.freq,library=mds.mktBars,date_range=[st1,dt_max]) # df1 = self.blp.getIntradayHistoricData(a.security, nfreq, st1, dt_max, event=a.event, # **a.kwargs) # except Exception as e: #first query of the day - get all times # df1 = self.blp.getIntradayHistoricData(a.security, nfreq, dt_today, dt_max, event=a.event, # **a.kwargs) # df1 = self.blp.getIntradayHistoricData(a.security, nfreq, dt_today, dt_max, event=a.event,**a.kwargs) self.t0_intra[i] = t1 try: md = mds.read_metadata( a.INTRA_SHORT_NAME, '1Minute', mds.assetTS2) #a.security.split(' ')[0] md.freq = a.freq except: md = metadataAsset(a.INTRA_SHORT_NAME, 'equity', freq=a.freq, feeder_id=a.FEEDER_ID) mds.blp = self.blp #dt_today df1 = self.blp.getIntradayHistoricDataBA( a.FEEDER_ID, nfreq, self.onlineTable.lastBlpIndex[i], dt_max, md, event=a.event, mds=mds) if df1.shape[0] == 0: continue self.onlineTable.lastBlpIndex.values[i] = df1.index[-1] # **a.kwargs) df1 = df1.rename(columns={'numEvents': 'trades'}) if df1.index.tzinfo is None: df1 = df1.tz_localize('GMT') print('doBars2 - ' + a.FEEDER_ID) try: mds.append(df1, library=mds.onlineTS, replaceIntersection=True, check_metadata=False) except Exception as e: warn('Unable to append {}'.format(df1.md.name)) uu.printException(e) # if len(a.addrs) : # self.putData({'messageType':'barsUpdate','data':a},a.addrs) if (t1 - self.t0_daily[i] > a.dailyUpdateInterval): # for each series in intradayQueries, check if the daily series is in onlineVS up to yesterday # If not, simply copy the series from assetVS to onlineVS. If it is not up-to-date, warn # self.t0_daily[i] = t1 dt_today1 = dt_today + datetime.timedelta(1) dt0 = dt(1900, 1, 1) if (df1.shape[0] == 0) or df1.index[-1] < dt_today_loc: warn( 'No prices for {}/{} today ({}) in bars - (intraday/onlineTS)' .format(a.INTRA_SHORT_NAME, nfreq, dt_today)) continue try: dfd = mds.read(a.daily_shortname, '1BDay', library=mds.assetVS, date_range=[dt0, dt_today1]) except Exception as e: print( 'Unable to read {}/{} from assetVS in bars - daily' .format(a.security, nfreq)) uu.printException(e) continue # df1 = df1.loc[df1.index<dt_today_loc] c1 = dfd.columns.intersection(df1.columns) c2 = dfd.columns.difference(df1.columns) dfi1 = df1[c1].iloc[-1] lastUpdate = dfi1.name dfi1.name = dfi1.name.normalize().tz_localize(None) for c in c2: dfi1[c] = array(nan) # if md.subtype == 'fut_rol': if 'underlying' in dfd: if not 'underlying' in df1: warn( 'Ignoring {}/{} for Daily. Underlying not present in bloomberg results' .format(a.INTRA_SHORT_NAME, nfreq)) continue dfi1['underlying'] = dfi1['underlying'].split(' ')[0] if dfd.underlying[-1] != dfi1['underlying']: continue #check if it is the corerct future, if not continue dfd_ = pd.DataFrame(dfi1).T for c in dfd_.columns: # if dfd[c].dtype in [float32,float64,int32,int64]: dfd_[c] = dfd_[c].astype(dfd[c].dtype) if dfd.md.subtype == 'di_fut': dfd_['yield_close'] = dfd_['close'] dfd_['close'] = NaN df2 = pd.concat((dfd, dfd_)) df2.md = dfd.md df2.md.lastUpdate = lastUpdate # if (not 't0_daily' in a): #first uptade in the day try: mds.delete(df2.md.name, df2.md.freq, library=mds.onlineVS ) #make sure not accumulating versions except Exception as e: pass try: mds.write(df2, library=mds.onlineVS, check_metadata=False, prune_previous_version=True) except Exception as e: print( 'Unable to read {}/{} from assetVS in bars - daily' .format(a.security, nfreq)) uu.printException(e)
from py_init import * from mDataStore.globalMongo import mds from bokeh.plotting import curdoc, figure from bokeh.layouts import row, column, gridplot TK='ES1 index' TEST=False df1 = mds.read(TK,ds.freqHelper.minute,date_range=[dt(2010,1,1),dt(2020,1,1)],library=mds.mktBars).tz_convert('America/Sao_Paulo') # df2 = mds.read('BZ1 index',ds.freqHelper.minute,date_range=[dt(2010,1,1),dt(2020,1,1)],library=mds.mktBars).tz_convert('America/Sao_Paulo') if TEST: T=1000 df1=df1.iloc[:-T] fig,inc_source=uu.candlestick_plot(df1,df1.md.name) df1['Date'] = df1.index df1['idx'] = np.arange(df1.shape[0]) df1 = df1.set_index('idx') INCREASING_COLOR = '#17BECF' DECREASING_COLOR = '#7F7F7F' from bokeh.driving import count @count() def update(tt): global df1 dt0=df1.index[-1] xaxis_dt_format = '%d %b %Y, %H:%M:%S' print(' ### READ ####') df1_ = mds.read(TK, ds.freqHelper.minute, date_range=[dt(2010,1,1), dt(2035, 1, 1)],
#eco = sl_estimate(y=Y_eco, X=X_eco) #tic = time.clock() #eco.calculate(al=True) #toc = time.clock() #time_eco = toc - tic #print('Eco model (10 variables) elapsed time:'+str(toc-tic)+'seconds') #eco.bench(bench=y_bench) #eco.plotP(md='avg_mcs',bench=y_bench) #eco.stats() db_pim = mds.read(name='pim_1 Industria geral', freq=f.monthBegin, library=mds.econVS) alpha = sl_clean(target=db_pim) alpha.transform_y(transform='yoy') alpha.get_db() alpha.make_lags() alpha.check_rel_date() Y = alpha.y_data X = alpha.X_df_l #Xl = alpha2.X_df_l beta = sl_select(y = Y, X = X) #beta_eco.select_prop(proportion = 0.8, method = 'f_regression') #mutual_info_regression beta.select_prop(proportion = 0.8, method = 'mutual_info_regression') #mutual_info_regression
'PRE_2A', 'ES1', 'TY1' ] freq = ['1BDay'] library = ['assetVS'] # get current date (d0) dt0 = dt.now().replace(hour=0, minute=0, second=0, microsecond=0) # get the workday before (d-1) dtm1 = pd.to_datetime(uu.workday(dt0, -1)[0]) # set date interval expDT = [dtm1, dt0] print('ativo \t\t\t\tdata \t\tdata_exp \tstatus') print('------------------------------------------------------') for a in al: for f in freq: for i, l in enumerate(library): expDT_ = expDT[i] try: dff = mds.read(a, freq=f, library=l) print('{:_<18} \t{:%d-%m-%Y} \t{:%d-%m-%Y} \t{}'.format( a, dff.index[-1], expDT_, 'ok' if dff.index[-1] >= expDT_ else 'ERROR')) except NoDataFoundException: print('{:_<20} \t{:%d-%m-%Y} \t{:%d-%m-%Y} \t{}'.format( a, dff.index[-1], expDT_, 'ERROR! Serie not found.')) print('------------------------------------------------------')