def cnv2minute1(tk,library=mds.assetTS): dti = library.min_date(tk+'_1Second') dtf = library.max_date(tk+'_1Second') dts = pd.date_range(dti,dtf,freq='6MS').normalize() lst1=list(pd.to_datetime(dts)) lst1 = [l.to_pydatetime() for l in lst1] dts=[dti]+lst1+[dtf] try: mds.delete(tk,ds.freqHelper.minute, library=library) except Exception as e: print('UNABLE TO DELETE:') uu.printException(e) for t,dt0 in enumerate(dts[:-1]): dt1 = dts[t+1] if dt0>= dt1: continue try: df1 = mds.read(tk, ds.freqHelper.second, library=library, date_range=[dt0, dt1]) except Exception as e: print('Error reading {}'.format(tk)) uu.printException(e) for fld in ['close','yield_close','ask','yield_ask','bid','yield_bid']: if fld in df1: df1[fld][df1[fld]==0]=np.NaN df2 = convertFreq(df1,ds.freqHelper.minute) mds.append(df2,library=library,check_metadata=False)
def loadFuturesFrompickle(mds:mongoDS,library='assetTS'): names=uu.save_obj('saveFutures2pickle',path=path1) path1=globalM.dataRoot+'/futPickle' for nm in names: print('uploading {}'.format(name)) df=uu.load_obj(name,path=path1) mds.append(df,library=library)
def importFromReutersCSV(inputPath, file1, outPath, columns=None, chunksize=4 * 2**20, library=None): #reutersPath = globalM.dataRoot + '/datascope_csv' with gzip.open(inputPath + file1) as f_: for i, srs in enumerate(pd.read_csv(f_, chunksize=chunksize)): # reutersPath = globalM.dataRoot+'/datascope_csv' print('{} - chunk {}'.format(file1, i)) dt1 = pd.to_datetime(srs['Date-Time'], format='%Y-%m-%dT%H:%M:%S.%fZ') srs['Date-Time'] = dt1 type1 = pd.unique(srs.Type) del srs['Domain'] del srs['Type'] if not columns is None: srs.columns = columns else: srs = srs.rename( columns={ '#RIC': 'code', 'Date-Time': 'dt', 'GMT Offset': 'gmt_off', 'Close Bid': 'bid', 'Close Ask': 'ask', 'Last': 'close', 'Volume': 'volume', 'Alias Underlying RIC': 'underlying' }) # srs.rename(columns={'code', 'underlying', 'dt', 'gmt_off'}) # srs.columns.values[:4] = ['code', 'underlying', 'dt', 'gmt_off'] srs.gmt_off = srs.gmt_off.astype(int) # delta1 = pd.to_timedelta(srs.gmt_off, unit='h') # srs.dt = srs.dt + delta1 codes = pd.unique(srs.code) out = Dict() fr = {'Intraday 5Sec': f.second5, 'Intraday 1Sec': f.second} for code in codes: out[code] = mDataFrame( srs[srs.code == code].iloc[:, 1:].set_index('dt')) out[code].md = metadataAsset(code, 'equity', freq=fr[type1[0]]) for nm in out: # if not mds.read_metadata(out[nm].md.name,out[nm].md.freq,mds.assetTS): out[nm].index = out[nm].index.tz_localize(pytz.timezone('GMT')) mds.append(out[nm], check_metadata=False, keep_metadata=True, library=library) file1_done = outPath + file1 os.rename(inputPath + file1, file1_done)
def updateAllContracts(df1): uCode = pd.unique(df1.underlying) monthCodes = array( ['F', 'G', 'H', 'J', 'K', 'M', 'N', 'Q', 'U', 'V', 'X', 'Z']) for i, code1 in enumerate(uCode): df_ = mDataFrame(df1.loc[df1.underlying == code1].copy()) del df_['underlying'] df_ = df_.astype(float64) match = re.match(r"([a-z]+)([0-9]+)", code1, re.I) assert (match) month1 = where(monthCodes == match[1][-1])[0][0] + 1 year1 = 2000 + int(match[2]) mat1 = dt(year1, month1, 1) if not df_.index.tz: df_.index = df_.index.tz_localize( pytz.timezone('GMT')) # Brazil/East md1 = mds.read_metadata(code1, f.second, mds.assetTS) if md1: tz = pytz.timezone('GMT') stDT = minimum(df_.index[0].to_pydatetime(), tz.localize(md1.stDT)) else: stDT = df_.index[0].to_pydatetime() df_.md = metadataAsset(code1, 'future', stDT, fut_like=True, maturity=mat1, freq=f.second) df_.md.subtype = 'di_fut_intraday' # print('appending {}. {}/{}'.format(code1,i,len(uCode))) # print('appending {}. {}/{}'.format(code1,i,len(uCode))) if not df_.index.is_unique: df_ = uu.drop_duplicates_index(df_) # try: # mds.delete(code1,f.second,mds.assetTS) # except: # pass mds.append(df_, mds.assetTS, check_metadata=False)
def doBars(self): #for intraday bars from mDataStore.mongo import mDataFrame, metadataAsset from mDataStore import mongo if self.onlineTable is None: return t1 = time.time() # print('doBars1') if hasattr(self, 't0_bars') and (t1 - self.t0_bars < 1): return self.t0_bars = t1 dt_today = dt.today().replace(hour=0, minute=0, second=0, microsecond=0) dt_today_loc = pd.Timestamp(dt_today).tz_localize('America/Sao_Paulo') dt_max = dt.today().replace(year=dt_today.year + 1, hour=0, minute=0, second=0, microsecond=0) if not 'lastBlpIndex' in self.onlineTable: self.onlineTable['lastBlpIndex'] = dt_today_loc for i in range(self.onlineTable.shape[0]): a = Dict(self.onlineTable.iloc[i].to_dict()) if (not isinstance(a.INTRA_SHORT_NAME, str)) or (not isinstance( a.freq, str)): continue nfreq, sfreq = mongo.splitFreq(a.freq) #assume freq is minutes if (t1 - self.t0_intra[i] > a.updateInterval): # st1 = dt.now() - datetime.timedelta(seconds=a.updateInterval*5) #np.maximum(a.startDate,dt_today) # try: # df_ = mds.read(a.security,a.freq,library=mds.mktBars,date_range=[st1,dt_max]) # df1 = self.blp.getIntradayHistoricData(a.security, nfreq, st1, dt_max, event=a.event, # **a.kwargs) # except Exception as e: #first query of the day - get all times # df1 = self.blp.getIntradayHistoricData(a.security, nfreq, dt_today, dt_max, event=a.event, # **a.kwargs) # df1 = self.blp.getIntradayHistoricData(a.security, nfreq, dt_today, dt_max, event=a.event,**a.kwargs) self.t0_intra[i] = t1 try: md = mds.read_metadata( a.INTRA_SHORT_NAME, '1Minute', mds.assetTS2) #a.security.split(' ')[0] md.freq = a.freq except: md = metadataAsset(a.INTRA_SHORT_NAME, 'equity', freq=a.freq, feeder_id=a.FEEDER_ID) mds.blp = self.blp #dt_today df1 = self.blp.getIntradayHistoricDataBA( a.FEEDER_ID, nfreq, self.onlineTable.lastBlpIndex[i], dt_max, md, event=a.event, mds=mds) if df1.shape[0] == 0: continue self.onlineTable.lastBlpIndex.values[i] = df1.index[-1] # **a.kwargs) df1 = df1.rename(columns={'numEvents': 'trades'}) if df1.index.tzinfo is None: df1 = df1.tz_localize('GMT') print('doBars2 - ' + a.FEEDER_ID) try: mds.append(df1, library=mds.onlineTS, replaceIntersection=True, check_metadata=False) except Exception as e: warn('Unable to append {}'.format(df1.md.name)) uu.printException(e) # if len(a.addrs) : # self.putData({'messageType':'barsUpdate','data':a},a.addrs) if (t1 - self.t0_daily[i] > a.dailyUpdateInterval): # for each series in intradayQueries, check if the daily series is in onlineVS up to yesterday # If not, simply copy the series from assetVS to onlineVS. If it is not up-to-date, warn # self.t0_daily[i] = t1 dt_today1 = dt_today + datetime.timedelta(1) dt0 = dt(1900, 1, 1) if (df1.shape[0] == 0) or df1.index[-1] < dt_today_loc: warn( 'No prices for {}/{} today ({}) in bars - (intraday/onlineTS)' .format(a.INTRA_SHORT_NAME, nfreq, dt_today)) continue try: dfd = mds.read(a.daily_shortname, '1BDay', library=mds.assetVS, date_range=[dt0, dt_today1]) except Exception as e: print( 'Unable to read {}/{} from assetVS in bars - daily' .format(a.security, nfreq)) uu.printException(e) continue # df1 = df1.loc[df1.index<dt_today_loc] c1 = dfd.columns.intersection(df1.columns) c2 = dfd.columns.difference(df1.columns) dfi1 = df1[c1].iloc[-1] lastUpdate = dfi1.name dfi1.name = dfi1.name.normalize().tz_localize(None) for c in c2: dfi1[c] = array(nan) # if md.subtype == 'fut_rol': if 'underlying' in dfd: if not 'underlying' in df1: warn( 'Ignoring {}/{} for Daily. Underlying not present in bloomberg results' .format(a.INTRA_SHORT_NAME, nfreq)) continue dfi1['underlying'] = dfi1['underlying'].split(' ')[0] if dfd.underlying[-1] != dfi1['underlying']: continue #check if it is the corerct future, if not continue dfd_ = pd.DataFrame(dfi1).T for c in dfd_.columns: # if dfd[c].dtype in [float32,float64,int32,int64]: dfd_[c] = dfd_[c].astype(dfd[c].dtype) if dfd.md.subtype == 'di_fut': dfd_['yield_close'] = dfd_['close'] dfd_['close'] = NaN df2 = pd.concat((dfd, dfd_)) df2.md = dfd.md df2.md.lastUpdate = lastUpdate # if (not 't0_daily' in a): #first uptade in the day try: mds.delete(df2.md.name, df2.md.freq, library=mds.onlineVS ) #make sure not accumulating versions except Exception as e: pass try: mds.write(df2, library=mds.onlineVS, check_metadata=False, prune_previous_version=True) except Exception as e: print( 'Unable to read {}/{} from assetVS in bars - daily' .format(a.security, nfreq)) uu.printException(e)
def bloombergUpdateIntraday(meta, dti, bOnlyUpdateMetadata=False, srsMeta=None, overwrite=False, dtm1=None, onlineTS='onlineTS',keep_metadata=False): from mDataStore import mDataFrame, metadataFundamental, metadataOption, metadataAsset, metadataIndicator, \ metadataStrategy meta1 = [m for m in meta if 'feeder_id_intraday' in m] N=len(meta1) if dtm1 is None: import pytz tz = pytz.timezone('GMT') dtm1 = (dt.now()-datetime.timedelta(1)).replace(hour=0, minute=0, second=0, microsecond=0) # success = np.full(hist1.shape[0], True) lst_failures=[] for k in tqdm(range(N)): #tqdm( # nfreq, sfreq = mongo.splitFreq(meta1[k]['freq_intraday']) interval = 1#nfreq # # if currency1[k] is None: # # try: # # currency = desc1.CRNCY[feeder_id[k]] # # except: # currency = 'BRL' # else: # currency = currency1[k] if not 'options_intraday' in meta1[k] or not isinstance(meta1[k]['options_intraday'], str) or meta1[k]['options_intraday']=='': options = {} else: options = eval('dict('+meta1[k]['options_intraday']+')') # get output columns md=copy(meta1[k]) md['freq']='1Minute' md = metadataAsset(**md) library1 = 'assetTS2'#md.pop('library') if 'stDT' in md: md.pop('stDT') if 'endDT' in md: md.pop('endDT') mt_ant = mds.read_metadata(md['name'],'1Minute',library1) if mt_ant and 'endDT' in mt_ant and dti > mt_ant['endDT']: dti=pd.Timestamp(mt_ant['endDT']).normalize().to_pydatetime() df = blp.getIntradayHistoricDataBA(md.feeder_id_intraday, interval, dti, dt.now(), md, **options) if df.shape[0] == 0: continue # df.md.stDT = df.index[0] # df.md.endDT = df.index[-1] # if overwrite: # for s in df.columns: # if (df[s].dtype == np.float) and np.all(np.isnan(df[s])): # del df[s] # # # lst=mds.find(df.md.name, df.md.freq, library=library1[k]) # att = df.md.name + '_' + df.md.freq # exists = att in getattr(mds, library1).list_symbols() # if exists: # mds.delete(df.md.name, df.md.freq, library=library1) # # mds.write(df, library=library1, check_metadata=False) # # mds.write(df, library=onlineTS, check_metadata=False) # # successi = True # # else: try: mds.append(df,date_range=[dti, dtm1], library=library1, check_metadata=False,keep_metadata=keep_metadata, replaceIntersection=overwrite) successi=True except: try: mds.append(df, date_range=[dti, dtm1], library=library1, check_metadata=False, replaceIntersection=True, keep_metadata=keep_metadata) except: successi=False # mds.append(df, date_range=[dti, dtm1], library=onlineTS, check_metadata=False, replaceIntersection=True,keep_metadata=True) if not successi: lst_failures.append(df.md.name) return lst_failures