def cnv2minute1(tk,library=mds.assetTS):
dti = library.min_date(tk+'_1Second')
dtf = library.max_date(tk+'_1Second')
dts = pd.date_range(dti,dtf,freq='6MS').normalize()
lst1=list(pd.to_datetime(dts))
lst1 = [l.to_pydatetime() for l in lst1]
dts=[dti]+lst1+[dtf]
try:
mds.delete(tk,ds.freqHelper.minute, library=library)
except Exception as e:
print('UNABLE TO DELETE:')
uu.printException(e)
for t,dt0 in enumerate(dts[:-1]):
dt1 = dts[t+1]
if dt0>= dt1:
continue
try:
df1 = mds.read(tk, ds.freqHelper.second, library=library, date_range=[dt0, dt1])
except Exception as e:
print('Error reading {}'.format(tk))
uu.printException(e)
for fld in ['close','yield_close','ask','yield_ask','bid','yield_bid']:
if fld in df1:
df1[fld][df1[fld]==0]=np.NaN
df2 = convertFreq(df1,ds.freqHelper.minute)
mds.append(df2,library=library,check_metadata=False)
def renameDIs():
from mDataStore.globalMongo import mds
lst = mds.find(library=mds.assetTS2)
for l in lst:
if l[2].name.startswith('OD'):
df1 = mds.read(l[2].name,l[2].freq,library=mds.assetTS2)
df1.md.name =df1.md.name.replace('OD','DI1')
mds.write(df1,library=mds.assetTS2)
mds.delete(l[2].name,l[2].freq,library=mds.assetTS2)
def reutersSplitGenTickerIntoContractsRoutine(
genTickers=['DIJc{}'.format(i + 1) for i in range(38)],
dt0=dt(2000, 1, 1),
freq='6MS',
library=mds.assetTS):
dts = pd.date_range(dt0, dt.now(), freq=freq, closed='left')
dts = dts.append(pd.DatetimeIndex([dt.now()]))
uCode0 = []
for t, dt1 in enumerate(dts[1:]):
dt0_ = dts[t]
dfs = mds.read(genTickers, f.second, library, date_range=[dt0_, dt1])
df1 = pd.concat(dfs, 0)
if df1.shape[0] == 0:
continue
print('dt: {}'.format(dt0_))
df1 = df1.rename(columns={
'close': 'yield_close',
'ask': 'yield_bid',
'bid': 'yield_ask'
})
uCode = pd.unique(df1.underlying)
#rename for 2 digits
for i, code1 in enumerate(uCode):
match = re.match(r"([a-z]+)([0-9]+)", code1, re.I)
items = match.groups()
if len(items[1]) == 1: #1 digit
try:
mds.delete(code1, f.second, library)
except:
pass
dig = 1 * (dt0_.year - 2000 - int(items[1][-1]) > 2)
newCode = items[0][:-1] + items[0][-1] + dig.__str__(
) + items[1][-1]
df1.underlying.loc[df1.underlying == uCode[i]] = newCode
uCode[i] = newCode
#year1 = 2000 + int(match[2])
#delete previous data
for nm1 in uCode:
if nm1 not in uCode0:
try:
mds.delete(nm1, f.second, mds.assetTS)
except:
pass
uCode0.append(nm1)
updateAllContracts(df1)
def cp2onlineVS():
from mDataStore.globalMongo import mds
ms = mds.find(library=mds.assetVS)
for i,m in enumerate(ms):
nm = m[2].name
print('Copying {} - {}/{}'.format(nm,i,len(ms)))
if m[2].freq=='1BDay':
df = mds.read(nm,'1BDay',mds.assetVS,date_range=[dt(1990,1,1),dt(2035,1,1)])
try:
mds.delete(nm,'1BDay',library=mds.onlineVS)
except:
pass
mds.write(df,library=mds.onlineVS,check_metadata=False)
def updateCDI(self):
df = mds.read('CDI', '1BDay', library=mds.assetVS)
dt0 = dt.now().replace(hour=0, minute=0, second=0, microsecond=0)
cdi0 = self.blp.getRefData(['BZDIOVRA Index'],
['px_last']).values[0] / 100
cdi0 = (1 + cdi0)**(1 / 252) - 1
j = df.index.get_loc(dt0, method='ffill')
if df.index[j] == dt0:
cdi_tm1 = df.close[j - 1]
else:
cdi_tm1 = df.close[j]
df.loc[dt0] = cdi_tm1 * (1 + cdi0)
mds.delete(df.md.name, df.md.freq, library=mds.onlineVS)
mds.write(df, library=mds.onlineVS, check_metadata=False)
def cp2blp(librarySource=mds.assetTS,libraryDest=mds.assetTS2):
# from mDataStore.bloomberg import get_srs_meta
# if srsMeta is None:
# srsMeta = get_srs_meta()
nms,nmsBlp=cnvAllReutersTickers(library=librarySource)
#k=[i for i,nm in enumerate(nms) if nm =='ESc1'][0]
for i,nm in enumerate(nms):
print('Copying {} - {}/{}'.format(nm,i,len(nms)))
# if not nmsBlp[i].startswith('DI1'):
# continue
df1=mds.read(nm,ds.freqHelper.minute,librarySource,date_range=[dt(1990,1,1),dt(2030,1,1)])
df1.md.name=nmsBlp[i]
try:
mds.delete(df1.md.name,df1.md.freq,library=libraryDest)
except:
pass
mds.write(df1,library=libraryDest)
def doBars(self): #for intraday bars
from mDataStore.mongo import mDataFrame, metadataAsset
from mDataStore import mongo
if self.onlineTable is None:
return
t1 = time.time()
# print('doBars1')
if hasattr(self, 't0_bars') and (t1 - self.t0_bars < 1):
return
self.t0_bars = t1
dt_today = dt.today().replace(hour=0,
minute=0,
second=0,
microsecond=0)
dt_today_loc = pd.Timestamp(dt_today).tz_localize('America/Sao_Paulo')
dt_max = dt.today().replace(year=dt_today.year + 1,
hour=0,
minute=0,
second=0,
microsecond=0)
if not 'lastBlpIndex' in self.onlineTable:
self.onlineTable['lastBlpIndex'] = dt_today_loc
for i in range(self.onlineTable.shape[0]):
a = Dict(self.onlineTable.iloc[i].to_dict())
if (not isinstance(a.INTRA_SHORT_NAME, str)) or (not isinstance(
a.freq, str)):
continue
nfreq, sfreq = mongo.splitFreq(a.freq) #assume freq is minutes
if (t1 - self.t0_intra[i] > a.updateInterval):
# st1 = dt.now() - datetime.timedelta(seconds=a.updateInterval*5) #np.maximum(a.startDate,dt_today)
# try:
# df_ = mds.read(a.security,a.freq,library=mds.mktBars,date_range=[st1,dt_max])
# df1 = self.blp.getIntradayHistoricData(a.security, nfreq, st1, dt_max, event=a.event,
# **a.kwargs)
# except Exception as e: #first query of the day - get all times
# df1 = self.blp.getIntradayHistoricData(a.security, nfreq, dt_today, dt_max, event=a.event,
# **a.kwargs)
# df1 = self.blp.getIntradayHistoricData(a.security, nfreq, dt_today, dt_max, event=a.event,**a.kwargs)
self.t0_intra[i] = t1
try:
md = mds.read_metadata(
a.INTRA_SHORT_NAME, '1Minute',
mds.assetTS2) #a.security.split(' ')[0]
md.freq = a.freq
except:
md = metadataAsset(a.INTRA_SHORT_NAME,
'equity',
freq=a.freq,
feeder_id=a.FEEDER_ID)
mds.blp = self.blp
#dt_today
df1 = self.blp.getIntradayHistoricDataBA(
a.FEEDER_ID,
nfreq,
self.onlineTable.lastBlpIndex[i],
dt_max,
md,
event=a.event,
mds=mds)
if df1.shape[0] == 0:
continue
self.onlineTable.lastBlpIndex.values[i] = df1.index[-1]
# **a.kwargs)
df1 = df1.rename(columns={'numEvents': 'trades'})
if df1.index.tzinfo is None:
df1 = df1.tz_localize('GMT')
print('doBars2 - ' + a.FEEDER_ID)
try:
mds.append(df1,
library=mds.onlineTS,
replaceIntersection=True,
check_metadata=False)
except Exception as e:
warn('Unable to append {}'.format(df1.md.name))
uu.printException(e)
# if len(a.addrs) :
# self.putData({'messageType':'barsUpdate','data':a},a.addrs)
if (t1 - self.t0_daily[i] > a.dailyUpdateInterval):
# for each series in intradayQueries, check if the daily series is in onlineVS up to yesterday
# If not, simply copy the series from assetVS to onlineVS. If it is not up-to-date, warn
#
self.t0_daily[i] = t1
dt_today1 = dt_today + datetime.timedelta(1)
dt0 = dt(1900, 1, 1)
if (df1.shape[0] == 0) or df1.index[-1] < dt_today_loc:
warn(
'No prices for {}/{} today ({}) in bars - (intraday/onlineTS)'
.format(a.INTRA_SHORT_NAME, nfreq, dt_today))
continue
try:
dfd = mds.read(a.daily_shortname,
'1BDay',
library=mds.assetVS,
date_range=[dt0, dt_today1])
except Exception as e:
print(
'Unable to read {}/{} from assetVS in bars - daily'
.format(a.security, nfreq))
uu.printException(e)
continue
# df1 = df1.loc[df1.index<dt_today_loc]
c1 = dfd.columns.intersection(df1.columns)
c2 = dfd.columns.difference(df1.columns)
dfi1 = df1[c1].iloc[-1]
lastUpdate = dfi1.name
dfi1.name = dfi1.name.normalize().tz_localize(None)
for c in c2:
dfi1[c] = array(nan)
# if md.subtype == 'fut_rol':
if 'underlying' in dfd:
if not 'underlying' in df1:
warn(
'Ignoring {}/{} for Daily. Underlying not present in bloomberg results'
.format(a.INTRA_SHORT_NAME, nfreq))
continue
dfi1['underlying'] = dfi1['underlying'].split(' ')[0]
if dfd.underlying[-1] != dfi1['underlying']:
continue
#check if it is the corerct future, if not continue
dfd_ = pd.DataFrame(dfi1).T
for c in dfd_.columns:
# if dfd[c].dtype in [float32,float64,int32,int64]:
dfd_[c] = dfd_[c].astype(dfd[c].dtype)
if dfd.md.subtype == 'di_fut':
dfd_['yield_close'] = dfd_['close']
dfd_['close'] = NaN
df2 = pd.concat((dfd, dfd_))
df2.md = dfd.md
df2.md.lastUpdate = lastUpdate
# if (not 't0_daily' in a): #first uptade in the day
try:
mds.delete(df2.md.name,
df2.md.freq,
library=mds.onlineVS
) #make sure not accumulating versions
except Exception as e:
pass
try:
mds.write(df2,
library=mds.onlineVS,
check_metadata=False,
prune_previous_version=True)
except Exception as e:
print(
'Unable to read {}/{} from assetVS in bars - daily'
.format(a.security, nfreq))
uu.printException(e)