def convertFreqAll_old(freq=ds.freqHelper.minute15):
from strategy import asset
futRolTKs = ['DOL', 'IND', 'ES', 'TY']
#treasury is in fact US/Centrall tz.
#in fact both are roughly 24hours, but I am getting the most liquid market time.
tzs = [
'America/Sao_Paulo', 'America/Sao_Paulo', 'US/Eastern', 'US/Eastern'
]
#['9:30','16:15'] is the ET time for the S&P big congtract
# trading_hours=[['9:00','18:00'],['9:00','18:00'],['7:30','16:15'],['7:30','16:15']]
for i, tk in enumerate(futRolTKs):
md = mds.read_metadata(tk + 'c1', ds.freqHelper.second, mds.assetTS)
df0 = feeder.get([tk + 'c1', tk + 'c2'],
freq=ds.freqHelper.second,
date_range=[dt(2000, 1, 1),
dt(2215, 1, 1)],
tz=tzs[i])
a = asset(feederCollection(df0))
df1 = a._mdf.copy()
df2 = convertFreq(df1, freq, trading_hours[i])
df2.md.name = tk + 'r1'
df2.md.subtype = 'fut_rol'
df2.md.freq = freq
mds.write(df2, mds.assetTS)
def fixInttradayMD(ats = ['IND','DOL','ES','TY'],freqs=['1Second','1Minute'],lib='assetTS'):
for at in ats:
for f in freqs:
for suff in ['c1','c2']:
print(at+suff)
md1 = mds.read_metadata(at+suff,f,lib)
md1.fut_like=True
mds.write_metadata(md1,lib)
def copyMetadata2Global(library):
from pydoc import locate
if isinstance(library,str):
library=getattr(mds,library)
nms = library.list_symbols()
library1 = mongo.getLibraryName(library)
for i,nm in enumerate(nms):
nm1, freq1=nm.split('_')
print('library: {} / name: {} (in library: {}/{})'.format(library1,nm,i,len(nms)))
try:
md1=mds.read_metadata(nm1,freq1,library=library)
# md1 = locate(md1['cls'])(md1)
mds.write_metadata(md1, library, onlyGlobal=True)
except Exception as e:
print('FAILED')
uu.printException(e)
def updateAllContracts(df1):
uCode = pd.unique(df1.underlying)
monthCodes = array(
['F', 'G', 'H', 'J', 'K', 'M', 'N', 'Q', 'U', 'V', 'X', 'Z'])
for i, code1 in enumerate(uCode):
df_ = mDataFrame(df1.loc[df1.underlying == code1].copy())
del df_['underlying']
df_ = df_.astype(float64)
match = re.match(r"([a-z]+)([0-9]+)", code1, re.I)
assert (match)
month1 = where(monthCodes == match[1][-1])[0][0] + 1
year1 = 2000 + int(match[2])
mat1 = dt(year1, month1, 1)
if not df_.index.tz:
df_.index = df_.index.tz_localize(
pytz.timezone('GMT')) # Brazil/East
md1 = mds.read_metadata(code1, f.second, mds.assetTS)
if md1:
tz = pytz.timezone('GMT')
stDT = minimum(df_.index[0].to_pydatetime(), tz.localize(md1.stDT))
else:
stDT = df_.index[0].to_pydatetime()
df_.md = metadataAsset(code1,
'future',
stDT,
fut_like=True,
maturity=mat1,
freq=f.second)
df_.md.subtype = 'di_fut_intraday'
# print('appending {}. {}/{}'.format(code1,i,len(uCode)))
# print('appending {}. {}/{}'.format(code1,i,len(uCode)))
if not df_.index.is_unique:
df_ = uu.drop_duplicates_index(df_)
# try:
# mds.delete(code1,f.second,mds.assetTS)
# except:
# pass
mds.append(df_, mds.assetTS, check_metadata=False)
def doBars(self): #for intraday bars
from mDataStore.mongo import mDataFrame, metadataAsset
from mDataStore import mongo
if self.onlineTable is None:
return
t1 = time.time()
# print('doBars1')
if hasattr(self, 't0_bars') and (t1 - self.t0_bars < 1):
return
self.t0_bars = t1
dt_today = dt.today().replace(hour=0,
minute=0,
second=0,
microsecond=0)
dt_today_loc = pd.Timestamp(dt_today).tz_localize('America/Sao_Paulo')
dt_max = dt.today().replace(year=dt_today.year + 1,
hour=0,
minute=0,
second=0,
microsecond=0)
if not 'lastBlpIndex' in self.onlineTable:
self.onlineTable['lastBlpIndex'] = dt_today_loc
for i in range(self.onlineTable.shape[0]):
a = Dict(self.onlineTable.iloc[i].to_dict())
if (not isinstance(a.INTRA_SHORT_NAME, str)) or (not isinstance(
a.freq, str)):
continue
nfreq, sfreq = mongo.splitFreq(a.freq) #assume freq is minutes
if (t1 - self.t0_intra[i] > a.updateInterval):
# st1 = dt.now() - datetime.timedelta(seconds=a.updateInterval*5) #np.maximum(a.startDate,dt_today)
# try:
# df_ = mds.read(a.security,a.freq,library=mds.mktBars,date_range=[st1,dt_max])
# df1 = self.blp.getIntradayHistoricData(a.security, nfreq, st1, dt_max, event=a.event,
# **a.kwargs)
# except Exception as e: #first query of the day - get all times
# df1 = self.blp.getIntradayHistoricData(a.security, nfreq, dt_today, dt_max, event=a.event,
# **a.kwargs)
# df1 = self.blp.getIntradayHistoricData(a.security, nfreq, dt_today, dt_max, event=a.event,**a.kwargs)
self.t0_intra[i] = t1
try:
md = mds.read_metadata(
a.INTRA_SHORT_NAME, '1Minute',
mds.assetTS2) #a.security.split(' ')[0]
md.freq = a.freq
except:
md = metadataAsset(a.INTRA_SHORT_NAME,
'equity',
freq=a.freq,
feeder_id=a.FEEDER_ID)
mds.blp = self.blp
#dt_today
df1 = self.blp.getIntradayHistoricDataBA(
a.FEEDER_ID,
nfreq,
self.onlineTable.lastBlpIndex[i],
dt_max,
md,
event=a.event,
mds=mds)
if df1.shape[0] == 0:
continue
self.onlineTable.lastBlpIndex.values[i] = df1.index[-1]
# **a.kwargs)
df1 = df1.rename(columns={'numEvents': 'trades'})
if df1.index.tzinfo is None:
df1 = df1.tz_localize('GMT')
print('doBars2 - ' + a.FEEDER_ID)
try:
mds.append(df1,
library=mds.onlineTS,
replaceIntersection=True,
check_metadata=False)
except Exception as e:
warn('Unable to append {}'.format(df1.md.name))
uu.printException(e)
# if len(a.addrs) :
# self.putData({'messageType':'barsUpdate','data':a},a.addrs)
if (t1 - self.t0_daily[i] > a.dailyUpdateInterval):
# for each series in intradayQueries, check if the daily series is in onlineVS up to yesterday
# If not, simply copy the series from assetVS to onlineVS. If it is not up-to-date, warn
#
self.t0_daily[i] = t1
dt_today1 = dt_today + datetime.timedelta(1)
dt0 = dt(1900, 1, 1)
if (df1.shape[0] == 0) or df1.index[-1] < dt_today_loc:
warn(
'No prices for {}/{} today ({}) in bars - (intraday/onlineTS)'
.format(a.INTRA_SHORT_NAME, nfreq, dt_today))
continue
try:
dfd = mds.read(a.daily_shortname,
'1BDay',
library=mds.assetVS,
date_range=[dt0, dt_today1])
except Exception as e:
print(
'Unable to read {}/{} from assetVS in bars - daily'
.format(a.security, nfreq))
uu.printException(e)
continue
# df1 = df1.loc[df1.index<dt_today_loc]
c1 = dfd.columns.intersection(df1.columns)
c2 = dfd.columns.difference(df1.columns)
dfi1 = df1[c1].iloc[-1]
lastUpdate = dfi1.name
dfi1.name = dfi1.name.normalize().tz_localize(None)
for c in c2:
dfi1[c] = array(nan)
# if md.subtype == 'fut_rol':
if 'underlying' in dfd:
if not 'underlying' in df1:
warn(
'Ignoring {}/{} for Daily. Underlying not present in bloomberg results'
.format(a.INTRA_SHORT_NAME, nfreq))
continue
dfi1['underlying'] = dfi1['underlying'].split(' ')[0]
if dfd.underlying[-1] != dfi1['underlying']:
continue
#check if it is the corerct future, if not continue
dfd_ = pd.DataFrame(dfi1).T
for c in dfd_.columns:
# if dfd[c].dtype in [float32,float64,int32,int64]:
dfd_[c] = dfd_[c].astype(dfd[c].dtype)
if dfd.md.subtype == 'di_fut':
dfd_['yield_close'] = dfd_['close']
dfd_['close'] = NaN
df2 = pd.concat((dfd, dfd_))
df2.md = dfd.md
df2.md.lastUpdate = lastUpdate
# if (not 't0_daily' in a): #first uptade in the day
try:
mds.delete(df2.md.name,
df2.md.freq,
library=mds.onlineVS
) #make sure not accumulating versions
except Exception as e:
pass
try:
mds.write(df2,
library=mds.onlineVS,
check_metadata=False,
prune_previous_version=True)
except Exception as e:
print(
'Unable to read {}/{} from assetVS in bars - daily'
.format(a.security, nfreq))
uu.printException(e)
def bloombergUpdateIntraday(meta, dti, bOnlyUpdateMetadata=False, srsMeta=None, overwrite=False,
dtm1=None, onlineTS='onlineTS',keep_metadata=False):
from mDataStore import mDataFrame, metadataFundamental, metadataOption, metadataAsset, metadataIndicator, \
metadataStrategy
meta1 = [m for m in meta if 'feeder_id_intraday' in m]
N=len(meta1)
if dtm1 is None:
import pytz
tz = pytz.timezone('GMT')
dtm1 = (dt.now()-datetime.timedelta(1)).replace(hour=0, minute=0, second=0, microsecond=0)
# success = np.full(hist1.shape[0], True)
lst_failures=[]
for k in tqdm(range(N)): #tqdm(
# nfreq, sfreq = mongo.splitFreq(meta1[k]['freq_intraday'])
interval = 1#nfreq
#
# if currency1[k] is None:
# # try:
# # currency = desc1.CRNCY[feeder_id[k]]
# # except:
# currency = 'BRL'
# else:
# currency = currency1[k]
if not 'options_intraday' in meta1[k] or not isinstance(meta1[k]['options_intraday'], str) or meta1[k]['options_intraday']=='':
options = {}
else:
options = eval('dict('+meta1[k]['options_intraday']+')')
# get output columns
md=copy(meta1[k])
md['freq']='1Minute'
md = metadataAsset(**md)
library1 = 'assetTS2'#md.pop('library')
if 'stDT' in md:
md.pop('stDT')
if 'endDT' in md:
md.pop('endDT')
mt_ant = mds.read_metadata(md['name'],'1Minute',library1)
if mt_ant and 'endDT' in mt_ant and dti > mt_ant['endDT']:
dti=pd.Timestamp(mt_ant['endDT']).normalize().to_pydatetime()
df = blp.getIntradayHistoricDataBA(md.feeder_id_intraday, interval, dti, dt.now(), md, **options)
if df.shape[0] == 0:
continue
# df.md.stDT = df.index[0]
# df.md.endDT = df.index[-1]
# if overwrite:
# for s in df.columns:
# if (df[s].dtype == np.float) and np.all(np.isnan(df[s])):
# del df[s]
#
# # lst=mds.find(df.md.name, df.md.freq, library=library1[k])
# att = df.md.name + '_' + df.md.freq
# exists = att in getattr(mds, library1).list_symbols()
# if exists:
# mds.delete(df.md.name, df.md.freq, library=library1)
#
# mds.write(df, library=library1, check_metadata=False)
# # mds.write(df, library=onlineTS, check_metadata=False)
#
# successi = True
#
# else:
try:
mds.append(df,date_range=[dti, dtm1], library=library1, check_metadata=False,keep_metadata=keep_metadata,
replaceIntersection=overwrite)
successi=True
except:
try:
mds.append(df, date_range=[dti, dtm1], library=library1, check_metadata=False, replaceIntersection=True,
keep_metadata=keep_metadata)
except:
successi=False
# mds.append(df, date_range=[dti, dtm1], library=onlineTS, check_metadata=False, replaceIntersection=True,keep_metadata=True)
if not successi:
lst_failures.append(df.md.name)
return lst_failures