def listen(self,sleepTime=0.00001):
import time
t0=time.time()
i=0
self.stopListen=False
try:
while not self.stopListen:
self.onListen()
if sleepTime>0:
time.sleep(sleepTime)
# if DEBUG==True:
# t1 = time.time()
# if t1-t0>2:
# v={'a':np.ones(M,)*i}
# print('putData: {}'.format(v))
# self.putData(v)
# i+=1
# t0=t1
events = self.sel.select(timeout=1)
if events:
for key, mask in events:
self.service_connection(key, mask)
# Check for a socket being monitored to continue.
if not self.sel.get_map():
break
except Exception as e:
print('Client Exception')
uu.printException(e)
def start_connection(self,server_addr):
# server_addr = (host, port)
# for i in range(0, num_conns):
# connid = i + 1
print("starting connection to", server_addr)
self.addr.append(server_addr)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setblocking(False)
try:
sock.connect_ex(server_addr)
except Exception as e:
print('Error Connection - Client')
print('Unable to connect to {}',server_addr)
uu.printException(e)
return
# if errno!= 0:
# print('Unable to connect to {}. Errno: {}',server_addr,errno)
# return
events = selectors.EVENT_READ | selectors.EVENT_WRITE
self.sock=sock
self.sel.register(sock, events, data=(server_addr,sock))
self.rec.r[server_addr] = b''
self.rec.ri[server_addr] = b''
self.rec.q[server_addr] = queue.Queue()
def onBlpReceive(self, obj, topic, dti):
#sending to everyone - try to separate
if DEBUG:
print('blp sending: {}'.format(obj))
# if not hasattr(self,'ii'):
# self.ii=-1
# self.ii+=1
# obj1={}
obj1 = {
k: v
for k, v in obj.items()
if (isinstance(k, str) and isinstance(v, (dt, datetime.time, float,
int, str)))
}
for k, v in obj1.items():
try:
if isinstance(v, dt):
v1 = dt(int(v.year), int(v.month), int(v.day), int(v.hour),
int(v.minute), int(v.second))
elif isinstance(v, datetime.time):
v1 = datetime.time(v.hour, v.minute, v.second)
# v1=v.replace(tzinfo=None)
else:
v1 = v
obj1[k] = v1
except Exception as e:
print('Unable to include date: {}'.format(v))
uu.printException(e)
obj2 = {k: v for k, v in obj.items() if not k in obj1}
if len(obj2) > 0:
print(
'########################################################################################################################'
)
print('Not Builtin: {}'.format(obj2))
print(
'########################################################################################################################'
)
self.feeder.putData((topic, dti, obj1))
def listen(self,sleepTime=0.00001):
import time
t0=time.time()
i=0
self.stopListen=False
try:
while not self.stopListen:
self.onListen()
if sleepTime>0:
time.sleep(sleepTime)
# if DEBUG==True:
# t1 = time.time()
# if t1-t0>2:
# v={'a':np.ones(M,)*i}
# print('putData: {}'.format(v))
# self.putData(v)
# i+=1
# t0=t1
events = self.sel.select(timeout=None)
for key, mask in events:
if key.data is None:
self.accept_wrapper(key.fileobj)
else:
self.service_connection(key, mask)
except Exception as e:
print('Exception. Closing Socket')
uu.printException(e)
try:
self.sel.unregister(self.sock)
except Exception as e:
print(
f"error: selector.unregister() exception for",
f"{self.host}:{self.port} ",
)
uu.printException(e)
try:
self.sock.close()
except OSError as e:
print(
f"error: socket.close() exception for",
f"{self.host}:{self.port} ",
)
uu.printException(e)
finally:
# Delete reference to socket object for garbage collection
self.sock = None
def service_connection(self, key, mask):
sock = key.fileobj
a,conn = key.data
if not a in self.rec.r:
self.rec.r[a] = b''
self.rec.ri[a] = b''
self.rec.q[a] = queue.Queue()
if mask & selectors.EVENT_READ:
if len(self.rec.ri[a]) == 8:
N = int.from_bytes(self.rec.ri[a], 'big')
else:
nHead = 8 - len(self.rec.ri[a])
try:
recv_data = sock.recv(nHead)
if not recv_data:
print("closing connection", a)
self.sel.unregister(sock)
sock.close()
except Exception as e:
print('Error while reading socket 1')
uu.printException(e)
if a in self.addr:
self.addr.remove(a)
else:
print('Closed socket was not in addr list')
self.endConnection(a,conn)
return
self.rec.ri[a] += recv_data # Should be ready to read
if DEBUG:
if len(recv_data) > 0:
print('recv {} - ri {}'.format(len(recv_data), len(self.rec.ri[a])))
if len(self.rec.ri[a]) == 8:
N = int.from_bytes(self.rec.ri[a], 'big')
self.N=N
if len(self.rec.ri[a]) == 8:
try:
recv_data1 = sock.recv(N)
if not recv_data1:
print("closing connection", a)
self.sel.unregister(sock)
sock.close()
except Exception as e:
print('Error while reading socket 2')
uu.printException(e)
if a in self.addr:
self.addr.remove(a)
else:
print('Closed socket was not')
self.endConnection(a,conn)
return
self.rec.r[a] += recv_data1 # Should be ready to read
if DEBUG:
print('recv1 {} N {}'.format(len(recv_data1), N))
if len(self.rec.r[a]) == N:
self.rec.ri[a] = b''
obj = pickle.loads(self.rec.r[a])
if DEBUG:
print('obj: {}'.format(obj))
# self.rec.q[a].put(obj)
self.rec.r[a] = b''
self.onReceive(a,obj)
# print("received", repr(recv_data), "from connection", a)
if mask & selectors.EVENT_WRITE:
if a in self.send.q and self.send.p[a]:
try:
sent = sock.send(self.send.p[a]) # Should be ready to write
except Exception as e:
print('Error while writing socket 1')
uu.printException(e)
if a in self.addr:
self.addr.remove(a)
else:
print('Closed socket was not')
self.endConnection(a,conn)
return
self.send.p[a] = self.send.p[a][sent:]
elif a in self.send.q and not self.send.q[a].empty():
data1 = self.send.q[a].get()
pdata1 = pickle.dumps(data1)
try:
sent = sock.send(len(pdata1).to_bytes(8, 'big') + pdata1) # Should be ready to write
except Exception as e:
print('Error while writing socket 1')
uu.printException(e)
if a in self.addr:
self.addr.remove(a)
else:
print('Closed socket was not')
self.endConnection(a,conn)
return
self.send.p[a] += pdata1[sent:]
def doBars(self): #for intraday bars
from mDataStore.mongo import mDataFrame, metadataAsset
from mDataStore import mongo
if self.onlineTable is None:
return
t1 = time.time()
# print('doBars1')
if hasattr(self, 't0_bars') and (t1 - self.t0_bars < 1):
return
self.t0_bars = t1
dt_today = dt.today().replace(hour=0,
minute=0,
second=0,
microsecond=0)
dt_today_loc = pd.Timestamp(dt_today).tz_localize('America/Sao_Paulo')
dt_max = dt.today().replace(year=dt_today.year + 1,
hour=0,
minute=0,
second=0,
microsecond=0)
if not 'lastBlpIndex' in self.onlineTable:
self.onlineTable['lastBlpIndex'] = dt_today_loc
for i in range(self.onlineTable.shape[0]):
a = Dict(self.onlineTable.iloc[i].to_dict())
if (not isinstance(a.INTRA_SHORT_NAME, str)) or (not isinstance(
a.freq, str)):
continue
nfreq, sfreq = mongo.splitFreq(a.freq) #assume freq is minutes
if (t1 - self.t0_intra[i] > a.updateInterval):
# st1 = dt.now() - datetime.timedelta(seconds=a.updateInterval*5) #np.maximum(a.startDate,dt_today)
# try:
# df_ = mds.read(a.security,a.freq,library=mds.mktBars,date_range=[st1,dt_max])
# df1 = self.blp.getIntradayHistoricData(a.security, nfreq, st1, dt_max, event=a.event,
# **a.kwargs)
# except Exception as e: #first query of the day - get all times
# df1 = self.blp.getIntradayHistoricData(a.security, nfreq, dt_today, dt_max, event=a.event,
# **a.kwargs)
# df1 = self.blp.getIntradayHistoricData(a.security, nfreq, dt_today, dt_max, event=a.event,**a.kwargs)
self.t0_intra[i] = t1
try:
md = mds.read_metadata(
a.INTRA_SHORT_NAME, '1Minute',
mds.assetTS2) #a.security.split(' ')[0]
md.freq = a.freq
except:
md = metadataAsset(a.INTRA_SHORT_NAME,
'equity',
freq=a.freq,
feeder_id=a.FEEDER_ID)
mds.blp = self.blp
#dt_today
df1 = self.blp.getIntradayHistoricDataBA(
a.FEEDER_ID,
nfreq,
self.onlineTable.lastBlpIndex[i],
dt_max,
md,
event=a.event,
mds=mds)
if df1.shape[0] == 0:
continue
self.onlineTable.lastBlpIndex.values[i] = df1.index[-1]
# **a.kwargs)
df1 = df1.rename(columns={'numEvents': 'trades'})
if df1.index.tzinfo is None:
df1 = df1.tz_localize('GMT')
print('doBars2 - ' + a.FEEDER_ID)
try:
mds.append(df1,
library=mds.onlineTS,
replaceIntersection=True,
check_metadata=False)
except Exception as e:
warn('Unable to append {}'.format(df1.md.name))
uu.printException(e)
# if len(a.addrs) :
# self.putData({'messageType':'barsUpdate','data':a},a.addrs)
if (t1 - self.t0_daily[i] > a.dailyUpdateInterval):
# for each series in intradayQueries, check if the daily series is in onlineVS up to yesterday
# If not, simply copy the series from assetVS to onlineVS. If it is not up-to-date, warn
#
self.t0_daily[i] = t1
dt_today1 = dt_today + datetime.timedelta(1)
dt0 = dt(1900, 1, 1)
if (df1.shape[0] == 0) or df1.index[-1] < dt_today_loc:
warn(
'No prices for {}/{} today ({}) in bars - (intraday/onlineTS)'
.format(a.INTRA_SHORT_NAME, nfreq, dt_today))
continue
try:
dfd = mds.read(a.daily_shortname,
'1BDay',
library=mds.assetVS,
date_range=[dt0, dt_today1])
except Exception as e:
print(
'Unable to read {}/{} from assetVS in bars - daily'
.format(a.security, nfreq))
uu.printException(e)
continue
# df1 = df1.loc[df1.index<dt_today_loc]
c1 = dfd.columns.intersection(df1.columns)
c2 = dfd.columns.difference(df1.columns)
dfi1 = df1[c1].iloc[-1]
lastUpdate = dfi1.name
dfi1.name = dfi1.name.normalize().tz_localize(None)
for c in c2:
dfi1[c] = array(nan)
# if md.subtype == 'fut_rol':
if 'underlying' in dfd:
if not 'underlying' in df1:
warn(
'Ignoring {}/{} for Daily. Underlying not present in bloomberg results'
.format(a.INTRA_SHORT_NAME, nfreq))
continue
dfi1['underlying'] = dfi1['underlying'].split(' ')[0]
if dfd.underlying[-1] != dfi1['underlying']:
continue
#check if it is the corerct future, if not continue
dfd_ = pd.DataFrame(dfi1).T
for c in dfd_.columns:
# if dfd[c].dtype in [float32,float64,int32,int64]:
dfd_[c] = dfd_[c].astype(dfd[c].dtype)
if dfd.md.subtype == 'di_fut':
dfd_['yield_close'] = dfd_['close']
dfd_['close'] = NaN
df2 = pd.concat((dfd, dfd_))
df2.md = dfd.md
df2.md.lastUpdate = lastUpdate
# if (not 't0_daily' in a): #first uptade in the day
try:
mds.delete(df2.md.name,
df2.md.freq,
library=mds.onlineVS
) #make sure not accumulating versions
except Exception as e:
pass
try:
mds.write(df2,
library=mds.onlineVS,
check_metadata=False,
prune_previous_version=True)
except Exception as e:
print(
'Unable to read {}/{} from assetVS in bars - daily'
.format(a.security, nfreq))
uu.printException(e)