def listen(self,sleepTime=0.00001): import time t0=time.time() i=0 self.stopListen=False try: while not self.stopListen: self.onListen() if sleepTime>0: time.sleep(sleepTime) # if DEBUG==True: # t1 = time.time() # if t1-t0>2: # v={'a':np.ones(M,)*i} # print('putData: {}'.format(v)) # self.putData(v) # i+=1 # t0=t1 events = self.sel.select(timeout=1) if events: for key, mask in events: self.service_connection(key, mask) # Check for a socket being monitored to continue. if not self.sel.get_map(): break except Exception as e: print('Client Exception') uu.printException(e)
def start_connection(self,server_addr): # server_addr = (host, port) # for i in range(0, num_conns): # connid = i + 1 print("starting connection to", server_addr) self.addr.append(server_addr) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setblocking(False) try: sock.connect_ex(server_addr) except Exception as e: print('Error Connection - Client') print('Unable to connect to {}',server_addr) uu.printException(e) return # if errno!= 0: # print('Unable to connect to {}. Errno: {}',server_addr,errno) # return events = selectors.EVENT_READ | selectors.EVENT_WRITE self.sock=sock self.sel.register(sock, events, data=(server_addr,sock)) self.rec.r[server_addr] = b'' self.rec.ri[server_addr] = b'' self.rec.q[server_addr] = queue.Queue()
def onBlpReceive(self, obj, topic, dti): #sending to everyone - try to separate if DEBUG: print('blp sending: {}'.format(obj)) # if not hasattr(self,'ii'): # self.ii=-1 # self.ii+=1 # obj1={} obj1 = { k: v for k, v in obj.items() if (isinstance(k, str) and isinstance(v, (dt, datetime.time, float, int, str))) } for k, v in obj1.items(): try: if isinstance(v, dt): v1 = dt(int(v.year), int(v.month), int(v.day), int(v.hour), int(v.minute), int(v.second)) elif isinstance(v, datetime.time): v1 = datetime.time(v.hour, v.minute, v.second) # v1=v.replace(tzinfo=None) else: v1 = v obj1[k] = v1 except Exception as e: print('Unable to include date: {}'.format(v)) uu.printException(e) obj2 = {k: v for k, v in obj.items() if not k in obj1} if len(obj2) > 0: print( '########################################################################################################################' ) print('Not Builtin: {}'.format(obj2)) print( '########################################################################################################################' ) self.feeder.putData((topic, dti, obj1))
def listen(self,sleepTime=0.00001): import time t0=time.time() i=0 self.stopListen=False try: while not self.stopListen: self.onListen() if sleepTime>0: time.sleep(sleepTime) # if DEBUG==True: # t1 = time.time() # if t1-t0>2: # v={'a':np.ones(M,)*i} # print('putData: {}'.format(v)) # self.putData(v) # i+=1 # t0=t1 events = self.sel.select(timeout=None) for key, mask in events: if key.data is None: self.accept_wrapper(key.fileobj) else: self.service_connection(key, mask) except Exception as e: print('Exception. Closing Socket') uu.printException(e) try: self.sel.unregister(self.sock) except Exception as e: print( f"error: selector.unregister() exception for", f"{self.host}:{self.port} ", ) uu.printException(e) try: self.sock.close() except OSError as e: print( f"error: socket.close() exception for", f"{self.host}:{self.port} ", ) uu.printException(e) finally: # Delete reference to socket object for garbage collection self.sock = None
def service_connection(self, key, mask): sock = key.fileobj a,conn = key.data if not a in self.rec.r: self.rec.r[a] = b'' self.rec.ri[a] = b'' self.rec.q[a] = queue.Queue() if mask & selectors.EVENT_READ: if len(self.rec.ri[a]) == 8: N = int.from_bytes(self.rec.ri[a], 'big') else: nHead = 8 - len(self.rec.ri[a]) try: recv_data = sock.recv(nHead) if not recv_data: print("closing connection", a) self.sel.unregister(sock) sock.close() except Exception as e: print('Error while reading socket 1') uu.printException(e) if a in self.addr: self.addr.remove(a) else: print('Closed socket was not in addr list') self.endConnection(a,conn) return self.rec.ri[a] += recv_data # Should be ready to read if DEBUG: if len(recv_data) > 0: print('recv {} - ri {}'.format(len(recv_data), len(self.rec.ri[a]))) if len(self.rec.ri[a]) == 8: N = int.from_bytes(self.rec.ri[a], 'big') self.N=N if len(self.rec.ri[a]) == 8: try: recv_data1 = sock.recv(N) if not recv_data1: print("closing connection", a) self.sel.unregister(sock) sock.close() except Exception as e: print('Error while reading socket 2') uu.printException(e) if a in self.addr: self.addr.remove(a) else: print('Closed socket was not') self.endConnection(a,conn) return self.rec.r[a] += recv_data1 # Should be ready to read if DEBUG: print('recv1 {} N {}'.format(len(recv_data1), N)) if len(self.rec.r[a]) == N: self.rec.ri[a] = b'' obj = pickle.loads(self.rec.r[a]) if DEBUG: print('obj: {}'.format(obj)) # self.rec.q[a].put(obj) self.rec.r[a] = b'' self.onReceive(a,obj) # print("received", repr(recv_data), "from connection", a) if mask & selectors.EVENT_WRITE: if a in self.send.q and self.send.p[a]: try: sent = sock.send(self.send.p[a]) # Should be ready to write except Exception as e: print('Error while writing socket 1') uu.printException(e) if a in self.addr: self.addr.remove(a) else: print('Closed socket was not') self.endConnection(a,conn) return self.send.p[a] = self.send.p[a][sent:] elif a in self.send.q and not self.send.q[a].empty(): data1 = self.send.q[a].get() pdata1 = pickle.dumps(data1) try: sent = sock.send(len(pdata1).to_bytes(8, 'big') + pdata1) # Should be ready to write except Exception as e: print('Error while writing socket 1') uu.printException(e) if a in self.addr: self.addr.remove(a) else: print('Closed socket was not') self.endConnection(a,conn) return self.send.p[a] += pdata1[sent:]
def doBars(self): #for intraday bars from mDataStore.mongo import mDataFrame, metadataAsset from mDataStore import mongo if self.onlineTable is None: return t1 = time.time() # print('doBars1') if hasattr(self, 't0_bars') and (t1 - self.t0_bars < 1): return self.t0_bars = t1 dt_today = dt.today().replace(hour=0, minute=0, second=0, microsecond=0) dt_today_loc = pd.Timestamp(dt_today).tz_localize('America/Sao_Paulo') dt_max = dt.today().replace(year=dt_today.year + 1, hour=0, minute=0, second=0, microsecond=0) if not 'lastBlpIndex' in self.onlineTable: self.onlineTable['lastBlpIndex'] = dt_today_loc for i in range(self.onlineTable.shape[0]): a = Dict(self.onlineTable.iloc[i].to_dict()) if (not isinstance(a.INTRA_SHORT_NAME, str)) or (not isinstance( a.freq, str)): continue nfreq, sfreq = mongo.splitFreq(a.freq) #assume freq is minutes if (t1 - self.t0_intra[i] > a.updateInterval): # st1 = dt.now() - datetime.timedelta(seconds=a.updateInterval*5) #np.maximum(a.startDate,dt_today) # try: # df_ = mds.read(a.security,a.freq,library=mds.mktBars,date_range=[st1,dt_max]) # df1 = self.blp.getIntradayHistoricData(a.security, nfreq, st1, dt_max, event=a.event, # **a.kwargs) # except Exception as e: #first query of the day - get all times # df1 = self.blp.getIntradayHistoricData(a.security, nfreq, dt_today, dt_max, event=a.event, # **a.kwargs) # df1 = self.blp.getIntradayHistoricData(a.security, nfreq, dt_today, dt_max, event=a.event,**a.kwargs) self.t0_intra[i] = t1 try: md = mds.read_metadata( a.INTRA_SHORT_NAME, '1Minute', mds.assetTS2) #a.security.split(' ')[0] md.freq = a.freq except: md = metadataAsset(a.INTRA_SHORT_NAME, 'equity', freq=a.freq, feeder_id=a.FEEDER_ID) mds.blp = self.blp #dt_today df1 = self.blp.getIntradayHistoricDataBA( a.FEEDER_ID, nfreq, self.onlineTable.lastBlpIndex[i], dt_max, md, event=a.event, mds=mds) if df1.shape[0] == 0: continue self.onlineTable.lastBlpIndex.values[i] = df1.index[-1] # **a.kwargs) df1 = df1.rename(columns={'numEvents': 'trades'}) if df1.index.tzinfo is None: df1 = df1.tz_localize('GMT') print('doBars2 - ' + a.FEEDER_ID) try: mds.append(df1, library=mds.onlineTS, replaceIntersection=True, check_metadata=False) except Exception as e: warn('Unable to append {}'.format(df1.md.name)) uu.printException(e) # if len(a.addrs) : # self.putData({'messageType':'barsUpdate','data':a},a.addrs) if (t1 - self.t0_daily[i] > a.dailyUpdateInterval): # for each series in intradayQueries, check if the daily series is in onlineVS up to yesterday # If not, simply copy the series from assetVS to onlineVS. If it is not up-to-date, warn # self.t0_daily[i] = t1 dt_today1 = dt_today + datetime.timedelta(1) dt0 = dt(1900, 1, 1) if (df1.shape[0] == 0) or df1.index[-1] < dt_today_loc: warn( 'No prices for {}/{} today ({}) in bars - (intraday/onlineTS)' .format(a.INTRA_SHORT_NAME, nfreq, dt_today)) continue try: dfd = mds.read(a.daily_shortname, '1BDay', library=mds.assetVS, date_range=[dt0, dt_today1]) except Exception as e: print( 'Unable to read {}/{} from assetVS in bars - daily' .format(a.security, nfreq)) uu.printException(e) continue # df1 = df1.loc[df1.index<dt_today_loc] c1 = dfd.columns.intersection(df1.columns) c2 = dfd.columns.difference(df1.columns) dfi1 = df1[c1].iloc[-1] lastUpdate = dfi1.name dfi1.name = dfi1.name.normalize().tz_localize(None) for c in c2: dfi1[c] = array(nan) # if md.subtype == 'fut_rol': if 'underlying' in dfd: if not 'underlying' in df1: warn( 'Ignoring {}/{} for Daily. Underlying not present in bloomberg results' .format(a.INTRA_SHORT_NAME, nfreq)) continue dfi1['underlying'] = dfi1['underlying'].split(' ')[0] if dfd.underlying[-1] != dfi1['underlying']: continue #check if it is the corerct future, if not continue dfd_ = pd.DataFrame(dfi1).T for c in dfd_.columns: # if dfd[c].dtype in [float32,float64,int32,int64]: dfd_[c] = dfd_[c].astype(dfd[c].dtype) if dfd.md.subtype == 'di_fut': dfd_['yield_close'] = dfd_['close'] dfd_['close'] = NaN df2 = pd.concat((dfd, dfd_)) df2.md = dfd.md df2.md.lastUpdate = lastUpdate # if (not 't0_daily' in a): #first uptade in the day try: mds.delete(df2.md.name, df2.md.freq, library=mds.onlineVS ) #make sure not accumulating versions except Exception as e: pass try: mds.write(df2, library=mds.onlineVS, check_metadata=False, prune_previous_version=True) except Exception as e: print( 'Unable to read {}/{} from assetVS in bars - daily' .format(a.security, nfreq)) uu.printException(e)