def __init__(self, name, traits):
UsesPort.__init__(self, name, traits.PortType)
# Type metadata
self.BurstType = traits.BurstType
self._bytesPerElement = traits.size()
# Logging; default logger is the class name
self._log = logging.getLogger(self.__class__.__name__)
# Perform latency monitoring and deferred pushes on a separate thread
self._monitor = ExecutorService()
# Queue management
self._queueMutex = threading.Lock()
self._defaultQueue = Queue(self, self.DEFAULT_MAX_BURSTS,
0.9 * 2 * (1024**2),
self.DEFAULT_LATENCY_THRESHOLD)
self._streamQueues = {}
# Default to all connections receiving all streams interleaved
self._routingMode = ROUTE_ALL_INTERLEAVED
self._routes = {}
def __init__(self, name, traits):
UsesPort.__init__(self, name, traits.PortType)
# Type metadata
self.BurstType = traits.BurstType
self._bytesPerElement = traits.size()
# Logging; default logger is the class name
self._log = logging.getLogger(self.__class__.__name__)
# Perform latency monitoring and deferred pushes on a separate thread
self._monitor = ExecutorService()
# Queue management
self._queueMutex = threading.Lock()
self._defaultQueue = Queue(self, self.DEFAULT_MAX_BURSTS, 0.9*2*(1024**2), self.DEFAULT_LATENCY_THRESHOLD)
self._streamQueues = {}
# Default to all connections receiving all streams interleaved
self._routingMode = ROUTE_ALL_INTERLEAVED
self._routes = {}
class OutPort(UsesPort, BULKIO__POA.UsesPortStatisticsProvider):
DEFAULT_MAX_BURSTS = 100
DEFAULT_LATENCY_THRESHOLD = 10000 # 10000us = 10ms
def __init__(self, name, traits):
UsesPort.__init__(self, name, traits.PortType)
# Type metadata
self.BurstType = traits.BurstType
self._bytesPerElement = traits.size()
# Logging; default logger is the class name
self._log = logging.getLogger(self.__class__.__name__)
# Perform latency monitoring and deferred pushes on a separate thread
self._monitor = ExecutorService()
# Queue management
self._queueMutex = threading.Lock()
self._defaultQueue = Queue(self, self.DEFAULT_MAX_BURSTS, 0.9*2*(1024**2), self.DEFAULT_LATENCY_THRESHOLD)
self._streamQueues = {}
# Default to all connections receiving all streams interleaved
self._routingMode = ROUTE_ALL_INTERLEAVED
self._routes = {}
def start(self):
self._monitor.start()
def stop(self):
# Stop the monitor thread and remove any queued checks, as the queue(s)
# will be flushed anyway
self._monitor.stop()
self._monitor.clear()
self.flush()
# Provide standard interface for start/stop
startPort = start
stopPort = stop
def getMaxBursts(self):
return self.getDefaultPolicy().getMaxBursts()
def setMaxBursts(self, bursts):
self.getDefaultPolicy().setMaxBursts(bursts)
def getByteThreshold(self):
return self.getDefaultPolicy().getByteThreshold()
def setByteThreshold(self, bytes):
self.getDefaultPolicy().setByteThreshold(bytes)
def getLatencyThreshold(self):
return self.getDefaultPolicy().getLatencyThreshold()
def setLatencyThreshold(self, usec):
self.getDefaultPolicy().setLatencyThreshold(usec)
def getDefaultPolicy(self):
return self._defaultQueue
def getStreamPolicy(self, streamID):
self._queueMutex.acquire()
try:
return self._getQueueForStream(streamID)
finally:
self._queueMutex.release()
def setRoutingMode(self, mode):
self._routingMode = mode
def setLogger(self, logger):
self._log = logger
def updateConnectionFilter(self, filterTable):
new_routes = {}
for route in filterTable:
if route.port_name != self._name:
continue
if not route.stream_id in self._routes:
new_routes[route.stream_id] = set()
new_routes[route.stream_id].add(route.connection_id)
self._connectionMutex.acquire()
try:
self._routes = new_routes
finally:
self._connectionMutex.release()
def addConnectionFilter(self, streamID, connectionID):
self._connectionMutex.acquire()
try:
if not streamID in self._routes:
self._routes[streamID] = set()
self._routes[streamID].add(connectionID)
finally:
self._connectionMutex.release()
def removeConnectionFilter(self, streamID, connectionID):
self._connectionMutex.acquire()
try:
if streamID in self._routes:
self._routes[streamID].discard(connectionID)
finally:
self._connectionMutex.release()
def state(self):
self._connectionMutex.acquire()
try:
if self._connections:
return BULKIO.ACTIVE
else:
return BULKIO.IDLE
finally:
self._connectionMutex.release()
def pushBurst(self, data, sri, timestamp=None, eos=False):
# Ensure data is in the correct format
data = self._formatData(data)
if timestamp is None:
timestamp = utils.now()
burst = self.BurstType(sri, data, timestamp, eos)
self._queueMutex.acquire()
try:
queue = self._getQueueForStream(sri.streamID)
queue._queueBurst(burst)
if eos:
if not self._isInterleaved():
self._log.debug("Flushing '%s' on EOS", sri.streamID)
queue.flush()
del self._streamQueues[sri.streamID]
finally:
self._queueMutex.release()
def pushBursts(self, bursts):
self._sendBursts(bursts, time.time(), 0.0)
def flush(self):
self._queueMutex.acquire()
try:
for queue in self._getAllQueues():
queue.flush()
finally:
self._queueMutex.release()
def _sendBursts(self, bursts, startTime, queueDepth, streamID=None):
self._log.debug('Pushing %d bursts', len(bursts))
# Count up total elements
total_elements = sum(len(burst.data) for burst in bursts)
self._connectionMutex.acquire()
try:
for connectionId, connection in self._connections.iteritems():
# Check stream routing
if not self._isStreamRoutedToConnection(streamID, connectionId):
continue
# Record statistics
delay = time.time() - startTime
try:
connection.port.pushBursts(bursts)
connection.alive = True
connection.stats.record(len(bursts), total_elements, queueDepth, delay)
except CORBA.MARSHAL, e:
if len(bursts) == 1:
if connection.alive:
self._log.error('pushBursts to %s failed because the burst size is too long')
connection.alive = False
else:
self._partitionBursts(bursts, startTime, queueDepth, connection)
except Exception, e:
if connection.alive:
self._log.error('pushBursts to %s failed: %s', connectionId, e)
connection.alive = False
except:
class OutPort(UsesPort, BULKIO__POA.UsesPortStatisticsProvider):
DEFAULT_MAX_BURSTS = 100
DEFAULT_LATENCY_THRESHOLD = 10000 # 10000us = 10ms
def __init__(self, name, traits):
UsesPort.__init__(self, name, traits.PortType)
# Type metadata
self.BurstType = traits.BurstType
self._bytesPerElement = traits.size()
# Logging; default logger is the class name
self._log = logging.getLogger(self.__class__.__name__)
# Perform latency monitoring and deferred pushes on a separate thread
self._monitor = ExecutorService()
# Queue management
self._queueMutex = threading.Lock()
self._defaultQueue = Queue(self, self.DEFAULT_MAX_BURSTS,
0.9 * 2 * (1024**2),
self.DEFAULT_LATENCY_THRESHOLD)
self._streamQueues = {}
# Default to all connections receiving all streams interleaved
self._routingMode = ROUTE_ALL_INTERLEAVED
self._routes = {}
def start(self):
self._monitor.start()
def stop(self):
# Stop the monitor thread and remove any queued checks, as the queue(s)
# will be flushed anyway
self._monitor.stop()
self._monitor.clear()
self.flush()
# Provide standard interface for start/stop
startPort = start
stopPort = stop
def getMaxBursts(self):
return self.getDefaultPolicy().getMaxBursts()
def setMaxBursts(self, bursts):
self.getDefaultPolicy().setMaxBursts(bursts)
def getByteThreshold(self):
return self.getDefaultPolicy().getByteThreshold()
def setByteThreshold(self, bytes):
self.getDefaultPolicy().setByteThreshold(bytes)
def getLatencyThreshold(self):
return self.getDefaultPolicy().getLatencyThreshold()
def setLatencyThreshold(self, usec):
self.getDefaultPolicy().setLatencyThreshold(usec)
def getDefaultPolicy(self):
return self._defaultQueue
def getStreamPolicy(self, streamID):
self._queueMutex.acquire()
try:
return self._getQueueForStream(streamID)
finally:
self._queueMutex.release()
def setRoutingMode(self, mode):
self._routingMode = mode
def setLogger(self, logger):
self._log = logger
def updateConnectionFilter(self, filterTable):
new_routes = {}
for route in filterTable:
if route.port_name != self._name:
continue
if not route.stream_id in self._routes:
new_routes[route.stream_id] = set()
new_routes[route.stream_id].add(route.connection_id)
self._connectionMutex.acquire()
try:
self._routes = new_routes
finally:
self._connectionMutex.release()
def addConnectionFilter(self, streamID, connectionID):
self._connectionMutex.acquire()
try:
if not streamID in self._routes:
self._routes[streamID] = set()
self._routes[streamID].add(connectionID)
finally:
self._connectionMutex.release()
def removeConnectionFilter(self, streamID, connectionID):
self._connectionMutex.acquire()
try:
if streamID in self._routes:
self._routes[streamID].discard(connectionID)
finally:
self._connectionMutex.release()
def state(self):
self._connectionMutex.acquire()
try:
if self._connections:
return BULKIO.ACTIVE
else:
return BULKIO.IDLE
finally:
self._connectionMutex.release()
def pushBurst(self, data, sri, timestamp=None, eos=False):
# Ensure data is in the correct format
data = self._formatData(data)
if timestamp is None:
timestamp = utils.now()
burst = self.BurstType(sri, data, timestamp, eos)
self._queueMutex.acquire()
try:
queue = self._getQueueForStream(sri.streamID)
queue._queueBurst(burst)
if eos:
if not self._isInterleaved():
self._log.debug("Flushing '%s' on EOS", sri.streamID)
queue.flush()
del self._streamQueues[sri.streamID]
finally:
self._queueMutex.release()
def pushBursts(self, bursts):
self._sendBursts(bursts, time.time(), 0.0)
def flush(self):
self._queueMutex.acquire()
try:
for queue in self._getAllQueues():
queue.flush()
finally:
self._queueMutex.release()
def _sendBursts(self, bursts, startTime, queueDepth, streamID=None):
self._log.debug('Pushing %d bursts', len(bursts))
# Count up total elements
total_elements = sum(len(burst.data) for burst in bursts)
self._connectionMutex.acquire()
try:
for connectionId, connection in self._connections.iteritems():
# Check stream routing
if not self._isStreamRoutedToConnection(
streamID, connectionId):
continue
# Record statistics
delay = time.time() - startTime
try:
connection.port.pushBursts(bursts)
connection.alive = True
connection.stats.record(len(bursts), total_elements,
queueDepth, delay)
except CORBA.MARSHAL, e:
if len(bursts) == 1:
if connection.alive:
self._log.error(
'pushBursts to %s failed because the burst size is too long'
)
connection.alive = False
else:
self._partitionBursts(bursts, startTime, queueDepth,
connection)
except Exception, e:
if connection.alive:
self._log.error('pushBursts to %s failed: %s',
connectionId, e)
connection.alive = False
except:
