def __init__(self,
dispatchtable,
logtimes=False,
encryption_required=True):
'Constructor for MessageDispatcher - requires a dispatch table as a parameter'
self.dispatchtable = dispatchtable
self.default = DispatchTarget()
self.io = None
self.dispatchcount = 0
self.logtimes = logtimes or CMAdb.debug
self.encryption_required = encryption_required
def __init__(self, dispatchtable, logtimes=False, encryption_required=True):
'Constructor for MessageDispatcher - requires a dispatch table as a parameter'
self.dispatchtable = dispatchtable
self.default = DispatchTarget()
self.io = None
self.dispatchcount = 0
self.logtimes = logtimes or CMAdb.debug
self.encryption_required = encryption_required
class MessageDispatcher(object):
'We dispatch incoming messages where they need to go.'
def __init__(self, dispatchtable, logtimes=False, encryption_required=True):
'Constructor for MessageDispatcher - requires a dispatch table as a parameter'
self.dispatchtable = dispatchtable
self.default = DispatchTarget()
self.io = None
self.dispatchcount = 0
self.logtimes = logtimes or CMAdb.debug
self.encryption_required = encryption_required
def dispatch(self, origaddr, frameset):
'Dispatch a Frameset where it will get handled.'
self.dispatchcount += 1
CMAdb.transaction = Transaction(encryption_required=self.encryption_required)
# W0703 == Too general exception catching...
# pylint: disable=W0703
try:
self._try_dispatch_action(origaddr, frameset)
if (self.dispatchcount % 100) == 1:
self._check_memory_usage()
except Exception as e:
self._process_exception(e, origaddr, frameset)
# We want to ack the packet even in the failed case - retries are unlikely to help
# and we need to avoid getting stuck in a loop retrying it forever...
if CMAdb.debug:
fstypename = FrameSetTypes.get(frameset.get_framesettype())[0]
CMAdb.log.debug('MessageDispatcher - ACKing %s message from %s'
% (fstypename, origaddr))
self.io.ackmessage(origaddr, frameset)
# [R0912:MessageDispatcher._try_dispatch_action] Too many branches (13/12)
# pylint: disable=R0912
def _try_dispatch_action(self, origaddr, frameset):
'''Core code to actually dispatch the Frameset.
It should be run inside a try/except construct so that anything
we barf up won't cause the CMA to die.
'''
fstype = frameset.get_framesettype()
#print >>sys.stderr, 'Got frameset of type %s [%s]' % (fstype, frameset)
dispatchstart = datetime.now()
if fstype in self.dispatchtable:
self.dispatchtable[fstype].dispatch(origaddr, frameset)
else:
self.default.dispatch(origaddr, frameset)
dispatchend = datetime.now()
if self.logtimes:
CMAdb.log.info('Initial dispatch time for %s frameset: %s'
% (fstype, dispatchend-dispatchstart))
# Commit the network transaction here
CMAdb.transaction.commit_trans(CMAdb.io)
if self.logtimes:
CMAdb.log.info('Network transaction time: %s'
% (str(CMAdb.transaction.stats['lastcommit'])))
# Commit the database transaction here
if CMAdb.store.transaction_pending:
result = CMAdb.store.commit()
if self.logtimes or CMAdb.debug:
CMAdb.log.info('Neo4j transaction time: %s'
% (str(CMAdb.store.stats['lastcommit'])))
if CMAdb.debug:
resultlines = str(result).splitlines()
CMAdb.log.debug('Commit results follow:')
for line in resultlines:
CMAdb.log.debug(line.expandtabs())
CMAdb.log.debug('end of commit results.')
# This is a VERY expensive call...
# Good thing we only do it when debug is enabled...
CMAdb.TheOneRing.AUDIT()
else:
if CMAdb.debug:
CMAdb.log.debug('No database changes this time')
CMAdb.store.abort()
for pkttype in CMAdb.transaction.post_transaction_packets:
CMAdb.transaction.add_packet(origaddr, pkttype, [])
if len(CMAdb.transaction.post_transaction_packets) > 0:
CMAdb.transaction.commit_trans(CMAdb.io)
CMAdb.post_transaction_packets = []
dispatchend = datetime.now()
if self.logtimes or CMAdb.debug:
CMAdb.log.info('Total dispatch time for %s frameset: %s'
% (fstype, dispatchend-dispatchstart))
@staticmethod
def _process_exception(e, origaddr, frameset):
'Handle an exception from our message dispatcher'
# Darn! Got an exception - let's try and put everything useful into the
# logs in a legible way
trace = sys.exc_info()[2]
# we ignore the etype and evalue returns from sys.exc_info
tblist = traceback.extract_tb(trace, 20)
fstype = frameset.get_framesettype()
fstypename = FrameSetTypes.get(fstype)[0]
print >> sys.stderr, ('MessageDispatcher exception [%s] occurred' % (e))
CMAdb.log.critical('MessageDispatcher exception [%s] occurred while'
' handling [%s] FrameSet from %s' % (e, fstypename, origaddr))
lines = str(frameset).splitlines()
CMAdb.log.info('FrameSet Contents follows (%d lines):' % len(lines))
for line in lines:
CMAdb.log.info(line.expandtabs())
CMAdb.log.info('======== Begin %s Message %s Exception Traceback ========'
% (fstypename, e))
for tb in tblist:
(filename, line, funcname, text) = tb
filename = os.path.basename(filename)
CMAdb.log.info('%s.%s:%s: %s'% (filename, line, funcname, text))
CMAdb.log.info('======== End %s Message %s Exception Traceback ========'
% (fstypename, e))
if CMAdb.store is not None:
CMAdb.log.critical("Aborting Neo4j transaction %s" % CMAdb.store)
CMAdb.store.abort()
if CMAdb.transaction is not None:
CMAdb.log.critical("Aborting network transaction %s" % CMAdb.transaction.tree)
CMAdb.transaction = None
@staticmethod
def _check_memory_usage():
'Check to see if we have too many objects outstanding right now'
gccount = gc.get_count()
gctotal = 0
for elem in gccount:
gctotal += elem
CMAdb.log.info('Total allocated Objects: %s. gc levels: %s'
% (gctotal, str(gccount)))
cobjcount = proj_class_live_object_count()
CMAdb.log.info('Total/max allocated C-Objects: %s/%s'
% (cobjcount, proj_class_max_object_count()))
if gctotal < 20 and cobjcount > 5000:
dump_c_objects()
if CMAdb.debug:
# Another very expensive set of debug-only calls
assimcount = 0
for obj in gc.get_objects():
if isinstance(obj, (pyAssimObj)):
assimcount += 1
CMAdb.log.info('Total allocated C-Objects: %s' % assimcount)
def setconfig(self, io, config):
'Save our configuration away. We need it before we can do anything.'
self.io = io
self.default.setconfig(io, config)
for msgtype in self.dispatchtable.keys():
self.dispatchtable[msgtype].setconfig(io, config)
class MessageDispatcher(object):
'We dispatch incoming messages where they need to go.'
def __init__(self,
dispatchtable,
logtimes=False,
encryption_required=True):
'Constructor for MessageDispatcher - requires a dispatch table as a parameter'
self.dispatchtable = dispatchtable
self.default = DispatchTarget()
self.io = None
self.dispatchcount = 0
self.logtimes = logtimes or CMAdb.debug
self.encryption_required = encryption_required
def dispatch(self, origaddr, frameset):
'Dispatch a Frameset where it will get handled.'
self.dispatchcount += 1
CMAdb.transaction = Transaction(
encryption_required=self.encryption_required)
# W0703 == Too general exception catching...
# pylint: disable=W0703
try:
self._try_dispatch_action(origaddr, frameset)
if (self.dispatchcount % 100) == 1:
self._check_memory_usage()
except Exception as e:
self._process_exception(e, origaddr, frameset)
# We want to ack the packet even in the failed case - retries are unlikely to help
# and we need to avoid getting stuck in a loop retrying it forever...
if CMAdb.debug:
fstypename = FrameSetTypes.get(frameset.get_framesettype())[0]
CMAdb.log.debug('MessageDispatcher - ACKing %s message from %s' %
(fstypename, origaddr))
self.io.ackmessage(origaddr, frameset)
# [R0912:MessageDispatcher._try_dispatch_action] Too many branches (13/12)
# pylint: disable=R0912
def _try_dispatch_action(self, origaddr, frameset):
'''Core code to actually dispatch the Frameset.
It should be run inside a try/except construct so that anything
we barf up won't cause the CMA to die.
'''
fstype = frameset.get_framesettype()
#print >>sys.stderr, 'Got frameset of type %s [%s]' % (fstype, frameset)
dispatchstart = datetime.now()
if fstype in self.dispatchtable:
self.dispatchtable[fstype].dispatch(origaddr, frameset)
else:
self.default.dispatch(origaddr, frameset)
dispatchend = datetime.now()
if self.logtimes:
CMAdb.log.info('Initial dispatch time for %s frameset: %s' %
(fstype, dispatchend - dispatchstart))
# Commit the network transaction here
CMAdb.transaction.commit_trans(CMAdb.io)
if self.logtimes:
CMAdb.log.info('Network transaction time: %s' %
(str(CMAdb.transaction.stats['lastcommit'])))
# Commit the database transaction here
if CMAdb.store.transaction_pending:
result = CMAdb.store.commit()
if self.logtimes or CMAdb.debug:
CMAdb.log.info('Neo4j transaction time: %s' %
(str(CMAdb.store.stats['lastcommit'])))
if CMAdb.debug:
resultlines = str(result).splitlines()
CMAdb.log.debug('Commit results follow:')
for line in resultlines:
CMAdb.log.debug(line.expandtabs())
CMAdb.log.debug('end of commit results.')
# This is a VERY expensive call...
# Good thing we only do it when debug is enabled...
CMAdb.TheOneRing.AUDIT()
else:
if CMAdb.debug:
CMAdb.log.debug('No database changes this time')
CMAdb.store.abort()
for pkttype in CMAdb.transaction.post_transaction_packets:
CMAdb.transaction.add_packet(origaddr, pkttype, [])
if len(CMAdb.transaction.post_transaction_packets) > 0:
CMAdb.transaction.commit_trans(CMAdb.io)
CMAdb.post_transaction_packets = []
dispatchend = datetime.now()
if self.logtimes or CMAdb.debug:
CMAdb.log.info('Total dispatch time for %s frameset: %s' %
(fstype, dispatchend - dispatchstart))
@staticmethod
def _process_exception(e, origaddr, frameset):
'Handle an exception from our message dispatcher'
# Darn! Got an exception - let's try and put everything useful into the
# logs in a legible way
trace = sys.exc_info()[2]
# we ignore the etype and evalue returns from sys.exc_info
tblist = traceback.extract_tb(trace, 20)
fstype = frameset.get_framesettype()
fstypename = FrameSetTypes.get(fstype)[0]
print >> sys.stderr, ('MessageDispatcher exception [%s] occurred' %
(e))
CMAdb.log.critical('MessageDispatcher exception [%s] occurred while'
' handling [%s] FrameSet from %s' %
(e, fstypename, origaddr))
lines = str(frameset).splitlines()
CMAdb.log.info('FrameSet Contents follows (%d lines):' % len(lines))
for line in lines:
CMAdb.log.info(line.expandtabs())
CMAdb.log.info(
'======== Begin %s Message %s Exception Traceback ========' %
(fstypename, e))
for tb in tblist:
(filename, line, funcname, text) = tb
filename = os.path.basename(filename)
CMAdb.log.info('%s.%s:%s: %s' % (filename, line, funcname, text))
CMAdb.log.info(
'======== End %s Message %s Exception Traceback ========' %
(fstypename, e))
if CMAdb.store is not None:
CMAdb.log.critical("Aborting Neo4j transaction %s" % CMAdb.store)
CMAdb.store.abort()
if CMAdb.transaction is not None:
CMAdb.log.critical("Aborting network transaction %s" %
CMAdb.transaction.tree)
CMAdb.transaction = None
@staticmethod
def _check_memory_usage():
'Check to see if we have too many objects outstanding right now'
gccount = gc.get_count()
gctotal = 0
for elem in gccount:
gctotal += elem
CMAdb.log.info('Total allocated Objects: %s. gc levels: %s' %
(gctotal, str(gccount)))
cobjcount = proj_class_live_object_count()
CMAdb.log.info('Total/max allocated C-Objects: %s/%s' %
(cobjcount, proj_class_max_object_count()))
if gctotal < 20 and cobjcount > 5000:
dump_c_objects()
if CMAdb.debug:
# Another very expensive set of debug-only calls
assimcount = 0
for obj in gc.get_objects():
if isinstance(obj, (pyAssimObj)):
assimcount += 1
CMAdb.log.info('Total allocated C-Objects: %s' % assimcount)
def setconfig(self, io, config):
'Save our configuration away. We need it before we can do anything.'
self.io = io
self.default.setconfig(io, config)
for msgtype in self.dispatchtable.keys():
self.dispatchtable[msgtype].setconfig(io, config)
