def __init__(self, monitor, host, port, protocol = 'http', debug = 0):
self.monitor = monitor
self.host = host
self.port = port
self.protocol = protocol
self.debug = debug
self.free_channels = []
self.active_channels = Dictionary()
self.inactive_channels = Dictionary()
self.parallel_channels = Counter()
self.pending_transactions = []
self.inflight_transactions = Dictionary()
self.state_lock = Lock()
self.debugout('Instantiated', 2)
class TransactionManager(object):
QUEUEDEBUG = False
def __init__(self, monitor, debug=0):
self.monitor = monitor
self.debug = debug
self.controllers = Dictionary()
self.controller_lock = Lock()
super(TransactionManager, self).__init__()
def _get_controller(self, host, port, protocol):
location = (host, port)
self.controller_lock.acquire()
try:
if not self.controllers.has_key(location):
self.controllers[location] = LocationController(
self.monitor, host, port, protocol, self.debug)
finally:
self.controller_lock.release()
return self.controllers.get(location)
def create_transaction(self, request, sid):
controller = self._get_controller(request.get_host(),
request.get_port(),
request.get_type())
if (controller.is_transaction_pending(sid)):
if self.debug or self.QUEUEDEBUG:
msglog.log(
'broadway', msglog.types.DB,
'Already Pending transaction, so not adding transaction. sid=%s'
% (sid))
controller.recruit_and_send()
return (None)
transaction = Transaction(request, sid=sid)
if self.debug or self.QUEUEDEBUG:
before = repr(controller)
controller.add_transaction(transaction)
if self.debug or self.QUEUEDEBUG:
after = repr(controller)
message = 'Controller before and after adding transaction:\n'
message += ' - Before: %s\n' % (before[1:-1])
message += ' - After: %s\n' % (after[1:-1])
msglog.log('broadway', msglog.types.DB, message)
return transaction
class TransactionManager(object):
QUEUEDEBUG = False
def __init__(self, monitor, debug = 0):
self.monitor = monitor
self.debug = debug
self.controllers = Dictionary()
self.controller_lock = Lock()
super(TransactionManager, self).__init__()
def _get_controller(self, host, port, protocol):
location = (host, port)
self.controller_lock.acquire()
try:
if not self.controllers.has_key(location):
self.controllers[location] = LocationController(
self.monitor, host, port, protocol, self.debug)
finally:
self.controller_lock.release()
return self.controllers.get(location)
def create_transaction(self, request,sid):
controller = self._get_controller(request.get_host(),
request.get_port(),
request.get_type())
if(controller.is_transaction_pending(sid)):
if self.debug or self.QUEUEDEBUG:
msglog.log('broadway',msglog.types.DB,'Already Pending transaction, so not adding transaction. sid=%s' %(sid))
controller.recruit_and_send()
return(None)
transaction = Transaction(request,sid=sid)
if self.debug or self.QUEUEDEBUG:
before = repr(controller)
controller.add_transaction(transaction)
if self.debug or self.QUEUEDEBUG:
after = repr(controller)
message = 'Controller before and after adding transaction:\n'
message += ' - Before: %s\n' % (before[1:-1])
message += ' - After: %s\n' % (after[1:-1])
msglog.log('broadway', msglog.types.DB, message)
return transaction
def __init__(self, monitor, host, port, protocol='http', debug=0):
self.monitor = monitor
self.host = host
self.port = port
self.protocol = protocol
self.debug = debug
self.free_channels = []
self.active_channels = Dictionary()
self.inactive_channels = Dictionary()
self.parallel_channels = Counter()
self.pending_transactions = []
self.inflight_transactions = Dictionary()
self.state_lock = Lock()
self.debugout('Instantiated', 2)
class LocationController(object):
IDEALBACKLOG = 5
MAXPARALLELCHANNELS = 10
MAXTRANSPERCHANNEL = 250
ZOMBIETIMEOUT = 900
def __init__(self, monitor, host, port, protocol = 'http', debug = 0):
self.monitor = monitor
self.host = host
self.port = port
self.protocol = protocol
self.debug = debug
self.free_channels = []
self.active_channels = Dictionary()
self.inactive_channels = Dictionary()
self.parallel_channels = Counter()
self.pending_transactions = []
self.inflight_transactions = Dictionary()
self.state_lock = Lock()
self.debugout('Instantiated', 2)
def is_transaction_pending(self,sid):
for tran in self.pending_transactions:
if(tran.sid == sid):
return(True)
return(False)
def recruit_and_send(self):
self.state_lock.acquire()
self.debugout('Pending Transactions in recruit and send are :%d' %(len(self.pending_transactions)))
try:
openchannel = self._recruit_channel()
finally:
self.state_lock.release()
if openchannel:
self.debugout('Initiating a transaction by recruit and send' )
self.initiate_next_transaction()
else:
self.debugout('Recruit channel returns None. Unable to init a transaction by recruit and send')
def add_transaction(self, transaction):
self.state_lock.acquire()
self.debugout('Pending Transactions are :%d' %(len(self.pending_transactions)))
try:
self.debugout('[%s] Adding transaction', 3)
transaction.set_manager(self)
openchannel = self._recruit_channel()
self.pending_transactions.append(transaction)
finally:
self.state_lock.release()
if openchannel:
self.initiate_next_transaction()
else:
self.debugout('Recruit channel returns None. Pending transactions =%d' %(len(self.pending_transactions)))
def initiate_next_transaction(self):
self.state_lock.acquire()
try:
channel = self._activate_channel()
if channel is not None:
transaction = self._activate_transaction()
finally:
self.state_lock.release()
if channel is None:
message = 'Activate channel returned None to initiate next.'
message += '\n\t\t-- %r\n'
self.debugout(message, 1, self)
else:
transaction.set_channel(channel)
message = '[%s] Initiating transaction.'
self.debugout(message, 3)
if channel.response_count() > self.MAXTRANSPERCHANNEL:
message = '[%s] Configuring transaction to close channel:'
message += '\n\t%r\n\t%r\n'
self.debugout(message, 1, transaction, channel)
transaction.close_when_done()
transaction.initiate()
def handle_completed_transaction(self, transaction):
self.debugout('[%s] Handling completed transaction', 3)
self.state_lock.acquire()
try:
self._deactivate_transaction(transaction)
self._deactivate_channel(transaction.channel)
if self.pending_transactions:
initiatenext = (self.inactive_channels or
self._recruit_channel())
else:
initiatenext = False
self._free_channel(transaction.channel)
finally:
self.state_lock.release()
if self.debug > 1:
message = '[%s] completed: ' % self
message += '%s' % transaction.stats()[1:-1]
self.msglog(message, msglog.types.DB, False)
if initiatenext:
self.initiate_next_transaction()
def handle_failed_transaction(self, transaction):
try:
self.state_lock.acquire()
try:
# Changes for CSCtg33093 (b.Avoid getting the channel number, if channel is None)
if(transaction.channel != None ):
transaction.channel.accepting_requests(False)
finally:
self.state_lock.release()
message = 'Handling failed transaction %r' % transaction
self.msglog(message, msglog.types.WARN)
finally:
self.handle_completed_transaction(transaction)
def _activate_transaction(self):
assert self.state_lock.locked()
transaction = self.pending_transactions.pop(0)
tid = transaction.transaction_number
self.inflight_transactions[tid] = transaction
return transaction
def _deactivate_transaction(self, transaction):
assert self.state_lock.locked()
tid = transaction.transaction_number
try:
self.inflight_transactions.pop(tid)
except KeyError:
self.debugout('Transaction %r not in active list', 1, transaction)
return False
else:
return True
def _recruit_channel(self):
assert self.state_lock.locked()
if not self.free_channels:
self._manage_channels()
channel = None
if self.free_channels:
channel = self.free_channels.pop()
cid = channel.channel_number
self.inactive_channels[cid] = channel
return channel
def _free_channel(self, channel):
assert self.state_lock.locked()
cid = channel.channel_number
if self.inactive_channels.has_key(cid):
self.inactive_channels.pop(cid)
if channel.accepting_requests():
self.free_channels.append(channel)
def _activate_channel(self):
assert self.state_lock.locked()
if not self.inactive_channels:
channel = self._recruit_channel()
else:
cid, channel = self.inactive_channels.popitem()
if channel is not None:
self.active_channels[channel.channel_number] = channel
return channel
def _deactivate_channel(self, channel):
assert self.state_lock.locked()
# Changes for CSCtg33093 (b.Avoid getting the channel number, if channel is None)
if(channel == None):
return(False)
cid = channel.channel_number
try:
self.inactive_channels[cid] = self.active_channels.pop(cid)
except KeyError:
self.debugout('Channel %r not in active list', 1, channel)
return False
else:
if not channel.accepting_requests():
self.inactive_channels.pop(cid)
self.parallel_channels.decrement()
if not channel.is_closed():
channel.close()
self.debugout('Channel %r decommissioned.', 1, channel)
return True
def _manage_channels(self):
assert self.state_lock.locked()
for cid, channel in self.active_channels.items():
message = 'Location channel management removing %r from active.'
if not channel.accepting_requests():
self.debugout(message % channel, 1)
self.active_channels.pop(cid)
self.parallel_channels.decrement()
maxchannels = self.MAXPARALLELCHANNELS
numpending = len(self.pending_transactions)
numchannels = self.parallel_channels.value
if numchannels == 0:
createchannel = True
elif numchannels >= maxchannels:
createchannel = False
elif (numpending / numchannels) > self.IDEALBACKLOG:
createchannel = True
else:
createchannel = False
if createchannel:
channel = Channel(self.monitor, self.debug)
channel.setup_connection(self.host, self.port, self.protocol)
self.free_channels.append(channel)
self.parallel_channels.increment()
return
def msglog(self, message, mtype = msglog.types.INFO, autoprefix = False):
if autoprefix:
message = '%s %s' % (self, message)
msglog.log('broadway', mtype, message)
def debugout(self, message, debuglevel = 1, *args):
if debuglevel <= self.debug:
if self not in args:
if message.count('%') == len(args):
message = '%s ' + message
args = (self,) + args
return self.msglog(message % args, msglog.types.DB)
def __repr__(self):
status = ['Controller']
status.append('[%s:%d]' % (self.host, self.port))
channels = self.parallel_channels.value
active = len(self.active_channels)
inactive = len(self.inactive_channels)
free = len(self.free_channels)
pending = len(self.pending_transactions)
inflight = len(self.inflight_transactions)
channelmsg = '%d Channels (%dA %dI %dF)'
status.append(channelmsg % (channels, active, inactive, free))
status.append('Transactions (%dA %dP)' % (inflight, pending))
return '<%s>' % (' '.join(status))
def __str__(self):
classname = self.__class__.__name__
channels = self.parallel_channels.value
status = '(%d => %s:%d)' % (channels, self.host, self.port)
return '%s %s' % (classname, status)
def __init__(self, monitor, debug = 0):
self.monitor = monitor
self.debug = debug
self.controllers = Dictionary()
self.controller_lock = Lock()
super(TransactionManager, self).__init__()
class LocationController(object):
IDEALBACKLOG = 5
MAXPARALLELCHANNELS = 10
MAXTRANSPERCHANNEL = 250
ZOMBIETIMEOUT = 900
def __init__(self, monitor, host, port, protocol='http', debug=0):
self.monitor = monitor
self.host = host
self.port = port
self.protocol = protocol
self.debug = debug
self.free_channels = []
self.active_channels = Dictionary()
self.inactive_channels = Dictionary()
self.parallel_channels = Counter()
self.pending_transactions = []
self.inflight_transactions = Dictionary()
self.state_lock = Lock()
self.debugout('Instantiated', 2)
def is_transaction_pending(self, sid):
for tran in self.pending_transactions:
if (tran.sid == sid):
return (True)
return (False)
def recruit_and_send(self):
self.state_lock.acquire()
self.debugout('Pending Transactions in recruit and send are :%d' %
(len(self.pending_transactions)))
try:
openchannel = self._recruit_channel()
finally:
self.state_lock.release()
if openchannel:
self.debugout('Initiating a transaction by recruit and send')
self.initiate_next_transaction()
else:
self.debugout(
'Recruit channel returns None. Unable to init a transaction by recruit and send'
)
def add_transaction(self, transaction):
self.state_lock.acquire()
self.debugout('Pending Transactions are :%d' %
(len(self.pending_transactions)))
try:
self.debugout('[%s] Adding transaction', 3)
transaction.set_manager(self)
openchannel = self._recruit_channel()
self.pending_transactions.append(transaction)
finally:
self.state_lock.release()
if openchannel:
self.initiate_next_transaction()
else:
self.debugout(
'Recruit channel returns None. Pending transactions =%d' %
(len(self.pending_transactions)))
def initiate_next_transaction(self):
self.state_lock.acquire()
try:
channel = self._activate_channel()
if channel is not None:
transaction = self._activate_transaction()
finally:
self.state_lock.release()
if channel is None:
message = 'Activate channel returned None to initiate next.'
message += '\n\t\t-- %r\n'
self.debugout(message, 1, self)
else:
transaction.set_channel(channel)
message = '[%s] Initiating transaction.'
self.debugout(message, 3)
if channel.response_count() > self.MAXTRANSPERCHANNEL:
message = '[%s] Configuring transaction to close channel:'
message += '\n\t%r\n\t%r\n'
self.debugout(message, 1, transaction, channel)
transaction.close_when_done()
transaction.initiate()
def handle_completed_transaction(self, transaction):
self.debugout('[%s] Handling completed transaction', 3)
self.state_lock.acquire()
try:
self._deactivate_transaction(transaction)
self._deactivate_channel(transaction.channel)
if self.pending_transactions:
initiatenext = (self.inactive_channels
or self._recruit_channel())
else:
initiatenext = False
self._free_channel(transaction.channel)
finally:
self.state_lock.release()
if self.debug > 1:
message = '[%s] completed: ' % self
message += '%s' % transaction.stats()[1:-1]
self.msglog(message, msglog.types.DB, False)
if initiatenext:
self.initiate_next_transaction()
def handle_failed_transaction(self, transaction):
try:
self.state_lock.acquire()
try:
# Changes for CSCtg33093 (b.Avoid getting the channel number, if channel is None)
if (transaction.channel != None):
transaction.channel.accepting_requests(False)
finally:
self.state_lock.release()
message = 'Handling failed transaction %r' % transaction
self.msglog(message, msglog.types.WARN)
finally:
self.handle_completed_transaction(transaction)
def _activate_transaction(self):
assert self.state_lock.locked()
transaction = self.pending_transactions.pop(0)
tid = transaction.transaction_number
self.inflight_transactions[tid] = transaction
return transaction
def _deactivate_transaction(self, transaction):
assert self.state_lock.locked()
tid = transaction.transaction_number
try:
self.inflight_transactions.pop(tid)
except KeyError:
self.debugout('Transaction %r not in active list', 1, transaction)
return False
else:
return True
def _recruit_channel(self):
assert self.state_lock.locked()
if not self.free_channels:
self._manage_channels()
channel = None
if self.free_channels:
channel = self.free_channels.pop()
cid = channel.channel_number
self.inactive_channels[cid] = channel
return channel
def _free_channel(self, channel):
assert self.state_lock.locked()
cid = channel.channel_number
if self.inactive_channels.has_key(cid):
self.inactive_channels.pop(cid)
if channel.accepting_requests():
self.free_channels.append(channel)
def _activate_channel(self):
assert self.state_lock.locked()
if not self.inactive_channels:
channel = self._recruit_channel()
else:
cid, channel = self.inactive_channels.popitem()
if channel is not None:
self.active_channels[channel.channel_number] = channel
return channel
def _deactivate_channel(self, channel):
assert self.state_lock.locked()
# Changes for CSCtg33093 (b.Avoid getting the channel number, if channel is None)
if (channel == None):
return (False)
cid = channel.channel_number
try:
self.inactive_channels[cid] = self.active_channels.pop(cid)
except KeyError:
self.debugout('Channel %r not in active list', 1, channel)
return False
else:
if not channel.accepting_requests():
self.inactive_channels.pop(cid)
self.parallel_channels.decrement()
if not channel.is_closed():
channel.close()
self.debugout('Channel %r decommissioned.', 1, channel)
return True
def _manage_channels(self):
assert self.state_lock.locked()
for cid, channel in self.active_channels.items():
message = 'Location channel management removing %r from active.'
if not channel.accepting_requests():
self.debugout(message % channel, 1)
self.active_channels.pop(cid)
self.parallel_channels.decrement()
maxchannels = self.MAXPARALLELCHANNELS
numpending = len(self.pending_transactions)
numchannels = self.parallel_channels.value
if numchannels == 0:
createchannel = True
elif numchannels >= maxchannels:
createchannel = False
elif (numpending / numchannels) > self.IDEALBACKLOG:
createchannel = True
else:
createchannel = False
if createchannel:
channel = Channel(self.monitor, self.debug)
channel.setup_connection(self.host, self.port, self.protocol)
self.free_channels.append(channel)
self.parallel_channels.increment()
return
def msglog(self, message, mtype=msglog.types.INFO, autoprefix=False):
if autoprefix:
message = '%s %s' % (self, message)
msglog.log('broadway', mtype, message)
def debugout(self, message, debuglevel=1, *args):
if debuglevel <= self.debug:
if self not in args:
if message.count('%') == len(args):
message = '%s ' + message
args = (self, ) + args
return self.msglog(message % args, msglog.types.DB)
def __repr__(self):
status = ['Controller']
status.append('[%s:%d]' % (self.host, self.port))
channels = self.parallel_channels.value
active = len(self.active_channels)
inactive = len(self.inactive_channels)
free = len(self.free_channels)
pending = len(self.pending_transactions)
inflight = len(self.inflight_transactions)
channelmsg = '%d Channels (%dA %dI %dF)'
status.append(channelmsg % (channels, active, inactive, free))
status.append('Transactions (%dA %dP)' % (inflight, pending))
return '<%s>' % (' '.join(status))
def __str__(self):
classname = self.__class__.__name__
channels = self.parallel_channels.value
status = '(%d => %s:%d)' % (channels, self.host, self.port)
return '%s %s' % (classname, status)
def __init__(self, monitor, debug=0):
self.monitor = monitor
self.debug = debug
self.controllers = Dictionary()
self.controller_lock = Lock()
super(TransactionManager, self).__init__()