Esempio n. 1
0
 def __init__(self, monitor, host, port, protocol = 'http', debug = 0):
     self.monitor = monitor
     self.host = host
     self.port = port
     self.protocol = protocol
     self.debug = debug
     self.free_channels = []
     self.active_channels = Dictionary()
     self.inactive_channels = Dictionary()
     self.parallel_channels = Counter()
     self.pending_transactions = []
     self.inflight_transactions = Dictionary()
     self.state_lock = Lock()
     self.debugout('Instantiated', 2)
Esempio n. 2
0
class TransactionManager(object):
    QUEUEDEBUG = False

    def __init__(self, monitor, debug=0):
        self.monitor = monitor
        self.debug = debug
        self.controllers = Dictionary()
        self.controller_lock = Lock()
        super(TransactionManager, self).__init__()

    def _get_controller(self, host, port, protocol):
        location = (host, port)
        self.controller_lock.acquire()
        try:
            if not self.controllers.has_key(location):
                self.controllers[location] = LocationController(
                    self.monitor, host, port, protocol, self.debug)
        finally:
            self.controller_lock.release()
        return self.controllers.get(location)

    def create_transaction(self, request, sid):
        controller = self._get_controller(request.get_host(),
                                          request.get_port(),
                                          request.get_type())

        if (controller.is_transaction_pending(sid)):
            if self.debug or self.QUEUEDEBUG:
                msglog.log(
                    'broadway', msglog.types.DB,
                    'Already Pending transaction, so not adding transaction. sid=%s'
                    % (sid))
            controller.recruit_and_send()
            return (None)

        transaction = Transaction(request, sid=sid)

        if self.debug or self.QUEUEDEBUG:
            before = repr(controller)
        controller.add_transaction(transaction)
        if self.debug or self.QUEUEDEBUG:
            after = repr(controller)
            message = 'Controller before and after adding transaction:\n'
            message += '    - Before: %s\n' % (before[1:-1])
            message += '    - After: %s\n' % (after[1:-1])
            msglog.log('broadway', msglog.types.DB, message)
        return transaction
Esempio n. 3
0
class TransactionManager(object):
    QUEUEDEBUG = False
    def __init__(self, monitor, debug = 0):
        self.monitor = monitor
        self.debug = debug
        self.controllers = Dictionary()
        self.controller_lock = Lock()
        super(TransactionManager, self).__init__()
    def _get_controller(self, host, port, protocol):
        location = (host, port)
        self.controller_lock.acquire()
        try:
            if not self.controllers.has_key(location):
                self.controllers[location] = LocationController(
                    self.monitor, host, port, protocol, self.debug)
        finally:
            self.controller_lock.release()
        return self.controllers.get(location)
    def create_transaction(self, request,sid):
        controller = self._get_controller(request.get_host(), 
                                          request.get_port(), 
                                          request.get_type())
        
        if(controller.is_transaction_pending(sid)):
            if self.debug or self.QUEUEDEBUG: 
                msglog.log('broadway',msglog.types.DB,'Already Pending transaction, so not adding transaction. sid=%s' %(sid))
            controller.recruit_and_send()
            return(None)
            
        transaction = Transaction(request,sid=sid)
        
        if self.debug or self.QUEUEDEBUG:
            before = repr(controller)
        controller.add_transaction(transaction)
        if self.debug or self.QUEUEDEBUG:
            after = repr(controller)
            message = 'Controller before and after adding transaction:\n'
            message += '    - Before: %s\n' % (before[1:-1])
            message += '    - After: %s\n' % (after[1:-1])
            msglog.log('broadway', msglog.types.DB, message)
        return transaction
Esempio n. 4
0
 def __init__(self, monitor, host, port, protocol='http', debug=0):
     self.monitor = monitor
     self.host = host
     self.port = port
     self.protocol = protocol
     self.debug = debug
     self.free_channels = []
     self.active_channels = Dictionary()
     self.inactive_channels = Dictionary()
     self.parallel_channels = Counter()
     self.pending_transactions = []
     self.inflight_transactions = Dictionary()
     self.state_lock = Lock()
     self.debugout('Instantiated', 2)
Esempio n. 5
0
class LocationController(object):
    IDEALBACKLOG = 5
    MAXPARALLELCHANNELS = 10
    MAXTRANSPERCHANNEL = 250
    ZOMBIETIMEOUT = 900
    def __init__(self, monitor, host, port, protocol = 'http', debug = 0):
        self.monitor = monitor
        self.host = host
        self.port = port
        self.protocol = protocol
        self.debug = debug
        self.free_channels = []
        self.active_channels = Dictionary()
        self.inactive_channels = Dictionary()
        self.parallel_channels = Counter()
        self.pending_transactions = []
        self.inflight_transactions = Dictionary()
        self.state_lock = Lock()
        self.debugout('Instantiated', 2)
    def is_transaction_pending(self,sid):
        for tran in self.pending_transactions:
            if(tran.sid == sid):
                return(True)
        return(False)
    
    def recruit_and_send(self):
        self.state_lock.acquire()
        self.debugout('Pending Transactions in recruit and send are :%d' %(len(self.pending_transactions)))
        try:
            openchannel = self._recruit_channel()
        finally:
            self.state_lock.release()
        if openchannel:
            self.debugout('Initiating a transaction by recruit and send' )
            self.initiate_next_transaction()
        else:
            self.debugout('Recruit channel returns None. Unable to init a transaction by recruit and send')
            
        
        
    def add_transaction(self, transaction):
        self.state_lock.acquire()
        self.debugout('Pending Transactions are :%d' %(len(self.pending_transactions)))
        try:
            self.debugout('[%s] Adding transaction', 3)
            transaction.set_manager(self)
            openchannel = self._recruit_channel()
            self.pending_transactions.append(transaction)     
        finally:
            self.state_lock.release()
        if openchannel:
            self.initiate_next_transaction()
        else:
            self.debugout('Recruit channel returns None. Pending transactions =%d' %(len(self.pending_transactions)))
    
                     
    def initiate_next_transaction(self):
        self.state_lock.acquire()
        try:
            channel = self._activate_channel()
            if channel is not None:
                transaction = self._activate_transaction()
        finally:
            self.state_lock.release()
        if channel is None:
            message = 'Activate channel returned None to initiate next.'
            message += '\n\t\t-- %r\n'
            self.debugout(message, 1, self)
        else:
            transaction.set_channel(channel)
            message = '[%s] Initiating transaction.'
            self.debugout(message, 3)
            if channel.response_count() > self.MAXTRANSPERCHANNEL:
                message = '[%s] Configuring transaction to close channel:'
                message += '\n\t%r\n\t%r\n'
                self.debugout(message, 1, transaction, channel)
                transaction.close_when_done()
            transaction.initiate()
    def handle_completed_transaction(self, transaction):
        self.debugout('[%s] Handling completed transaction', 3)
        self.state_lock.acquire()
        try:
            self._deactivate_transaction(transaction)
            self._deactivate_channel(transaction.channel)
            if self.pending_transactions:
                initiatenext = (self.inactive_channels or 
                                self._recruit_channel())
            else:
                initiatenext = False
                self._free_channel(transaction.channel)
        finally:
            self.state_lock.release()
        if self.debug > 1:
            message = '[%s] completed: ' % self
            message += '%s' % transaction.stats()[1:-1]
            self.msglog(message, msglog.types.DB, False)
        if initiatenext:
            self.initiate_next_transaction()
    def handle_failed_transaction(self, transaction):
        try:
            self.state_lock.acquire()
            try:
                # Changes for CSCtg33093 (b.Avoid getting the channel number, if channel is None)
                if(transaction.channel != None ):
                    transaction.channel.accepting_requests(False)
            finally:
                self.state_lock.release()
            message = 'Handling failed transaction %r' % transaction
            self.msglog(message, msglog.types.WARN)
        finally:
            self.handle_completed_transaction(transaction)
    def _activate_transaction(self):
        assert self.state_lock.locked()
        transaction = self.pending_transactions.pop(0)
        tid = transaction.transaction_number
        self.inflight_transactions[tid] = transaction
        return transaction
    def _deactivate_transaction(self, transaction):
        assert self.state_lock.locked()
        tid = transaction.transaction_number
        try:
            self.inflight_transactions.pop(tid)
        except KeyError:
            self.debugout('Transaction %r not in active list', 1, transaction)
            return False
        else: 
            return True
    def _recruit_channel(self):
        assert self.state_lock.locked()
        if not self.free_channels:
            self._manage_channels()
            channel = None
        if self.free_channels:
            channel = self.free_channels.pop()
            cid = channel.channel_number
            self.inactive_channels[cid] = channel
        return channel
    def _free_channel(self, channel):
        assert self.state_lock.locked()
        cid = channel.channel_number
        if self.inactive_channels.has_key(cid):
            self.inactive_channels.pop(cid)
            if channel.accepting_requests():
                self.free_channels.append(channel)
    def _activate_channel(self):
        assert self.state_lock.locked()
        if not self.inactive_channels:
            channel = self._recruit_channel()
        else:
            cid, channel = self.inactive_channels.popitem()
        if channel is not None:
            self.active_channels[channel.channel_number] = channel
        return channel
    def _deactivate_channel(self, channel):
        assert self.state_lock.locked()
        # Changes for CSCtg33093 (b.Avoid getting the channel number, if channel is None)
        if(channel == None):
            return(False)
        cid = channel.channel_number
        try: 
            self.inactive_channels[cid] = self.active_channels.pop(cid)
        except KeyError:
            self.debugout('Channel %r not in active list', 1, channel)
            return False
        else:
            if not channel.accepting_requests():
                self.inactive_channels.pop(cid)
                self.parallel_channels.decrement()
                if not channel.is_closed():
                    channel.close()
                self.debugout('Channel %r decommissioned.', 1, channel)
            return True
    def _manage_channels(self):
        assert self.state_lock.locked()
        for cid, channel in self.active_channels.items():
            message = 'Location channel management removing %r from active.'
            if not channel.accepting_requests():
                self.debugout(message % channel, 1)
                self.active_channels.pop(cid)
                self.parallel_channels.decrement()
        maxchannels = self.MAXPARALLELCHANNELS
        numpending = len(self.pending_transactions)
        numchannels = self.parallel_channels.value
        if numchannels == 0:
            createchannel = True
        elif numchannels >= maxchannels:
            createchannel = False
        elif (numpending / numchannels) > self.IDEALBACKLOG:
            createchannel = True
        else:
            createchannel = False
        if createchannel:
            channel = Channel(self.monitor, self.debug)
            channel.setup_connection(self.host, self.port, self.protocol)
            self.free_channels.append(channel)
            self.parallel_channels.increment()
        return
    def msglog(self, message, mtype = msglog.types.INFO, autoprefix = False):
        if autoprefix:
            message = '%s %s' % (self, message)
        msglog.log('broadway', mtype, message)
    def debugout(self, message, debuglevel = 1, *args):
        if debuglevel <= self.debug:
            if self not in args:
                if message.count('%') == len(args):
                    message = '%s ' + message
                args = (self,) + args
            return self.msglog(message % args, msglog.types.DB)
    def __repr__(self):
        status = ['Controller']
        status.append('[%s:%d]' % (self.host, self.port))
        channels = self.parallel_channels.value
        active = len(self.active_channels)
        inactive = len(self.inactive_channels)
        free = len(self.free_channels)
        pending = len(self.pending_transactions)
        inflight = len(self.inflight_transactions)
        channelmsg = '%d Channels (%dA %dI %dF)'
        status.append(channelmsg % (channels, active, inactive, free))
        status.append('Transactions (%dA %dP)' % (inflight, pending))
        return '<%s>' % (' '.join(status))
    def __str__(self):
        classname = self.__class__.__name__
        channels = self.parallel_channels.value
        status = '(%d => %s:%d)' % (channels, self.host, self.port)
        return '%s %s' % (classname, status)
Esempio n. 6
0
 def __init__(self, monitor, debug = 0):
     self.monitor = monitor
     self.debug = debug
     self.controllers = Dictionary()
     self.controller_lock = Lock()
     super(TransactionManager, self).__init__()
Esempio n. 7
0
class LocationController(object):
    IDEALBACKLOG = 5
    MAXPARALLELCHANNELS = 10
    MAXTRANSPERCHANNEL = 250
    ZOMBIETIMEOUT = 900

    def __init__(self, monitor, host, port, protocol='http', debug=0):
        self.monitor = monitor
        self.host = host
        self.port = port
        self.protocol = protocol
        self.debug = debug
        self.free_channels = []
        self.active_channels = Dictionary()
        self.inactive_channels = Dictionary()
        self.parallel_channels = Counter()
        self.pending_transactions = []
        self.inflight_transactions = Dictionary()
        self.state_lock = Lock()
        self.debugout('Instantiated', 2)

    def is_transaction_pending(self, sid):
        for tran in self.pending_transactions:
            if (tran.sid == sid):
                return (True)
        return (False)

    def recruit_and_send(self):
        self.state_lock.acquire()
        self.debugout('Pending Transactions in recruit and send are :%d' %
                      (len(self.pending_transactions)))
        try:
            openchannel = self._recruit_channel()
        finally:
            self.state_lock.release()
        if openchannel:
            self.debugout('Initiating a transaction by recruit and send')
            self.initiate_next_transaction()
        else:
            self.debugout(
                'Recruit channel returns None. Unable to init a transaction by recruit and send'
            )

    def add_transaction(self, transaction):
        self.state_lock.acquire()
        self.debugout('Pending Transactions are :%d' %
                      (len(self.pending_transactions)))
        try:
            self.debugout('[%s] Adding transaction', 3)
            transaction.set_manager(self)
            openchannel = self._recruit_channel()
            self.pending_transactions.append(transaction)
        finally:
            self.state_lock.release()
        if openchannel:
            self.initiate_next_transaction()
        else:
            self.debugout(
                'Recruit channel returns None. Pending transactions =%d' %
                (len(self.pending_transactions)))

    def initiate_next_transaction(self):
        self.state_lock.acquire()
        try:
            channel = self._activate_channel()
            if channel is not None:
                transaction = self._activate_transaction()
        finally:
            self.state_lock.release()
        if channel is None:
            message = 'Activate channel returned None to initiate next.'
            message += '\n\t\t-- %r\n'
            self.debugout(message, 1, self)
        else:
            transaction.set_channel(channel)
            message = '[%s] Initiating transaction.'
            self.debugout(message, 3)
            if channel.response_count() > self.MAXTRANSPERCHANNEL:
                message = '[%s] Configuring transaction to close channel:'
                message += '\n\t%r\n\t%r\n'
                self.debugout(message, 1, transaction, channel)
                transaction.close_when_done()
            transaction.initiate()

    def handle_completed_transaction(self, transaction):
        self.debugout('[%s] Handling completed transaction', 3)
        self.state_lock.acquire()
        try:
            self._deactivate_transaction(transaction)
            self._deactivate_channel(transaction.channel)
            if self.pending_transactions:
                initiatenext = (self.inactive_channels
                                or self._recruit_channel())
            else:
                initiatenext = False
                self._free_channel(transaction.channel)
        finally:
            self.state_lock.release()
        if self.debug > 1:
            message = '[%s] completed: ' % self
            message += '%s' % transaction.stats()[1:-1]
            self.msglog(message, msglog.types.DB, False)
        if initiatenext:
            self.initiate_next_transaction()

    def handle_failed_transaction(self, transaction):
        try:
            self.state_lock.acquire()
            try:
                # Changes for CSCtg33093 (b.Avoid getting the channel number, if channel is None)
                if (transaction.channel != None):
                    transaction.channel.accepting_requests(False)
            finally:
                self.state_lock.release()
            message = 'Handling failed transaction %r' % transaction
            self.msglog(message, msglog.types.WARN)
        finally:
            self.handle_completed_transaction(transaction)

    def _activate_transaction(self):
        assert self.state_lock.locked()
        transaction = self.pending_transactions.pop(0)
        tid = transaction.transaction_number
        self.inflight_transactions[tid] = transaction
        return transaction

    def _deactivate_transaction(self, transaction):
        assert self.state_lock.locked()
        tid = transaction.transaction_number
        try:
            self.inflight_transactions.pop(tid)
        except KeyError:
            self.debugout('Transaction %r not in active list', 1, transaction)
            return False
        else:
            return True

    def _recruit_channel(self):
        assert self.state_lock.locked()
        if not self.free_channels:
            self._manage_channels()
            channel = None
        if self.free_channels:
            channel = self.free_channels.pop()
            cid = channel.channel_number
            self.inactive_channels[cid] = channel
        return channel

    def _free_channel(self, channel):
        assert self.state_lock.locked()
        cid = channel.channel_number
        if self.inactive_channels.has_key(cid):
            self.inactive_channels.pop(cid)
            if channel.accepting_requests():
                self.free_channels.append(channel)

    def _activate_channel(self):
        assert self.state_lock.locked()
        if not self.inactive_channels:
            channel = self._recruit_channel()
        else:
            cid, channel = self.inactive_channels.popitem()
        if channel is not None:
            self.active_channels[channel.channel_number] = channel
        return channel

    def _deactivate_channel(self, channel):
        assert self.state_lock.locked()
        # Changes for CSCtg33093 (b.Avoid getting the channel number, if channel is None)
        if (channel == None):
            return (False)
        cid = channel.channel_number
        try:
            self.inactive_channels[cid] = self.active_channels.pop(cid)
        except KeyError:
            self.debugout('Channel %r not in active list', 1, channel)
            return False
        else:
            if not channel.accepting_requests():
                self.inactive_channels.pop(cid)
                self.parallel_channels.decrement()
                if not channel.is_closed():
                    channel.close()
                self.debugout('Channel %r decommissioned.', 1, channel)
            return True

    def _manage_channels(self):
        assert self.state_lock.locked()
        for cid, channel in self.active_channels.items():
            message = 'Location channel management removing %r from active.'
            if not channel.accepting_requests():
                self.debugout(message % channel, 1)
                self.active_channels.pop(cid)
                self.parallel_channels.decrement()
        maxchannels = self.MAXPARALLELCHANNELS
        numpending = len(self.pending_transactions)
        numchannels = self.parallel_channels.value
        if numchannels == 0:
            createchannel = True
        elif numchannels >= maxchannels:
            createchannel = False
        elif (numpending / numchannels) > self.IDEALBACKLOG:
            createchannel = True
        else:
            createchannel = False
        if createchannel:
            channel = Channel(self.monitor, self.debug)
            channel.setup_connection(self.host, self.port, self.protocol)
            self.free_channels.append(channel)
            self.parallel_channels.increment()
        return

    def msglog(self, message, mtype=msglog.types.INFO, autoprefix=False):
        if autoprefix:
            message = '%s %s' % (self, message)
        msglog.log('broadway', mtype, message)

    def debugout(self, message, debuglevel=1, *args):
        if debuglevel <= self.debug:
            if self not in args:
                if message.count('%') == len(args):
                    message = '%s ' + message
                args = (self, ) + args
            return self.msglog(message % args, msglog.types.DB)

    def __repr__(self):
        status = ['Controller']
        status.append('[%s:%d]' % (self.host, self.port))
        channels = self.parallel_channels.value
        active = len(self.active_channels)
        inactive = len(self.inactive_channels)
        free = len(self.free_channels)
        pending = len(self.pending_transactions)
        inflight = len(self.inflight_transactions)
        channelmsg = '%d Channels (%dA %dI %dF)'
        status.append(channelmsg % (channels, active, inactive, free))
        status.append('Transactions (%dA %dP)' % (inflight, pending))
        return '<%s>' % (' '.join(status))

    def __str__(self):
        classname = self.__class__.__name__
        channels = self.parallel_channels.value
        status = '(%d => %s:%d)' % (channels, self.host, self.port)
        return '%s %s' % (classname, status)
Esempio n. 8
0
 def __init__(self, monitor, debug=0):
     self.monitor = monitor
     self.debug = debug
     self.controllers = Dictionary()
     self.controller_lock = Lock()
     super(TransactionManager, self).__init__()