def __init__(self, server_socket, to_eventprocessor, from_eventprocessor, pdbhelper, rootcc): """@type server_socket: ServerSocket @type to_eventprocessor: L{Queue.Queue} @type from_eventprocessor: L{Queue.Queue} @param rootcc: root counter collection @type rootcc: L{satella.instrumentation.CounterCollection}""" SelectHandlingLayer.__init__(self) # Set up the server socket and attach it to SL self.server_socket = server_socket self.register_channel(server_socket) # dictionary (user pids => respective PlayerSocket) self.authenticated_channels = {} # Queue section self.to_ep = to_eventprocessor self.from_ep = from_eventprocessor # PDB section self.pdbhelper = pdbhelper # Instrumentation section cc = CounterCollection('selectlayer') self.connections_counter = DeltaCounter('connections', units='connections', description='SSL connections to server') self.outbound_events = PulseCounter('events_to_ep', resolution=60, units='events per minute', description='events sent to EventProcessor') cc.add(self.connections_counter) cc.add(self.outbound_events) rootcc.add(cc)
def __init__(self, oxmi, recovery_message, cc, confsection): BaseThread.__init__(self) self.name = confsection['handle'] self.comport, self.baudrate = confsection['serial_port'], confsection['serial_baudrate'] self.notifier = SMTPNotifier(confsection) self.mk_serport() self.mte = None #: message to execute self.interesting_shit = Lock() self.interesting_shit.acquire() self.oxm = oxmi self.conseq_reset = 0 self.recovery_message = recovery_message self.is_attempting_recovery = False self.cc_hard_resets = PulseCounter('hard_resets', resolution=24*60*60, units='resets', description=u'Hard resets during last 24 hours') self.cc_soft_resets = PulseCounter('soft_resets', resolution=24*60*60, units='resets', description=u'Hard resets during last 24 hours') self.cc_send_time = NumericValueCounter('send_time', units='seconds', description='Time it took to send last SMS', history=20) cc.add(self.cc_hard_resets) cc.add(self.cc_soft_resets) cc.add(self.cc_send_time)
def __init__(self, host, username, password, dbname, rootcc, dbtype='mysql'): """@type rootcc: L{satella.instrumentation.CounterCollection}""" assert dbtype == 'mysql', 'I cannot support other databases!' dd = DatabaseDefinition( MySQLdb.connect, (MySQLdb.OperationalError, MySQLdb.InterfaceError), (host, username, password, dbname)) self.cp = ConnectionPool(dd) # Set up instrumentation insmgr = CounterCollection('database') self.cursors_counter = PulseCounter('cursors', resolution=60, units=u'cursors per minute', description='SQL cursors created') insmgr.add(self.cursors_counter) rootcc.add(insmgr)
def __init__(self, pdb, rootcc): """ @param pdb: PlayerDatabase @param rootcc: Root CounterCollection """ BaseThread.__init__(self) self.pdb = pdb cc = CounterCollection('match_maker', description=u'Module that organizes matches from enqueued players') self.total_matches_made = DeltaCounter('matches_made', units=u'matches', description=u'total matches made') self.matches_dt = PulseCounter('matches_made_p', resolution=60, units=u'matches per minute', description=u'organized matches per minute') cc.add(self.total_matches_made) cc.add(self.matches_dt) rootcc.add(cc)
class DatabaseManager(object): def __init__(self, host, username, password, dbname, rootcc, dbtype='mysql'): """@type rootcc: L{satella.instrumentation.CounterCollection}""" assert dbtype == 'mysql', 'I cannot support other databases!' dd = DatabaseDefinition( MySQLdb.connect, (MySQLdb.OperationalError, MySQLdb.InterfaceError), (host, username, password, dbname)) self.cp = ConnectionPool(dd) # Set up instrumentation insmgr = CounterCollection('database') self.cursors_counter = PulseCounter('cursors', resolution=60, units=u'cursors per minute', description='SQL cursors created') insmgr.add(self.cursors_counter) rootcc.add(insmgr) def query_interface(self, ifc): if ifc == SelectLayerInterface: return SelectLayerProxy(self) elif ifc == PlayerDBInterface: return PlayerDBProxy(self) else: raise ValueError, 'Unknown interface' def __call__(self): """ Use as in: with database_manager() as cur: cur.execute('I CAN DO SQL') """ self.cursors_counter.update() return self.cp.cursor()
def __init__(self, pdb, rootcc): """ @param pdb: PlayerDatabase @param rootcc: Root CounterCollection """ BaseThread.__init__(self) self.pdb = pdb cc = CounterCollection('alpha_counter', description=u'Module that creates games from hero picking rooms') self.matches_dt = PulseCounter('matches_made', resolution=60, units=u'matches per minute', description=u'successful allocations') self.matches_dt_so = PulseCounter('matches_failed_so', resolution=60, units=u'matches per minute', description=u'fails to allocate due to shard overload') self.matches_dt_fl = PulseCounter('matches_failed_fl', resolution=60, units=u'matches per minute', description=u'fails to allocate due to other reasons') cc.add(self.matches_dt) cc.add(self.matches_dt_so) cc.add(self.matches_dt_fl) rootcc.add(cc)
def __init__(self, pdb, rootcc): """ @param pdb: PlayerDatabase @param rootcc: Root CounterCollection """ BaseThread.__init__(self) self.pdb = pdb cc = CounterCollection( 'alpha_counter', description=u'Module that creates games from hero picking rooms') self.matches_dt = PulseCounter('matches_made', resolution=60, units=u'matches per minute', description=u'successful allocations') self.matches_dt_so = PulseCounter( 'matches_failed_so', resolution=60, units=u'matches per minute', description=u'fails to allocate due to shard overload') self.matches_dt_fl = PulseCounter( 'matches_failed_fl', resolution=60, units=u'matches per minute', description=u'fails to allocate due to other reasons') cc.add(self.matches_dt) cc.add(self.matches_dt_so) cc.add(self.matches_dt_fl) rootcc.add(cc)
class OpportunisticMatchMaker(BaseThread): """ A worker that determines whether a match can be organized from a queue, and it this happens, launches a transaction """ def __init__(self, pdb, rootcc): """ @param pdb: PlayerDatabase @param rootcc: Root CounterCollection """ BaseThread.__init__(self) self.pdb = pdb cc = CounterCollection('match_maker', description=u'Module that organizes matches from enqueued players') self.total_matches_made = DeltaCounter('matches_made', units=u'matches', description=u'total matches made') self.matches_dt = PulseCounter('matches_made_p', resolution=60, units=u'matches per minute', description=u'organized matches per minute') cc.add(self.total_matches_made) cc.add(self.matches_dt) rootcc.add(cc) def process(self): """Attempts to make a match""" from lobbyapp.playerdb.transactions.queues import TRMatchFound for queue in self.pdb.qmangr.get_queues(): if queue.can_make_match(): if TRMatchFound(self.pdb, queue.qname).start(): self.total_matches_made.update(+1) self.matches_dt.update() def run(self): while not self._terminating: self.process() sleep(5)
class DatabaseManager(object): def __init__(self, host, username, password, dbname, rootcc, dbtype='mysql'): """@type rootcc: L{satella.instrumentation.CounterCollection}""" assert dbtype == 'mysql', 'I cannot support other databases!' dd = DatabaseDefinition(MySQLdb.connect, (MySQLdb.OperationalError, MySQLdb.InterfaceError), (host, username, password, dbname)) self.cp = ConnectionPool(dd) # Set up instrumentation insmgr = CounterCollection('database') self.cursors_counter = PulseCounter('cursors', resolution=60, units=u'cursors per minute', description='SQL cursors created') insmgr.add(self.cursors_counter) rootcc.add(insmgr) def query_interface(self, ifc): if ifc == SelectLayerInterface: return SelectLayerProxy(self) elif ifc == PlayerDBInterface: return PlayerDBProxy(self) else: raise ValueError, 'Unknown interface' def __call__(self): """ Use as in: with database_manager() as cur: cur.execute('I CAN DO SQL') """ self.cursors_counter.update() return self.cp.cursor()
def __init__(self, host, username, password, dbname, rootcc, dbtype='mysql'): """@type rootcc: L{satella.instrumentation.CounterCollection}""" assert dbtype == 'mysql', 'I cannot support other databases!' dd = DatabaseDefinition(MySQLdb.connect, (MySQLdb.OperationalError, MySQLdb.InterfaceError), (host, username, password, dbname)) self.cp = ConnectionPool(dd) # Set up instrumentation insmgr = CounterCollection('database') self.cursors_counter = PulseCounter('cursors', resolution=60, units=u'cursors per minute', description='SQL cursors created') insmgr.add(self.cursors_counter) rootcc.add(insmgr)
def __init__(self, server_sockets, rootcc): SelectHandlingLayer.__init__(self) # process server sockets self.server_sockets = server_sockets # there may be many for server_socket in server_sockets: self.register_channel(server_socket) self.reqtasks = [] # ---------- instrumentation stuff self.c_lshardmgrs_connected = DeltaCounter('lshardmgrs_connected', description='Amount of lshardmgrs connected') self.c_requests_dispatched = PulseCounter('requests_dispatched', resolution=60, units=u'requests per minute', description='Requests to allocate a server sent') self.c_alloc_orders = PulseCounter('allocation_orders', resolution=60, units=u'allocation requests per minute', description=u'Allocation requests received') self.c_requests_success = PulseCounter('requests_successful', resolution=60, units=u'successful requests per minute', description=u'Requests successfully executed') self.c_requests_failed = PulseCounter('requests_failed', resolution=60, units=u'failed requests per minute', description=u'Failed requests') def calculate_shards_available(): a = [x for x in self.channels if x not in self.server_sockets] a = [x for x in a if x.who == 'l'] return sum([x.shards for x in a]) self.c_shards_available = CallbackCounter('shards_available', calculate_shards_available, units=u'shards', description=u'Available shards') rootcc.add(self.c_lshardmgrs_connected) rootcc.add(self.c_requests_dispatched) rootcc.add(self.c_alloc_orders) rootcc.add(self.c_requests_success) rootcc.add(self.c_requests_failed) rootcc.add(self.c_shards_available)
def __init__(self, pdbhelper, rootcc, qmangr, to_eventprocessor, cshardmgr_interface): """@type pdbhelper: L{lobbyapp.playerdb.api.PDBHelperInterface} implementation. @type qmangr: L{lobbyapp.queuemangr.QueueManager} @param to_eventprocessor: Queue that can be used to send orders to EventProcessor @type to_eventprocessor: L{Queue.Queue} @param cshardmgr_interface: interface to CShardMgr @type cshardmgr_interface: tuple(str, int)""" Monitor.__init__(self) self.pirs = {} #: pid => PlayerInformationRecord self.cshardmgr_interface = cshardmgr_interface self.pdbhelper = pdbhelper self.qmangr = qmangr self.alphas = [] # all existing AlphaMatches self.betagammas = [] # all existing BetaGammas self.to_eventprocessor = to_eventprocessor cc = CounterCollection('player_database') self.transactions_counter = PulseCounter( 'transactions', resolution=60, units=u'transactions per minute', description=u'player database transactions performed') alphas_waiting = CallbackCounter('alphas_waiting', self.alphas.__len__, description=u'matches being prepared') betagammas = CallbackCounter('betagammas', self.betagammas.__len__, description=u'matches being played') cc.add(self.transactions_counter) cc.add(alphas_waiting) cc.add(betagammas) rootcc.add(cc)
class AlphaCounter(BaseThread): """ A worker that examines existing alpha matches and makes matches if they are OK """ def __init__(self, pdb, rootcc): """ @param pdb: PlayerDatabase @param rootcc: Root CounterCollection """ BaseThread.__init__(self) self.pdb = pdb cc = CounterCollection('alpha_counter', description=u'Module that creates games from hero picking rooms') self.matches_dt = PulseCounter('matches_made', resolution=60, units=u'matches per minute', description=u'successful allocations') self.matches_dt_so = PulseCounter('matches_failed_so', resolution=60, units=u'matches per minute', description=u'fails to allocate due to shard overload') self.matches_dt_fl = PulseCounter('matches_failed_fl', resolution=60, units=u'matches per minute', description=u'fails to allocate due to other reasons') cc.add(self.matches_dt) cc.add(self.matches_dt_so) cc.add(self.matches_dt_fl) rootcc.add(cc) def process(self): """Attempts to make a match""" from lobbyapp.playerdb.transactions.start_match import TRStartMatch for alpha in [a for a in self.pdb.alphas if a.should_start_due_to_time() or a.has_everybody_locked_in()]: # This access is exempt from atomicity from lobbyapp.playerdb.transactions import TRStartMatch k = TRStartMatch(self.pdb, alpha).start() if isinstance(k, tuple): self.matches_dt.update() # SUCCESS!!! target_ip, target_port_tcp, target_port_udp = k for pid in alpha.team1+alpha.team2: mf = MatchOKAndWillStart(pid, target_ip, target_port_tcp, target_port_udp) self.pdb.to_eventprocessor.put(mf) elif isinstance(k, str): if k == 'shard overload': self.matches_dt_so.update() else: self.matches_dt_fl.update() # loud failure for pid in alpha.team1+alpha.team2: mf = MatchFAILED(pid, k) self.pdb.to_eventprocessor.put(mf) else: pass # silent failure def run(self): try: while not self._terminating: self.process() sleep(5) except: print Trackback().pretty_print()
class ExecutorThread(BaseThread): class ATCommsError(Exception): def __init__(self, instant_failure=False): self.instant_failure = instant_failure def mk_serport(self): try: self.serport except: pass else: self.serport.close() self.serport = serial.Serial(self.comport, self.baudrate, timeout=0) def __init__(self, oxmi, recovery_message, cc, confsection): BaseThread.__init__(self) self.name = confsection['handle'] self.comport, self.baudrate = confsection['serial_port'], confsection['serial_baudrate'] self.notifier = SMTPNotifier(confsection) self.mk_serport() self.mte = None #: message to execute self.interesting_shit = Lock() self.interesting_shit.acquire() self.oxm = oxmi self.conseq_reset = 0 self.recovery_message = recovery_message self.is_attempting_recovery = False self.cc_hard_resets = PulseCounter('hard_resets', resolution=24*60*60, units='resets', description=u'Hard resets during last 24 hours') self.cc_soft_resets = PulseCounter('soft_resets', resolution=24*60*60, units='resets', description=u'Hard resets during last 24 hours') self.cc_send_time = NumericValueCounter('send_time', units='seconds', description='Time it took to send last SMS', history=20) cc.add(self.cc_hard_resets) cc.add(self.cc_soft_resets) cc.add(self.cc_send_time) def on_interesting_shit(self): try: self.interesting_shit.release() except: pass def terminate(self): BaseThread.terminate(self) self.on_interesting_shit() def send_message(self, order): """Attempts sending the message. Raises ATCommsError if something failed""" sms_sent_speed = time() # switch to text mode self.serport.write('AT+CMGF=1\r') sleep(0.1) k = self.serport.read(1024) started_waiting_on = time() while not 'OK' in k: if (time() - started_waiting_on) > DASH_TIMEOUT: raise self.ATCommsError() sleep(0.1) k += self.serport.read(1024) if 'ERROR' in k: raise self.ATCommsError(True) # transforms content into msgcnt msgtext = unicode_to_sane_ascii(order.content) # clear any URCs k = self.serport.read(1024) while len(k) > 0: k = self.serport.read(1024) self.serport.write('AT+CMGS="%s"\r' % (order.target.encode('utf8'), )) sleep(0.1) k = self.serport.read(1024) started_waiting_on = time() while not '>' in k: if (time() - started_waiting_on) > DASH_TIMEOUT: raise self.ATCommsError() sleep(0.1) k += self.serport.read(1024) if 'ERROR' in k: raise self.ATCommsError(True) self.serport.write('%s\x1A' % (msgtext, )) started_waiting_on = time() k = '' while True: if (time() - started_waiting_on) > OK_TIMEOUT: raise self.ATCommsError() sleep(0.5) k += self.serport.read(1024) if '\r\nERROR\r\n' in k: raise self.ATCommsError(True) if '\n\r\nOK\r\n' in k: self.cc_send_time.update(time() - sms_sent_speed) return def run(self): self._run() if self.mte != None: self.oxm.eio.order_passback(self.mte) def _run(self): while not self._terminating: self.interesting_shit.acquire() if self.is_attempting_recovery: while not self._terminating: # We will be attempting to periodically send a SMS for i in xrange(0, 15): # delay loop sleep(3) if self._terminating: break try: self.send_message(self.recovery_message) except self.ATCommsError: # still f****d up self.attempt_recovery() else: # success! self.is_attempting_recovery = False self.oxm.eio.on_fail_status(False) self.notifier.notify('EUNIKE: %s left failed state' % (self.name, )) self.conseq_reset = 0 break if self._terminating: return elif self.mte != None: try: self.send_message(self.mte) except self.ATCommsError: if not self.attempt_recovery(): self.oxm.eio.on_fail_status(True) self.notifier.notify('EUNIKE: %s entered failed state' % (self.name, )) self.oxm.eio.on_ready() # We won't be scheduled anyway :) self.oxm.eio.on_order_failed(self.mte) self.mte = None self.is_attempting_recovery = True self.on_interesting_shit() continue else: # Try again self.on_interesting_shit() continue else: self.conseq_reset = 0 self.oxm.eio.on_sent_successfully(self.mte) self.mte = None self.oxm.eio.on_ready() def attempt_recovery(self): """Returns False when it's hopeless""" self.conseq_reset += 1 if self.conseq_reset == 7: return False elif self.conseq_reset > 4: self.attempt_hard_reset() else: self.attempt_soft_reset() return True # after conseq_reset==7 there's no need to return False def attempt_soft_reset(self): """To be used in case of transmission failure""" self.serport.write(' \x1AAT+CSQ\r\r') # Vulcan nerve pinch sleep(1) self.serport.read(2048) self.cc_soft_resets.update() def attempt_hard_reset(self): """To be used in case of hard failures""" self.serport.write(' \x1A\r\rAT+CFUN=1\r\r') sleep(5) self.serport.read(2048) self.cc_hard_resets.update() self.mk_serport()
class PlayersHandlingLayer(SelectHandlingLayer): """ The layer's tasks is to provide (login, unserialized data) or events (login, event instance) of data that could interest the upper layer (ie. filter out pings). This layer also handles connects and disconnects """ def __init__(self, server_socket, to_eventprocessor, from_eventprocessor, pdbhelper, rootcc): """@type server_socket: ServerSocket @type to_eventprocessor: L{Queue.Queue} @type from_eventprocessor: L{Queue.Queue} @param rootcc: root counter collection @type rootcc: L{satella.instrumentation.CounterCollection}""" SelectHandlingLayer.__init__(self) # Set up the server socket and attach it to SL self.server_socket = server_socket self.register_channel(server_socket) # dictionary (user pids => respective PlayerSocket) self.authenticated_channels = {} # Queue section self.to_ep = to_eventprocessor self.from_ep = from_eventprocessor # PDB section self.pdbhelper = pdbhelper # Instrumentation section cc = CounterCollection('selectlayer') self.connections_counter = DeltaCounter('connections', units='connections', description='SSL connections to server') self.outbound_events = PulseCounter('events_to_ep', resolution=60, units='events per minute', description='events sent to EventProcessor') cc.add(self.connections_counter) cc.add(self.outbound_events) rootcc.add(cc) def on_iteration(self): # Check timeouts for channel in self.channels: if channel != self.server_socket: if channel.has_expired(): self.close_channel(channel) # Process inbound commands while True: try: evt = self.from_ep.get(False) except Empty: break if isinstance(evt, SendData): # A request is made to send something to a target socket if evt.pid not in self.authenticated_channels: # User not logged in - cannot satisfy continue # Else - send it self.authenticated_channels[evt.pid].write(evt.data) def on_data_frame(self, channel, frame): """ Called by on_readable when there is a frame of data from a channel """ # If the user was just authenticated... if isinstance(frame, channel.AuthenticatedSuccessfully): self.to_ep.put(PlayerOnline(channel.pid)) self.authenticated_channels[channel.pid] = channel self.outbound_events.update() else: # Player is all clear! # Relay the data self.to_ep.put(DataArrived(channel.pid, frame)) self.outbound_events.update() def on_readable(self, channel): if channel == self.server_socket: try: self.register_channel(PlayerSocket(self.server_socket.read(), self.pdbhelper)) except DataNotAvailable: # Honest to God. Eg. SSL may fail at accept(). return self.connections_counter.update(+1) else: while True: # process all frames! try: data = channel.read() except DataNotAvailable: break # next one, please self.on_data_frame(channel, data) def on_closed(self, channel): # If server socket is closed then something is VERY wrong assert channel != self.server_socket # Signal the counters that we have a connection less self.connections_counter.update(-1) # If channel was an authenticated channel, we need to inform upstream # that the player is no longer logged in # note that if we have S1 with given user and S1 connects and presents with # the same credentials, S1 channel won't be on authenticated_channels # and it's closure won't affect user's logon status if channel in self.authenticated_channels.itervalues(): del self.authenticated_channels[channel.pid] self.to_ep.put(PlayerOffline(channel.pid)) self.outbound_events.update()
class SelectLayer(SelectHandlingLayer): """ Each attached client socket will have a property set. sock.who == None if I don't know this yet sock.who == 'l' lshardmgr in that case he has a sock.shards == (int) with the number of available shards he has sock.scheduled = (ReqTask) task scheduled right now, or None sock.who == 'c' a client requesting server in that case he has a sock.reqtask = (ReqTask) task that this client wants us to carry out """ def __init__(self, server_sockets, rootcc): SelectHandlingLayer.__init__(self) # process server sockets self.server_sockets = server_sockets # there may be many for server_socket in server_sockets: self.register_channel(server_socket) self.reqtasks = [] # ---------- instrumentation stuff self.c_lshardmgrs_connected = DeltaCounter('lshardmgrs_connected', description='Amount of lshardmgrs connected') self.c_requests_dispatched = PulseCounter('requests_dispatched', resolution=60, units=u'requests per minute', description='Requests to allocate a server sent') self.c_alloc_orders = PulseCounter('allocation_orders', resolution=60, units=u'allocation requests per minute', description=u'Allocation requests received') self.c_requests_success = PulseCounter('requests_successful', resolution=60, units=u'successful requests per minute', description=u'Requests successfully executed') self.c_requests_failed = PulseCounter('requests_failed', resolution=60, units=u'failed requests per minute', description=u'Failed requests') def calculate_shards_available(): a = [x for x in self.channels if x not in self.server_sockets] a = [x for x in a if x.who == 'l'] return sum([x.shards for x in a]) self.c_shards_available = CallbackCounter('shards_available', calculate_shards_available, units=u'shards', description=u'Available shards') rootcc.add(self.c_lshardmgrs_connected) rootcc.add(self.c_requests_dispatched) rootcc.add(self.c_alloc_orders) rootcc.add(self.c_requests_success) rootcc.add(self.c_requests_failed) rootcc.add(self.c_shards_available) def on_iteration(self): for channel in self.channels: if isinstance(channel, JSONSocket): if channel.has_expired(): self.close_channel(channel) def on_readable(self, channel): if channel in self.server_sockets: try: nc = JSONSocket(channel.read()) except DataNotAvailable: pass self.register_channel(nc) else: while True: try: elem = channel.read() except DataNotAvailable: break if channel.who == None: # this packet will ID who this is if 'request' not in elem: return if elem['request'] == 'first-login': # this is a LSHARDMGR try: channel.who = 'l' channel.shards = elem['shards'] channel.id_name = elem['id_name'] channel.scheduled = None except KeyError: self.close_channel(channel) return # do we have somebody else allocated on this id_name? for chan in self.channels: if chan == channel: continue # must not be this channel if chan in self.server_sockets: continue # must not be server socket if chan.who != 'l': continue # must be lshardmgr if chan.id_name == channel.id_name: # this is the channel that we should replace self.close_channel(chan) self.c_lshardmgrs_connected.update(+1) self.ac_on_new_lshardmgr_connected(channel) elif elem['request'] == 'allocate-shard': # this is a REQUEST try: channel.who = 'c' channel.reqtask = ReqTask(elem['gugid'], elem['bpf_chunk']) self.reqtasks.append(channel.reqtask) except KeyError: self.close_channel(channel) return self.c_alloc_orders.update() self.ac_on_alloc_request(channel, elem) elif channel.who == 'l': if 'shards' in elem: # ping response if isinstance(elem['shards'], int): channel.chards = elem['shards'] if 'response' in elem: # let's see that happened if elem['response'] == 'allocated': # success! allocated self.ac_on_allocation_success(channel, elem) self.c_requests_success.update() elif elem['response'] == 'recess': # failure! not allocate self.ac_on_allocation_recess(channel) self.c_requests_failed.update() def on_closed(self, channel): if channel in self.server_sockets: return if channel.who == 'l': # lshardmgr channel killed self.c_lshardmgrs_connected.update(-1) if channel.scheduled != None: # it it was doing something ... self.c_requests_failed.update() self.ac_on_lshard_channel_killed(channel) # .. do the appropriate # --------------------------- extra logic for allocation scheduling def hlac_alloc(self, reqtask): # pick a lshardmgr that has shards chans = [x for x in self.channels if x not in self.server_sockets] chans = [x for x in chans if x.who == 'l'] chans = [x for x in chans if x.shards > 0] if len(chans) == 0: return False # simply no shards chan = choice(chans) chan.scheduled = reqtask try: chan.write({'request': 'allocate-shard', 'bpf_chunk': reqtask.bpf_chunk, 'gugid': reqtask.gugid}) except FatalException: # chan failed! self.close_channel(chan) return True def ac_on_lshard_channel_killed(self, channel): """ a lshardmgr channel that was executing something has just been killed @param channel: the casualty """ # it's essentially the same as recess.. self.ac_on_allocation_recess(channel) def ac_on_allocation_recess(self, channel): """a lshardmgr has failed to allocate a game @param channel: lshardmgr channel that executed the query @param elem: response that the lshardmgr channel sent""" # get the task, unschedule it from channel task = channel.scheduled channel.scheduled = None # first, it's imperative to find the channel that wanted this chan = [x for x in self.channels if x not in self.server_sockets] # no server sockets chan = [x for x in chan if x.who == 'c'] # only incoming requests chan = [x for x in chan if x.reqtask == task] # filter the proper channel if len(chan) == 0: # if this is an orphaned request... return # then it's fine chan, = chan try: if not task.retry(): raise IOError if not self.hlac_alloc(task): raise IOError # attempt to reschedule except IOError: # this request will not be fulfilled chan.reqtask = None chan.write({'response': 'recess', 'gugid': task.gugid}) def ac_on_new_lshardmgr_connected(self, channel): """A new lshardmgr has just connected @param channel: that lshardmgr's channel""" pass # well, good for you! def ac_on_allocation_success(self, channel, elem): """a lshardmgr has successfully allocated a game @param channel: lshardmgr channel that executed the query""" # get the task, unschedule it from channel task = channel.scheduled channel.scheduled = None # first, it's imperative to find the channel that wanted this chan = [x for x in self.channels if x not in self.server_sockets] # no server sockets chan = [x for x in chan if x.who == 'c'] # only incoming requests chan = [x for x in chan if x.reqtask == task] # filter the proper channel if len(chan) == 0: # if this is an orphaned request... return # then it's fine chan, = chan chan.reqtask = None chan.write(elem) def ac_on_alloc_request(self, channel, elem): """a request to allocate has been received. channel is modified to contains a .reqtask field before this is called. @param channel: channel that requested it @param elem: allocation request message @return bool: True if allocation succeeded, False if it didn't """ if not self.hlac_alloc(channel.reqtask): # if that happens then there are simply NO SHARDS to allocate channel.write({'response': 'recess', 'gugid': elem['gugid']}) channel.reqtask = None self.c_requests_failed.update()
class AlphaCounter(BaseThread): """ A worker that examines existing alpha matches and makes matches if they are OK """ def __init__(self, pdb, rootcc): """ @param pdb: PlayerDatabase @param rootcc: Root CounterCollection """ BaseThread.__init__(self) self.pdb = pdb cc = CounterCollection( 'alpha_counter', description=u'Module that creates games from hero picking rooms') self.matches_dt = PulseCounter('matches_made', resolution=60, units=u'matches per minute', description=u'successful allocations') self.matches_dt_so = PulseCounter( 'matches_failed_so', resolution=60, units=u'matches per minute', description=u'fails to allocate due to shard overload') self.matches_dt_fl = PulseCounter( 'matches_failed_fl', resolution=60, units=u'matches per minute', description=u'fails to allocate due to other reasons') cc.add(self.matches_dt) cc.add(self.matches_dt_so) cc.add(self.matches_dt_fl) rootcc.add(cc) def process(self): """Attempts to make a match""" from lobbyapp.playerdb.transactions.start_match import TRStartMatch for alpha in [ a for a in self.pdb.alphas if a.should_start_due_to_time() or a.has_everybody_locked_in() ]: # This access is exempt from atomicity from lobbyapp.playerdb.transactions import TRStartMatch k = TRStartMatch(self.pdb, alpha).start() if isinstance(k, tuple): self.matches_dt.update() # SUCCESS!!! target_ip, target_port_tcp, target_port_udp = k for pid in alpha.team1 + alpha.team2: mf = MatchOKAndWillStart(pid, target_ip, target_port_tcp, target_port_udp) self.pdb.to_eventprocessor.put(mf) elif isinstance(k, str): if k == 'shard overload': self.matches_dt_so.update() else: self.matches_dt_fl.update() # loud failure for pid in alpha.team1 + alpha.team2: mf = MatchFAILED(pid, k) self.pdb.to_eventprocessor.put(mf) else: pass # silent failure def run(self): try: while not self._terminating: self.process() sleep(5) except: print Trackback().pretty_print()