class Patrol(object): def __init__(self, configFile, identifier='default', sync_frequency=15.0, logging_dest='/dev/log', realm='global', scale=None, actorsRoot=None): self._stopEvent = gevent.event.Event() self._logger = None self._log_level = logging.INFO self._log_dest = logging_dest self._realm = realm self._initLogging(self._log_level, logging_dest) self._threads = gevent.pool.Group() self._owner = 'beach.patrol/%s' % (identifier, ) self._entries = OrderedDict() self._watch = {} self._freq = sync_frequency self._beach = Beach(configFile, realm=realm) self._scale = scale self._actorsRoot = actorsRoot if self._actorsRoot is not None and not self._actorsRoot.endswith('/'): self._actorsRoot += '/' def _initLogging(self, level, dest): self._logger = logging.getLogger() self._logger.setLevel(level) handler = logging.handlers.SysLogHandler(address=dest) handler.setFormatter(logging.Formatter("%(asctime)-15s %(message)s")) self._logger.addHandler(handler) def _log(self, msg): self._logger.info('%s : %s', self.__class__.__name__, msg) def _logCritical(self, msg): self._logger.error('%s : %s', self.__class__.__name__, msg) def _scanForExistingActors(self): tally = {} mtd = self._beach.getAllNodeMetadata() for node_mtd in mtd.itervalues(): if node_mtd is False: continue for aid, actor_mtd in node_mtd.get('data', {}).get('mtd', {}).iteritems(): if self._stopEvent.wait(0): break owner = actor_mtd.get('owner', None) if owner in self._entries: # Looks like a version of that actor was maintained by us before # so we'll add it to our roster. self._watch[aid] = self._entries[owner] self._log('adding pre-existing actor %s to patrol' % aid) tally.setdefault(self._entries[owner].name, 0) tally[self._entries[owner].name] += 1 return tally def _initializeMissingActors(self, existing): if type(self._scale) is int: currentScale = self._scale elif self._scale is not None: currentScale = self._scale() else: currentScale = None for actorEntry in self._entries.itervalues(): if self._stopEvent.wait(0): break actorName = actorEntry.name current = existing.get(actorName, 0) targetNum = actorEntry.initialInstances if currentScale is not None and actorEntry.scalingFactor is not None: targetNum = int(currentScale / actorEntry.scalingFactor) if 0 != (currentScale % actorEntry.scalingFactor): targetNum += 1 if actorEntry.maxInstances is not None and targetNum > actorEntry.maxInstances: targetNum = actor.maxInstances if actorEntry.initialInstances is not None and targetNum < actorEntry.initialInstances: targetNum = actorEntry.initialInstances self._log('actor %s scale %s / factor %s: %d' % (actorName, currentScale, actorEntry.scalingFactor, targetNum)) if current < targetNum: newOwner = '%s/%s' % (self._owner, actorName) self._log( 'actor %s has %d instances but requires %d, spawning' % (actorName, current, targetNum)) for _ in range(targetNum - current): status = self._beach.addActor(*(actorEntry.actorArgs[0]), **(actorEntry.actorArgs[1])) self._log('actor launched: %s' % status) if type(status) is dict and status.get('status', {}).get( 'success', False): self._watch[status['data']['uid']] = actorEntry else: self._log('actor %s is satisfied' % actorName) def start(self): self._stopEvent.clear() self._log('starting, patrolling %d actors' % len(self._entries)) self._log('discovering pre-existing actors') existing = self._scanForExistingActors() if self._stopEvent.wait(0): return self._log('%d pre-existing actors' % len(existing)) self._initializeMissingActors(existing) if self._stopEvent.wait(0): return self._log('starting patrol') gevent.sleep(10) self._threads.add(gevent.spawn(self._sync)) def stop(self): self._log('stopping patrol') self._stopEvent.set() self._threads.join(timeout=30) self._threads.kill(timeout=10) self._log('patrol stopped') def monitor(self, name, initialInstances, maxInstances=None, scalingFactor=None, relaunchOnFailure=True, onFailureCall=None, actorArgs=[], actorKwArgs={}): actorArgs = list(actorArgs) if self._actorsRoot is not None: actorArgs = [self._actorsRoot + actorArgs[0]] + actorArgs[1:] record = _PatrolEntry() record.name = name record.initialInstances = initialInstances record.maxInstances = maxInstances record.scalingFactor = scalingFactor record.relaunchOnFailure = relaunchOnFailure record.onFailureCall = onFailureCall actorKwArgs['owner'] = '%s/%s' % (self._owner, name) record.actorArgs = (actorArgs, actorKwArgs) self._entries['%s/%s' % (self._owner, name)] = record def _processFallenActor(self, actorEntry): isRelaunch = False if actorEntry.relaunchOnFailure: self._log('actor is set to relaunch on failure') status = self._beach.addActor(*(actorEntry.actorArgs[0]), **(actorEntry.actorArgs[1])) if status is not False and status is not None and 'data' in status and 'uid' in status[ 'data']: self._watch[status['data']['uid']] = actorEntry self._log('actor relaunched: %s' % status) isRelaunch = True else: self._log('failed to launch actor: %s' % status) else: self._log('actor is not set to relaunch on failure') return isRelaunch def _sync(self): while not self._stopEvent.wait(self._freq): self._log('running sync') directory = self._beach.getDirectory(timeout=120) if type(directory) is not dict: self._logCritical('error getting directory') continue self._log('found %d actors, testing for %d' % (len(directory['reverse']), len(self._watch))) for actorId in self._watch.keys(): if self._stopEvent.wait(0): break if actorId not in directory.get('reverse', {}): self._log('actor %s has fallen' % actorId) if self._processFallenActor(self._watch[actorId]): del (self._watch[actorId]) def remove(self, name=None, isStopToo=True): removed = [] if name is not None: k = '%s/%s' % (self._owner, name) if k not in self._entries: return False record = self._entries[k] for uid, entry in self._watch.items(): if entry == record: del (self._watch[uid]) removed.append(uid) if isStopToo: self._beach.stopActors(withId=removed) else: if self._beach.stopActors(withId=self._watch.keys()): removed = self._watch.keys() self._watch = {} return removed def loadFromUrl(self, url): if '://' in url: patrolFilePath = url if patrolFilePath.startswith('file://'): patrolFilePath = 'file://%s' % os.path.abspath( patrolFilePath[len('file://'):]) patrolFile = urllib2.urlopen(patrolFilePath) else: patrolFilePath = os.path.abspath(url) patrolFile = open(patrolFilePath, 'r') exec(patrolFile.read(), { 'Patrol': self.monitor, '__file__': patrolFilePath })
class BeachShell ( cmd.Cmd ): intro = 'Welcome to Beach shell. Type help or ? to list commands.\n' prompt = '(beach) ' def __init__( self, configFile, realm = None ): cmd.Cmd.__init__( self ) self.realm = 'global' self.updatePrompt() self.beach = Beach( configFile ) def updatePrompt( self ): if self.realm is not None: self.prompt = '(beach/%s) ' % self.realm else: self.prompt = '(beach/global) ' def parse( self, parser, line ): try: return parser.parse_args( shlex.split( line ) ) except SystemExit: return None def do_exit( self, s ): self.beach.close() return True def do_quit( self, s ): self.beach.close() return True def emptyline( self ): pass def printOut( self, data ): print( json.dumps( data, indent = 4 ) ) @report_errors def do_gen_key( self, s ): '''Generate a key that can be used as a beach private key.''' parser = argparse.ArgumentParser( prog = inspect.stack()[0][3][ 3 : ] ) parser.add_argument( 'out', type = str, help = 'the path where to store the key.' ) arguments = self.parse( parser, s ) if arguments is None: return with open( arguments.out, 'w' ) as f: f.write( M2Crypto.Rand.rand_bytes( 0x20 ) ) self.printOut( 'New private key written to %s.' % arguments.out ) @report_errors def do_realm( self, s ): '''Login as a specific user.''' parser = argparse.ArgumentParser( prog = inspect.stack()[0][3][ 3 : ] ) parser.add_argument( 'realm', type = str, default = 'global', help = 'switch context to a specific realm.' ) arguments = self.parse( parser, s ) if arguments is None: return self.realm = arguments.realm if self.realm is None or self.realm.strip() == '': self.ream = 'global' self.updatePrompt() self.beach.setRealm( self.realm ) @report_errors def do_get_dir( self, s ): '''Retrieve the directory of all Actors.''' parser = argparse.ArgumentParser( prog = inspect.stack()[0][3][ 3 : ] ) parser.add_argument( '-c', '--category', type = str, dest = 'category', default = None, help = 'only show the directory for a specific category.' ) arguments = self.parse( parser, s ) if arguments is None: return category = arguments.category resp = self.beach.getDirectory() wanted = False if resp is not False and 'realms' in resp: wanted = resp[ 'realms' ].get( self.realm, {} ) if category is not None: wanted = wanted.get( category, {} ) self.printOut( wanted ) @report_errors def do_flush( self, s ): '''Remove all Actors from all nodes in the cluster.''' parser = argparse.ArgumentParser( prog = inspect.stack()[0][3][ 3 : ] ) parser.add_argument( '--confirm', action = 'store_true', help = 'This command flushes ALL ACTORS from the cluster REGARDLESS of the realm. ' 'Add this flag to confirm you understand this.' ) arguments = self.parse( parser, s ) if arguments is None: return resp = 'Please confirm ( see command help )' if arguments.confirm: resp = self.beach.flush() self.printOut( resp ) @report_errors def do_add_actor( self, s ): '''Add a new Actor to the cluster.''' parser = argparse.ArgumentParser( prog = inspect.stack()[0][3][ 3 : ] ) parser.add_argument( '-n', '--name', type = str, dest = 'name', required = True, help = 'the name of the actor to spawn.' ) parser.add_argument( '-c', '--category', type = str, dest = 'category', required = True, nargs = '+', help = 'category or categories to add the Actor to.' ) parser.add_argument( '-s', '--strategy', type = str, dest = 'strategy', default = None, help = 'the strategy to use to spawn the actor in the beach.' ) parser.add_argument( '-sh', '--hint', type = str, dest = 'strat_hint', default = None, help = 'hint used as part of some strategies.' ) parser.add_argument( '-p', '--params', type = json.loads, dest = 'params', default = None, help = 'parameters to provide to the Actor, as a JSON string.' ) parser.add_argument( '-i', '--isisolated', dest = 'isIsolated', default = False, action = 'store_true', help = 'if the Actor should be started in isolation mode (standalone process).' ) parser.add_argument( '-id', '--ident', type = str, dest = 'ident', default = None, help = 'identifier secret token used for Actor trust model.' ) parser.add_argument( '-t', '--trusted', type = str, dest = 'trusted', default = [], action = 'append', help = 'identifier token trusted by the Actor trust model.' ) parser.add_argument( '-ll', '--log-level', type = str, dest = 'loglevel', default = None, help = 'custom logging level for actor.' ) parser.add_argument( '-ld', '--log-dest', type = str, dest = 'logdest', default = None, help = 'custom logging destination for actor.' ) parser.add_argument( '-o', '--concurrent', type = int, dest = 'n_concurrent', required = False, default = 1, help = 'the number of concurrent requests handled by the actor.' ) arguments = self.parse( parser, s ) if arguments is None: return resp = self.beach.addActor( arguments.name, arguments.category, strategy = arguments.strategy, strategy_hint = arguments.strat_hint, parameters = arguments.params, isIsolated = arguments.isIsolated, secretIdent = arguments.ident, trustedIdents = arguments.trusted, n_concurrent = arguments.n_concurrent, log_level = arguments.log_level, log_dest = arguments.log_dest ) self.printOut( resp ) @report_errors def do_stop_actor( self, s ): '''Stop a specific set of actors.''' parser = argparse.ArgumentParser( prog = inspect.stack()[0][3][ 3 : ] ) parser.add_argument( '-i', '--id', type = str, dest = 'id', required = False, nargs = '+', help = 'the IDs of actors to stop.' ) parser.add_argument( '-c', '--category', type = str, dest = 'cat', required = False, nargs = '+', help = 'the categories of actors to stop.' ) arguments = self.parse( parser, s ) if arguments is None: return if arguments.id is None and arguments.cat is None: argparse.error( 'Must specify one of -i or -c.' ) resp = self.beach.stopActors( withId = arguments.id, withCategory = arguments.cat ) self.printOut( resp ) @report_errors def do_get_cluster_health( self, s ): '''Retrieve the health information of all nodes of the cluster.''' parser = argparse.ArgumentParser( prog = inspect.stack()[0][3][ 3 : ] ) arguments = self.parse( parser, s ) if arguments is None: return resp = self.beach.getClusterHealth() self.printOut( resp ) @report_errors def do_get_load_info( self, s ): '''Retrieve the number of free handlers per actor.''' parser = argparse.ArgumentParser( prog = inspect.stack()[0][3][ 3 : ] ) arguments = self.parse( parser, s ) if arguments is None: return resp = self.beach.getLoadInfo() self.printOut( resp ) @report_errors def do_get_mtd( self, s ): '''Retrieve metadata from all nodes.''' parser = argparse.ArgumentParser( prog = inspect.stack()[0][3][ 3 : ] ) resp = self.beach.getAllNodeMetadata() self.printOut( resp ) @report_errors def do_remove_from_category( self, s ): '''Remove an Actor from a category.''' parser = argparse.ArgumentParser( prog = inspect.stack()[0][3][ 3 : ] ) parser.add_argument( '-i', '--id', type = str, dest = 'id', required = True, help = 'the ID of the actor to add to the category.' ) parser.add_argument( '-c', '--category', type = str, dest = 'category', required = True, help = 'category to add the actor to.' ) arguments = self.parse( parser, s ) if arguments is None: return resp = self.beach.removeFromCategory( arguments.id, arguments.category ) self.printOut( resp ) @report_errors def do_add_to_category( self, s ): '''Add an Actor to a category.''' parser = argparse.ArgumentParser( prog = inspect.stack()[0][3][ 3 : ] ) parser.add_argument( '-i', '--id', type = str, dest = 'id', required = True, help = 'the ID of the actor to add to the category.' ) parser.add_argument( '-c', '--category', type = str, dest = 'category', required = True, help = 'category to add the actor to.' ) arguments = self.parse( parser, s ) if arguments is None: return resp = self.beach.addToCategory( arguments.id, arguments.category ) self.printOut( resp )
class BeachShell(cmd.Cmd): intro = 'Welcome to Beach shell. Type help or ? to list commands.\n' prompt = '(beach) ' def __init__(self, configFile, realm=None): cmd.Cmd.__init__(self) self.realm = 'global' self.updatePrompt() self.beach = Beach(configFile) def updatePrompt(self): if self.realm is not None: self.prompt = '(beach/%s) ' % self.realm else: self.prompt = '(beach/global) ' def parse(self, parser, line): try: return parser.parse_args(shlex.split(line)) except SystemExit: return None def do_exit(self, s): self.beach.close() return True def do_quit(self, s): self.beach.close() return True def emptyline(self): pass def printOut(self, data): print(json.dumps(data, indent=4)) @report_errors def do_gen_key(self, s): '''Generate a key that can be used as a beach private key.''' parser = argparse.ArgumentParser(prog=inspect.stack()[0][3][3:]) parser.add_argument('out', type=str, help='the path where to store the key.') arguments = self.parse(parser, s) if arguments is None: return with open(arguments.out, 'w') as f: f.write(M2Crypto.Rand.rand_bytes(0x20)) self.printOut('New private key written to %s.' % arguments.out) @report_errors def do_realm(self, s): '''Login as a specific user.''' parser = argparse.ArgumentParser(prog=inspect.stack()[0][3][3:]) parser.add_argument('realm', type=str, default='global', help='switch context to a specific realm.') arguments = self.parse(parser, s) if arguments is None: return self.realm = arguments.realm if self.realm is None or self.realm.strip() == '': self.ream = 'global' self.updatePrompt() self.beach.setRealm(self.realm) @report_errors def do_get_dir(self, s): '''Retrieve the directory of all Actors.''' parser = argparse.ArgumentParser(prog=inspect.stack()[0][3][3:]) parser.add_argument( '-c', '--category', type=str, dest='category', default=None, help='only show the directory for a specific category.') arguments = self.parse(parser, s) if arguments is None: return category = arguments.category resp = self.beach.getDirectory() wanted = False if resp is not False and 'realms' in resp: wanted = resp['realms'].get(self.realm, {}) if category is not None: wanted = wanted.get(category, {}) self.printOut(wanted) @report_errors def do_flush(self, s): '''Remove all Actors from all nodes in the cluster.''' parser = argparse.ArgumentParser(prog=inspect.stack()[0][3][3:]) parser.add_argument( '--confirm', action='store_true', help= 'This command flushes ALL ACTORS from the cluster REGARDLESS of the realm. ' 'Add this flag to confirm you understand this.') arguments = self.parse(parser, s) if arguments is None: return resp = 'Please confirm ( see command help )' if arguments.confirm: resp = self.beach.flush() self.printOut(resp) @report_errors def do_add_actor(self, s): '''Add a new Actor to the cluster.''' parser = argparse.ArgumentParser(prog=inspect.stack()[0][3][3:]) parser.add_argument('-n', '--name', type=str, dest='name', required=True, help='the name of the actor to spawn.') parser.add_argument('-c', '--category', type=str, dest='category', required=True, nargs='+', help='category or categories to add the Actor to.') parser.add_argument( '-s', '--strategy', type=str, dest='strategy', default=None, help='the strategy to use to spawn the actor in the beach.') parser.add_argument('-sh', '--hint', type=str, dest='strat_hint', default=None, help='hint used as part of some strategies.') parser.add_argument( '-p', '--params', type=json.loads, dest='params', default=None, help='parameters to provide to the Actor, as a JSON string.') parser.add_argument( '-i', '--isisolated', dest='isIsolated', default=False, action='store_true', help= 'if the Actor should be started in isolation mode (standalone process).' ) parser.add_argument( '-id', '--ident', type=str, dest='ident', default=None, help='identifier secret token used for Actor trust model.') parser.add_argument( '-t', '--trusted', type=str, dest='trusted', default=[], action='append', help='identifier token trusted by the Actor trust model.') parser.add_argument('-ll', '--log-level', type=str, dest='loglevel', default=None, help='custom logging level for actor.') parser.add_argument('-ld', '--log-dest', type=str, dest='logdest', default=None, help='custom logging destination for actor.') parser.add_argument( '-o', '--concurrent', type=int, dest='n_concurrent', required=False, default=1, help='the number of concurrent requests handled by the actor.') parser.add_argument( '-d', '--isdrainable', dest='isDrainable', default=False, action='store_true', help='if the Actor can be requested to drain gracefully.') arguments = self.parse(parser, s) if arguments is None: return resp = self.beach.addActor(arguments.name, arguments.category, strategy=arguments.strategy, strategy_hint=arguments.strat_hint, parameters=arguments.params, isIsolated=arguments.isIsolated, secretIdent=arguments.ident, trustedIdents=arguments.trusted, n_concurrent=arguments.n_concurrent, is_drainable=arguments.isDrainable, log_level=arguments.log_level, log_dest=arguments.log_dest) self.printOut(resp) @report_errors def do_stop_actor(self, s): '''Stop a specific set of actors.''' parser = argparse.ArgumentParser(prog=inspect.stack()[0][3][3:]) parser.add_argument('-i', '--id', type=str, dest='id', required=False, nargs='+', help='the IDs of actors to stop.') parser.add_argument('-c', '--category', type=str, dest='cat', required=False, nargs='+', help='the categories of actors to stop.') parser.add_argument( '-d', '--delay', type=int, dest='delay', required=False, default=None, help='the number of seconds between stopping each actor.') arguments = self.parse(parser, s) if arguments is None: return if arguments.id is None and arguments.cat is None: argparse.error('Must specify one of -i or -c.') resp = self.beach.stopActors(withId=arguments.id, withCategory=arguments.cat, delay=arguments.delay) self.printOut(resp) @report_errors def do_get_cluster_health(self, s): '''Retrieve the health information of all nodes of the cluster.''' parser = argparse.ArgumentParser(prog=inspect.stack()[0][3][3:]) arguments = self.parse(parser, s) if arguments is None: return resp = self.beach.getClusterHealth() self.printOut(resp) @report_errors def do_get_load_info(self, s): '''Retrieve the number of free handlers per actor.''' parser = argparse.ArgumentParser(prog=inspect.stack()[0][3][3:]) arguments = self.parse(parser, s) if arguments is None: return resp = self.beach.getLoadInfo() self.printOut(resp) @report_errors def do_get_mtd(self, s): '''Retrieve metadata from all nodes.''' parser = argparse.ArgumentParser(prog=inspect.stack()[0][3][3:]) resp = self.beach.getAllNodeMetadata() self.printOut(resp) @report_errors def do_remove_from_category(self, s): '''Remove an Actor from a category.''' parser = argparse.ArgumentParser(prog=inspect.stack()[0][3][3:]) parser.add_argument('-i', '--id', type=str, dest='id', required=True, help='the ID of the actor to add to the category.') parser.add_argument('-c', '--category', type=str, dest='category', required=True, help='category to add the actor to.') arguments = self.parse(parser, s) if arguments is None: return resp = self.beach.removeFromCategory(arguments.id, arguments.category) self.printOut(resp) @report_errors def do_add_to_category(self, s): '''Add an Actor to a category.''' parser = argparse.ArgumentParser(prog=inspect.stack()[0][3][3:]) parser.add_argument('-i', '--id', type=str, dest='id', required=True, help='the ID of the actor to add to the category.') parser.add_argument('-c', '--category', type=str, dest='category', required=True, help='category to add the actor to.') arguments = self.parse(parser, s) if arguments is None: return resp = self.beach.addToCategory(arguments.id, arguments.category) self.printOut(resp)
class BeachShell(cmd.Cmd): intro = "Welcome to Beach shell. Type help or ? to list commands.\n" prompt = "(beach) " def __init__(self, configFile): cmd.Cmd.__init__(self) self.realm = "global" self.updatePrompt() self.beach = Beach(configFile) def updatePrompt(self): if self.realm is not None: self.prompt = "(beach/%s) " % self.realm else: self.prompt = "(beach/global) " def parse(self, parser, line): try: return parser.parse_args(shlex.split(line)) except SystemExit: return None def do_exit(self, s): self.beach.close() return True def do_quit(self, s): self.beach.close() return True def emptyline(self): pass def printOut(self, data): print(json.dumps(data, indent=4)) @report_errors def do_realm(self, s): """Login as a specific user.""" parser = argparse.ArgumentParser(prog=inspect.stack()[0][3][3:]) parser.add_argument("realm", type=str, default="global", help="switch context to a specific realm.") arguments = self.parse(parser, s) if arguments is None: return self.realm = arguments.realm if self.realm is None or self.realm.strip() == "": self.ream = "global" self.updatePrompt() self.beach.setRealm(self.realm) @report_errors def do_get_dir(self, s): """Retrieve a specific user's profile by UID.""" parser = argparse.ArgumentParser(prog=inspect.stack()[0][3][3:]) parser.add_argument( "-c", "--category", type=str, dest="category", default=None, help="only show the directory for a specific category.", ) arguments = self.parse(parser, s) if arguments is None: return category = arguments.category resp = self.beach.getDirectory() wanted = False if isMessageSuccess(resp) and "realms" in resp: wanted = resp["realms"].get(self.realm, {}) if category is not None: wanted = wanted.get(category, {}) self.printOut(wanted) @report_errors def do_flush(self, s): """Retrieve a specific user's profile by UID.""" parser = argparse.ArgumentParser(prog=inspect.stack()[0][3][3:]) parser.add_argument( "--confirm", action="store_true", help="This command flushes ALL ACTORS from the cluster REGARDLESS of the realm. " "Add this flag to confirm you understand this.", ) arguments = self.parse(parser, s) if arguments is None: return resp = "Please confirm ( see command help )" if arguments.confirm: resp = self.beach.flush() self.printOut(resp) @report_errors def do_add_actor(self, s): """Retrieve a specific user's profile by UID.""" parser = argparse.ArgumentParser(prog=inspect.stack()[0][3][3:]) parser.add_argument( "-n", "--name", type=str, dest="name", required=True, help="the name of the actor to spawn." ) parser.add_argument( "-c", "--category", type=str, dest="category", required=True, help="only show the directory for a specific category.", ) parser.add_argument( "-s", "--strategy", type=str, dest="strategy", default=None, help="the strategy to use to spawn the actor in the beach.", ) parser.add_argument( "-sh", "--hint", type=str, dest="strat_hint", default=None, help="hint used as part of some strategies." ) arguments = self.parse(parser, s) if arguments is None: return resp = self.beach.addActor(arguments.name, arguments.category, arguments.strategy, arguments.strat_hint) self.printOut(resp) @report_errors def do_stop_actor(self, s): """Stop a specific set of actors.""" parser = argparse.ArgumentParser(prog=inspect.stack()[0][3][3:]) parser.add_argument( "-i", "--id", type=str, dest="id", required=False, nargs="+", help="the IDs of actors to stop." ) parser.add_argument( "-c", "--category", type=str, dest="cat", required=False, nargs="+", help="the categories of actors to stop.", ) arguments = self.parse(parser, s) if arguments is None: return if arguments.id is None and arguments.cat is None: argparse.error("Must specify one of -i or -c.") resp = self.beach.stopActors(withId=arguments.id, withCategory=arguments.cat) self.printOut(resp) @report_errors def do_get_cluster_health(self, s): """Retrieve the health information of all nodes of the cluster.""" parser = argparse.ArgumentParser(prog=inspect.stack()[0][3][3:]) arguments = self.parse(parser, s) if arguments is None: return resp = self.beach.getClusterHealth() self.printOut(resp)
class Patrol ( object ): def __init__( self, configFile, identifier = 'default', sync_frequency = 15.0, logging_dest = '/dev/log', realm = 'global', scale = None, actorsRoot = None ): self._stopEvent = gevent.event.Event() self._logger = None self._log_level = logging.INFO self._log_dest = logging_dest self._realm = realm self._initLogging( self._log_level, logging_dest ) self._threads = gevent.pool.Group() self._owner = 'beach.patrol/%s' % ( identifier, ) self._entries = OrderedDict() self._watch = {} self._freq = sync_frequency self._beach = Beach( configFile, realm = realm ) self._scale = scale self._actorsRoot = actorsRoot if self._actorsRoot is not None and not self._actorsRoot.endswith( '/' ): self._actorsRoot += '/' def _initLogging( self, level, dest ): self._logger = logging.getLogger() self._logger.setLevel( level ) handler = logging.handlers.SysLogHandler( address = dest ) handler.setFormatter( logging.Formatter( "%(asctime)-15s %(message)s" ) ) self._logger.addHandler( handler ) def _log( self, msg ): self._logger.info( '%s : %s', self.__class__.__name__, msg ) def _logCritical( self, msg ): self._logger.error( '%s : %s', self.__class__.__name__, msg ) def _scanForExistingActors( self ): tally = {} mtd = self._beach.getAllNodeMetadata() for node_mtd in mtd.itervalues(): if node_mtd is False: continue for aid, actor_mtd in node_mtd.get( 'data', {} ).get( 'mtd', {} ).iteritems(): if self._stopEvent.wait( 0 ): break owner = actor_mtd.get( 'owner', None ) if owner in self._entries: # Looks like a version of that actor was maintained by us before # so we'll add it to our roster. self._watch[ aid ] = self._entries[ owner ] self._log( 'adding pre-existing actor %s to patrol' % aid ) tally.setdefault( self._entries[ owner ].name, 0 ) tally[ self._entries[ owner ].name ] += 1 return tally def _initializeMissingActors( self, existing ): if type( self._scale ) is int: currentScale = self._scale elif self._scale is not None: currentScale = self._scale() else: currentScale = None for actorEntry in self._entries.itervalues(): if self._stopEvent.wait( 0 ): break actorName = actorEntry.name current = existing.get( actorName, 0 ) targetNum = actorEntry.initialInstances if currentScale is not None and actorEntry.scalingFactor is not None: targetNum = int( currentScale / actorEntry.scalingFactor ) if 0 != ( currentScale % actorEntry.scalingFactor ): targetNum += 1 if actorEntry.maxInstances is not None and targetNum > actorEntry.maxInstances: targetNum = actor.maxInstances if actorEntry.initialInstances is not None and targetNum < actorEntry.initialInstances: targetNum = actorEntry.initialInstances self._log( 'actor %s scale %s / factor %s: %d' % ( actorName, currentScale, actorEntry.scalingFactor, targetNum ) ) if current < targetNum: newOwner = '%s/%s' % ( self._owner, actorName ) self._log( 'actor %s has %d instances but requires %d, spawning' % ( actorName, current, targetNum ) ) for _ in range( targetNum - current ): status = self._beach.addActor( *(actorEntry.actorArgs[ 0 ]), **(actorEntry.actorArgs[ 1 ]) ) self._log( 'actor launched: %s' % status ) if type( status ) is dict and status.get( 'status', {} ).get( 'success', False ): self._watch[ status[ 'data' ][ 'uid' ] ] = actorEntry else: self._log( 'actor %s is satisfied' % actorName ) def start( self ): self._stopEvent.clear() self._log( 'starting, patrolling %d actors' % len( self._entries ) ) self._log( 'discovering pre-existing actors' ) existing = self._scanForExistingActors() if self._stopEvent.wait( 0 ): return self._log( '%d pre-existing actors' % len( existing ) ) self._initializeMissingActors( existing ) if self._stopEvent.wait( 0 ): return self._log( 'starting patrol' ) gevent.sleep(10) self._threads.add( gevent.spawn( self._sync ) ) def stop( self ): self._log( 'stopping patrol' ) self._stopEvent.set() self._threads.join( timeout = 30 ) self._threads.kill( timeout = 10 ) self._log( 'patrol stopped' ) def monitor( self, name, initialInstances, maxInstances = None, scalingFactor = None, relaunchOnFailure = True, onFailureCall = None, actorArgs = [], actorKwArgs = {} ): actorArgs = list( actorArgs ) if self._actorsRoot is not None: actorArgs = [ self._actorsRoot + actorArgs[ 0 ] ] + actorArgs[ 1 : ] record = _PatrolEntry() record.name = name record.initialInstances = initialInstances record.maxInstances = maxInstances record.scalingFactor = scalingFactor record.relaunchOnFailure = relaunchOnFailure record.onFailureCall = onFailureCall actorKwArgs[ 'owner' ] = '%s/%s' % ( self._owner, name ) record.actorArgs = ( actorArgs, actorKwArgs ) self._entries[ '%s/%s' % ( self._owner, name ) ] = record def _processFallenActor( self, actorEntry ): isRelaunch = False if actorEntry.relaunchOnFailure: self._log( 'actor is set to relaunch on failure' ) status = self._beach.addActor( *(actorEntry.actorArgs[ 0 ]), **(actorEntry.actorArgs[ 1 ]) ) if status is not False and status is not None and 'data' in status and 'uid' in status[ 'data' ]: self._watch[ status[ 'data' ][ 'uid' ] ] = actorEntry self._log( 'actor relaunched: %s' % status ) isRelaunch = True else: self._log( 'failed to launch actor: %s' % status ) else: self._log( 'actor is not set to relaunch on failure' ) return isRelaunch def _sync( self ): while not self._stopEvent.wait( self._freq ): self._log( 'running sync' ) directory = self._beach.getDirectory( timeout = 120 ) if type( directory ) is not dict: self._logCritical( 'error getting directory' ) continue self._log( 'found %d actors, testing for %d' % ( len( directory[ 'reverse' ] ), len( self._watch ) ) ) for actorId in self._watch.keys(): if self._stopEvent.wait( 0 ): break if actorId not in directory.get( 'reverse', {} ): self._log( 'actor %s has fallen' % actorId ) if self._processFallenActor( self._watch[ actorId ] ): del( self._watch[ actorId ] ) def remove( self, name = None, isStopToo = True ): removed = [] if name is not None: k ='%s/%s' % ( self._owner, name ) if k not in self._entries: return False record = self._entries[ k ] for uid, entry in self._watch.items(): if entry == record: del( self._watch[ uid ] ) removed.append( uid ) if isStopToo: self._beach.stopActors( withId = removed ) else: if self._beach.stopActors( withId = self._watch.keys() ): removed = self._watch.keys() self._watch = {} return removed def loadFromUrl( self, url ): if '://' in url: patrolFilePath = url if patrolFilePath.startswith( 'file://' ): patrolFilePath = 'file://%s' % os.path.abspath( patrolFilePath[ len( 'file://' ) : ] ) patrolFile = urllib2.urlopen( patrolFilePath ) else: patrolFilePath = os.path.abspath( url ) patrolFile = open( patrolFilePath, 'r' ) exec( patrolFile.read(), { 'Patrol' : self.monitor, '__file__' : patrolFilePath } )