def run(): # Setup the gantry arguments parser = argparse.ArgumentParser(description='gantry continuous deployment system') parser.add_argument('config_file', help = 'The configuration file') parser.add_argument('action', help = 'The action to perform', choices = ACTIONS.keys()) parser.add_argument('component_name', help = 'The name of the component to manage') parser.add_argument('-m', dest='monitor', action='store_true', help = 'If specified and the action is "start" or "update", gantry will remain running to monitor components, auto restarting them as necessary') args = parser.parse_args() component_name = args.component_name action = args.action should_monitor = args.monitor config_file = args.config_file # Load the config. config = loadConfig(config_file) if not config: return # Create the manager. manager = RuntimeManager(config) # Find the component component = manager.getComponent(component_name) if not component: raise Exception('Unknown component: ' + component_name) # Run the action with the component and config. result = ACTIONS[action](component) if result and should_monitor: report('Starting monitoring of component: ' + component_name) monitor(component)
def run(): #setup logging logging.basicConfig(level=logging.DEBUG) # Setup the gantry arguments parser = argparse.ArgumentParser(description='gantry continuous deployment system') parser.add_argument('config_file', help='The configuration file') parser.add_argument('action', help='The action to perform', choices=ACTIONS.keys()) parser.add_argument('component_name', help='The name of the component to manage') parser.add_argument('-m', dest='monitor', action='store_true', help='If specified and the action is "start" or "update", gantry will remain running to monitor components, auto restarting them as necessary') parser.add_argument('--setconfig', dest='config_overrides', action='append', help='Configuration overrides for the component') args = parser.parse_args() component_name = args.component_name action = args.action should_monitor = args.monitor config_file = args.config_file config_overrides = args.config_overrides # Load the config. config = loadConfig(config_file) if not config: return # Create the manager. manager = RuntimeManager(config) # Find the component component = manager.getComponent(component_name) if not component: raise Exception('Unknown component: ' + component_name) # Apply the config overrides (if any). if config_overrides: component.applyConfigOverrides(config_overrides) # Run the action with the component and config. result = ACTIONS[action](component) if result and should_monitor: try: report('Starting monitoring of component: ' + component_name) monitor(component) except KeyboardInterrupt: report('Terminating monitoring of component: ' + component_name) def cleanup_monitor(signum, frame): manager.join() # Set the signal handler and a 5-second alarm signal.signal(signal.SIGINT, cleanup_monitor) # We may have to call cleanup manually if we weren't asked to monitor cleanup_monitor(None, None)
def initialize(self, component_names): """ Initializes this client for working with the components given. """ # Load the project configuration. self.getConfig() # Initialize the runtime manager. self.runtime_manager = RuntimeManager(self.config) # Find all the components for this machine. for component_name in component_names: component = self.runtime_manager.getComponent(component_name) if not component: fail('Unknown component named ' + component_name, project=self.project_name) self.components.append(component)
def initialize(self, componentNames): """ Initializes this client for working with the components given. """ # Load the project configuration. self.getConfig() # Initialize the runtime manager. self.runtime_manager = RuntimeManager(self.config, daemon_mode = True) # Find all the components for this machine. for componentName in componentNames: component = self.runtime_manager.getComponent(componentName) if not component: fail('Unknown component named ' + componentName, project = self.project_name) self.components.append(component)
def run(): # Setup the gantry arguments parser = argparse.ArgumentParser( description='gantry continuous deployment system') parser.add_argument('config_file', help='The configuration file') parser.add_argument('action', help='The action to perform', choices=ACTIONS.keys()) parser.add_argument('component_name', help='The name of the component to manage') parser.add_argument( '-m', dest='monitor', action='store_true', help= 'If specified and the action is "start" or "update", gantry will remain running to monitor components, auto restarting them as necessary' ) parser.add_argument('--setconfig', dest='config_overrides', action='append', help='Configuration overrides for the component') args = parser.parse_args() component_name = args.component_name action = args.action should_monitor = args.monitor config_file = args.config_file config_overrides = args.config_overrides # Load the config. config = loadConfig(config_file) if not config: return # Create the manager. manager = RuntimeManager(config) # Find the component component = manager.getComponent(component_name) if not component: raise Exception('Unknown component: ' + component_name) # Apply the config overrides (if any). if config_overrides: component.applyConfigOverrides(config_overrides) # Run the action with the component and config. result = ACTIONS[action](component) if result and should_monitor: try: report('Starting monitoring of component: ' + component_name) monitor(component) except KeyboardInterrupt: report('Terminating monitoring of component: ' + component_name) def cleanup_monitor(signum, frame): manager.join() # Set the signal handler and a 5-second alarm signal.signal(signal.SIGINT, cleanup_monitor) # We may have to call cleanup manually if we weren't asked to monitor cleanup_monitor(None, None)
class GantryDClient(object): """ A client in gantryd. """ def __init__(self, etcdHost, projectName): self.project_name = projectName self.runtime_manager = None self.components = [] self.is_running = False # Generate a unique ID for this machine/client. self.machine_id = str(uuid.uuid1()) # Logging. self.logger = logging.getLogger(__name__) # Initialize the etcd client that we'll use. self.etcd_client = etcd.Client(host = etcdHost) # Initialize the thread used for reporting the status of this machine to etcd. self.reporting_thread = threading.Thread(target = self.reportMachineStatus, args = []) self.reporting_thread.daemon = True def getConfigJSON(self): """ Returns the project's config JSON or raises an exception if none. """ # Lookup the project on etcd. If none, report an error. config_json = None try: self.logger.debug('Looking up configuration for project %s in etcd', self.project_name) config_json = self.etcd_client.get(getProjectConfigPath(self.project_name)).value except KeyError as k: self.logger.exception(k) fail('Unknown project ' + self.project_name, project = self.project_name) return config_json def getConfig(self): """ Returns the project's config or raises an exception if none. """ config_json = self.getConfigJSON() # Parse the project's configuration and save it. try: self.config = Configuration.parse(config_json) except ConfigParseException as cpe: fail('Error parsing gantry config', project = self.project_name, exception = cpe) except Exception as e: self.logger.exception(e) return self.config def setConfig(self, config): """ Sets the project's config in etcd. """ config_json = json.dumps(config) self.logger.debug('Updating configuration for project %s', self.project_name) self.etcd_client.set(getProjectConfigPath(self.project_name), config_json) def stopComponents(self, componentNames): """ Tells all the given components on all systems to stop. """ self.initialize(componentNames) report('Marking components as stopped', project = self.project_name) for component in self.components: report('Marking component as stopped', project = self.project_name, component = component, level = ReportLevels.EXTRA) state = ComponentState(self.project_name, component, self.etcd_client) state.setStatus(STOPPED_STATUS) def killComponents(self, componentNames): """ Tells all the given components on all systems to die. """ self.initialize(componentNames) report('Marking components as killed', project = self.project_name) for component in self.components: report('Marking component as killed', project = self.project_name, component = component, level = ReportLevels.EXTRA) state = ComponentState(self.project_name, component, self.etcd_client) state.setStatus(KILLED_STATUS) def markUpdated(self, componentNames): """ Tells all the given components to update themselves. """ self.initialize(componentNames) report('Updating the image IDs on components', project = self.project_name) for component in self.components: image_id = component.getImageId() state = ComponentState(self.project_name, component, self.etcd_client) report('Component ' + component.getName() + ' -> ' + image_id[0:12], project = self.project_name, component = component) state.setReadyStatus(image_id) def listStatus(self): """ Lists the status of all components in this project. """ self.getConfig() self.initialize([c.name for c in self.config.components]) print "%-20s %-20s %-20s" % ('COMPONENT', 'STATUS', 'IMAGE ID') for component in self.components: state = ComponentState(self.project_name, component, self.etcd_client).getState() status = ComponentState.getStatusOf(state) imageid = ComponentState.getImageIdOf(state) print "%-20s %-20s %-20s" % (component.getName(), status, imageid) def run(self, componentNames): """ Runs the given components on this machine. """ self.initialize(componentNames) # Register a handler to remove this machine from the list when the daemon is # shutdown. The controller will also occasionally ping a machine to verify it # is present. self.logger.debug('Registering exit listener') atexit.register(self.handleExit) # Start the thread to register this machine as being part of the project. self.startReporter() # Start watcher thread(s), one for each component, to see when to update them. report('Gantryd running', project = self.project_name) for component in self.components: self.logger.debug('Starting component watcher for component: %s', component.getName()) watcher = ComponentWatcher(component, self.project_name, self.machine_id, self.etcd_client) watcher.start() # And sleep until new stuff comes in. while True: time.sleep(1) ######################################################################## def initialize(self, componentNames): """ Initializes this client for working with the components given. """ # Load the project configuration. self.getConfig() # Initialize the runtime manager. self.runtime_manager = RuntimeManager(self.config, daemon_mode = True) # Find all the components for this machine. for componentName in componentNames: component = self.runtime_manager.getComponent(componentName) if not component: fail('Unknown component named ' + componentName, project = self.project_name) self.components.append(component) def handleExit(self): """ Function executed when the Python system exits. This unregisters the machine in etcd. """ self.is_running = False try: machine_state = MachineState(self.project_name, self.machine_id, self.etcd_client) machine_state.removeMachine() except Exception as e: self.logger.exception(e) pass def startReporter(self): """ Starts reporting that this machine is running. """ self.is_running = True self.reporting_thread.start() def reportMachineStatus(self): """ Reports that this machine has running components. """ while self.is_running: # Perform the update. self.logger.debug('Reporting status for machine %s to etcd', self.machine_id) machine_state = MachineState(self.project_name, self.machine_id, self.etcd_client) machine_state.registerMachine([c.getName() for c in self.components], ttl = REPORT_TTL) # Sleep for the TTL minus a few seconds. time.sleep(REPORT_TTL - 5)
class GantryDClient(object): """ A client in gantryd. """ def __init__(self, etcdHost, projectName, etcdPort): self.project_name = projectName self.runtime_manager = None self.components = [] self.is_running = False # Generate a unique ID for this machine/client. self.machine_id = str(uuid.uuid1()) # Logging. self.logger = logging.getLogger(__name__) # Initialize the etcd client that we'll use. self.etcd_client = etcd.Client(host=etcdHost, port=etcdPort) # Initialize the thread used for reporting the status of this machine to etcd. self.reporting_thread = threading.Thread( target=self.reportMachineStatus, args=[]) self.reporting_thread.daemon = True def getConfigJSON(self): """ Returns the project's config JSON or raises an exception if none. """ # Lookup the project on etcd. If none, report an error. config_json = None try: self.logger.debug( 'Looking up configuration for project %s in etcd', self.project_name) config_json = self.etcd_client.get( getProjectConfigPath(self.project_name)).value except KeyError as k: self.logger.exception(k) fail('Unknown project ' + self.project_name, project=self.project_name) return config_json def getConfig(self): """ Returns the project's config or raises an exception if none. """ config_json = self.getConfigJSON() # Parse the project's configuration and save it. try: self.config = Configuration.parse(config_json) except ConfigParseException as cpe: fail('Error parsing gantry config', project=self.project_name, exception=cpe) except Exception as e: self.logger.exception(e) return self.config def setConfig(self, config): """ Sets the project's config in etcd. """ config_json = json.dumps(config) self.logger.debug('Updating configuration for project %s', self.project_name) self.etcd_client.set(getProjectConfigPath(self.project_name), config_json) def stopComponents(self, component_names): """ Tells all the given components on all systems to stop. """ self.initialize(component_names) report('Marking components as stopped', project=self.project_name) for component in self.components: report('Marking component as stopped', project=self.project_name, component=component, level=ReportLevels.EXTRA) state = ComponentState(self.project_name, component, self.etcd_client) state.setStatus(STOPPED_STATUS) def killComponents(self, component_names): """ Tells all the given components on all systems to die. """ self.initialize(component_names) report('Marking components as killed', project=self.project_name) for component in self.components: report('Marking component as killed', project=self.project_name, component=component, level=ReportLevels.EXTRA) state = ComponentState(self.project_name, component, self.etcd_client) state.setStatus(KILLED_STATUS) def markUpdated(self, component_names): """ Tells all the given components to update themselves. """ self.initialize(component_names) report('Updating the image IDs on components', project=self.project_name) for component in self.components: image_id = component.getImageId() state = ComponentState(self.project_name, component, self.etcd_client) report('Component %s->%s' % (component.getName(), image_id[0:12]), project=self.project_name, component=component) state.setReadyStatus(image_id) def listStatus(self): """ Lists the status of all components in this project. """ self.getConfig() self.initialize([c.name for c in self.config.components]) print "%-20s %-20s %-20s" % ('COMPONENT', 'STATUS', 'IMAGE ID') for component in self.components: state = ComponentState(self.project_name, component, self.etcd_client).getState() status = ComponentState.getStatusOf(state) imageid = ComponentState.getImageIdOf(state) print "%-20s %-20s %-20s" % (component.getName(), status, imageid) def run(self, component_names): """ Runs the given components on this machine. """ self.initialize(component_names) # Register a handler to remove this machine from the list when the daemon is # shutdown. The controller will also occasionally ping a machine to verify it # is present. self.logger.debug('Registering exit listener') atexit.register(self.handleExit) # Start the thread to register this machine as being part of the project. self.startReporter() # Start watcher thread(s), one for each component, to see when to update them. report('Gantryd running', project=self.project_name) for component in self.components: self.logger.debug('Starting component watcher for component: %s', component.getName()) watcher = ComponentWatcher(component, self.project_name, self.machine_id, self.etcd_client) watcher.start() # And sleep until new stuff comes in. while True: time.sleep(1) ######################################################################## def initialize(self, component_names): """ Initializes this client for working with the components given. """ # Load the project configuration. self.getConfig() # Initialize the runtime manager. self.runtime_manager = RuntimeManager(self.config) # Find all the components for this machine. for component_name in component_names: component = self.runtime_manager.getComponent(component_name) if not component: fail('Unknown component named ' + component_name, project=self.project_name) self.components.append(component) def handleExit(self): """ Function executed when the Python system exits. This unregisters the machine in etcd. """ self.is_running = False try: machine_state = MachineState(self.project_name, self.machine_id, self.etcd_client) machine_state.removeMachine() # Shut down the runtime manager if we have one if self.runtime_manager is not None: self.runtime_manager.join() except Exception as e: self.logger.exception(e) pass def startReporter(self): """ Starts reporting that this machine is running. """ self.is_running = True self.reporting_thread.start() def reportMachineStatus(self): """ Reports that this machine has running components. """ while self.is_running: # Perform the update. self.logger.debug('Reporting status for machine %s to etcd', self.machine_id) machine_state = MachineState(self.project_name, self.machine_id, self.etcd_client) machine_state.registerMachine( [c.getName() for c in self.components], ttl=REPORT_TTL) # Sleep for the TTL minus a few seconds. time.sleep(REPORT_TTL - 5)