def idle(self): """ perform a check on the status of the current node, verifying how long it has been idle (if at all) marking the current timestamp since idle and determine if the node needs to be terminated. """ if not self.node: abort(404, 'could not find UUID: %s' % self.identifier) provider_for_node = conf.nodes[self.node.name]['provider'] provider = providers.get(provider_for_node) if request.method != 'POST': abort(405) now = datetime.utcnow() if self.node.idle: # it was idle before so check how many seconds since it was lazy. # `idle` is a property that will only be true-ish if idle_since has # been set. difference = now - self.node.idle_since if difference.seconds > 1200: # 20 minutes # we need to terminate this couch potato logger.info("Destroying node: %s" % self.node.cloud_name) try: provider.destroy_node(name=self.node.cloud_name) except CloudNodeNotFound: logger.info("node does not exist in cloud provider") conn = jenkins_connection() if conn.node_exists(self.node.jenkins_name): logger.info("Deleting node in jenkins: %s" % self.node.jenkins_name) conn.delete_node(self.node.jenkins_name) # delete from our database self.node.delete() else: # mark it as being idle self.node.idle_since = now
def check_orphaned(): """ Machines created in providers might be in an error state or some configuration in between may have prevented them to join Jenkins (or manually removed). This task will go through the nodes it knows about, make sure they exist in the provider and if so, remove them from the mita database and the provider. """ conn = connections.jenkins_connection() nodes = models.Node.query.all() for node in nodes: # it is all good if this node exists in Jenkins. That is the whole # reason for its miserable existence, to work for Mr. Jenkins. Let it # be. if node.jenkins_name: if conn.node_exists(node.jenkins_name): continue # So this node is not in Jenkins. If it is less than 15 minutes then # don't do anything because it might be just taking a while to join. # ALERT MR ROBINSON: 15 minutes is a magical number. now = datetime.utcnow() difference = now - node.created if difference.seconds > 900: # magical number alert logger.info("found created node that didn't join Jenkins: %s", node) provider = providers.get(node.provider) # "We often miss opportunity because it's dressed in overalls and # looks like work". Node missed his opportunity here. try: provider.destroy_node(name=node.cloud_name) except Exception: logger.exception("unable to destroy node: %s", node.cloud_name) logger.info("removed useless node from provider and database: %s", node) node.delete() models.commit()
def idle(self): """ perform a check on the status of the current node, verifying how long it has been idle (if at all) marking the current timestamp since idle and determine if the node needs to be terminated. """ if not self.node: abort(404, 'could not find UUID: %s' % self.identifier) provider_for_node = conf.nodes[self.node.name]['provider'] provider = providers.get(provider_for_node) if request.method != 'POST': abort(405) now = datetime.utcnow() if self.node.idle: # it was idle before so check how many seconds since it was lazy. # `idle` is a property that will only be true-ish if idle_since has # been set. difference = now - self.node.idle_since if difference.seconds > 600: # 10 minutes # we need to terminate this couch potato logger.info("Destroying node: %s" % self.node.cloud_name) try: provider.destroy_node(name=self.node.cloud_name) except CloudNodeNotFound: logger.info("node does not exist in cloud provider") conn = jenkins_connection() if conn.node_exists(self.node.jenkins_name): logger.info("Deleting node in jenkins: %s" % self.node.jenkins_name) conn.delete_node(self.node.jenkins_name) # delete from our database self.node.delete() else: # mark it as being idle self.node.idle_since = now
def delete_jenkins_node(name): conn = jenkins_connection() logger.info("Deleting node in jenkins: %s" % name) if not name: logger.info('Node does not have a jenkins_name, will skip') return if conn.node_exists(name): conn.delete_node(name) return logger.info("Node does not exist in Jenkins, cannot delete")
def check_orphaned(): """ Machines created in providers might be in an error state or some configuration in between may have prevented them to join Jenkins (or manually removed). This task will go through the nodes it knows about, make sure they exist in the provider and if so, remove them from the mita database and the provider. """ conn = connections.jenkins_connection() try: nodes = models.Node.query.all() except InvalidRequestError: logger.exception('could not list nodes') models.rollback() # we can try again at the next scheduled task run return for node in nodes: # it is all good if this node exists in Jenkins. That is the whole # reason for its miserable existence, to work for Mr. Jenkins. Let it # be. if node.jenkins_name: if conn.node_exists(node.jenkins_name): continue # So this node is not in Jenkins. If it is less than 15 minutes then # don't do anything because it might be just taking a while to join. # ALERT MR ROBINSON: 15 minutes is a magical number. now = datetime.utcnow() difference = now - node.created if difference.seconds > 900: # magical number alert logger.info("found created node that didn't join Jenkins: %s", node) provider = providers.get(node.provider) # "We often miss opportunity because it's dressed in overalls and # looks like work". Node missed his opportunity here. try: provider.destroy_node(name=node.cloud_name) except CloudNodeNotFound: logger.info("cloud was not found on provider: %s", node.cloud_name) logger.info( "will remove node from database, API confirms it no longer exists" ) node.delete() models.commit() except Exception: logger.exception("unable to destroy node: %s", node.cloud_name) logger.error("will skip database removal") continue # providers can purge nodes in error state too, try to prune those as well providers_conf = pecan.conf.provider.to_dict() for provider_name in providers_conf.keys(): provider = providers.get(provider_name) provider.purge()
def get_jenkins_name(uuid): """ Given a node's identifier use the jenkins api to find a node name in jenkins that includes that uuid and return it. """ conn = jenkins_connection() nodes = conn.get_nodes() for node in nodes: if uuid in node['name']: return node['name'] return None
def get_node_labels(node_name, _xml_configuration=None): """ Useful when a custom node was added with a name that mita does not understand due to odd/unsupported naming conventions. The Jenkins Python module doesn't have a nice JSON representation from the Jenkins API response to look at the labels, so this parses that output looking for the right tag and extracting the labels from there. """ conn = jenkins_connection() try: xml_configuration = _xml_configuration or conn.get_node_config(node_name) except JenkinsNotFoundException: logging.warning('"%s" was not found in Jenkins', node_name) return [] xml_object = ElementTree.fromstring(xml_configuration) for node in xml_object: if node.tag == 'label': # node labels are in this tag, parse the text. The XML should look # like: <label>amd64 centos7 x86_64 huge</label> return node.text.split() return []
def get_node_labels(node_name, _xml_configuration=None): """ Useful when a custom node was added with a name that mita does not understand due to odd/unsupported naming conventions. The Jenkins Python module doesn't have a nice JSON representation from the Jenkins API response to look at the labels, so this parses that output looking for the right tag and extracting the labels from there. """ conn = jenkins_connection() try: xml_configuration = _xml_configuration or conn.get_node_config( node_name) except JenkinsNotFoundException: logging.warning('"%s" was not found in Jenkins', node_name) return [] xml_object = ElementTree.fromstring(xml_configuration) for node in xml_object: if node.tag == 'label': # node labels are in this tag, parse the text. The XML should look # like: <label>amd64 centos7 x86_64 huge</label> return node.text.split() return []
def check_orphaned(): """ Machines created in providers might be in an error state or some configuration in between may have prevented them to join Jenkins (or manually removed). This task will go through the nodes it knows about, make sure they exist in the provider and if so, remove them from the mita database and the provider. """ conn = connections.jenkins_connection() nodes = models.Node.query.all() for node in nodes: # it is all good if this node exists in Jenkins. That is the whole # reason for its miserable existence, to work for Mr. Jenkins. Let it # be. if node.jenkins_name: if conn.node_exists(node.jenkins_name): continue # So this node is not in Jenkins. If it is less than 15 minutes then # don't do anything because it might be just taking a while to join. # ALERT MR ROBINSON: 15 minutes is a magical number. now = datetime.utcnow() difference = now - node.created if difference.seconds > 900: # magical number alert logger.info("found created node that didn't join Jenkins: %s", node) provider = providers.get(node.provider) # "We often miss opportunity because it's dressed in overalls and # looks like work". Node missed his opportunity here. try: provider.destroy_node(name=node.cloud_name) except Exception: logger.exception("unable to destroy node: %s", node.cloud_name) logger.error("will skip database removal") return logger.info("removed useless node from provider and database: %s", node) node.delete() models.commit()