def test_ceph_create_ssh_setup_jobs(): setattr(__builtin__, "NS", maps.NamedDict()) NS.publisher_id = "node_context" param = maps.NamedDict() param['Node[]'] = [] param["Cluster.node_configuration"] = { "test_node": maps.NamedDict(role="osd", provisioning_ip="test_ip")} NS.ceph_provisioner = importlib.import_module( "tendrl.commons.tests.fixtures.plugin").Plugin() ret = utils.ceph_create_ssh_setup_jobs(param) assert ret == [] param['Node[]'] = ['test_node'] NS.node_context = maps.NamedDict() NS.node_context.node_id = "test_node" ret = utils.ceph_create_ssh_setup_jobs(param) assert ret == [] NS.node_context.node_id = "node" param["job_id"] = "test_id" param["flow_id"] = "test_flow_id" ret = utils.ceph_create_ssh_setup_jobs(param) str(uuid.uuid4()) assert ret is not None
def run(self): try: # Locking nodes create_cluster_utils.acquire_node_lock(self.parameters) integration_id = self.parameters['TendrlContext.integration_id'] sds_name = self.parameters["TendrlContext.sds_name"] ssh_job_ids = [] if "ceph" in sds_name: ssh_job_ids = create_cluster_utils.ceph_create_ssh_setup_jobs( self.parameters) else: ssh_job_ids = \ create_cluster_utils.gluster_create_ssh_setup_jobs( self.parameters ) while True: gevent.sleep(3) all_status = {} for job_id in ssh_job_ids: # noinspection PyUnresolvedReferences all_status[job_id] = NS._int.client.read( "/queue/%s/status" % job_id).value _failed = { _jid: status for _jid, status in all_status.iteritems() if status == "failed" } if _failed: raise AtomExecutionFailedError( "SSH setup failed for jobs %s cluster %s" % (str(_failed), integration_id)) if all( [status == "finished" for status in all_status.values()]): Event( Message(job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], priority="info", publisher=NS.publisher_id, payload={ "message": "SSH setup completed for all " "nodes in cluster %s" % integration_id })) # set this node as gluster provisioner if "gluster" in self.parameters["TendrlContext.sds_name"]: tags = ["provisioner/%s" % integration_id] NS.node_context = NS.node_context.load() tags += NS.node_context.tags NS.node_context.tags = list(set(tags)) NS.node_context.save() break Event( Message(job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], priority="info", publisher=NS.publisher_id, payload={ "message": "Starting SDS install and config %s" % integration_id })) # SSH setup jobs finished above, now install sds bits and create # cluster if "ceph" in sds_name: Event( Message(job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], priority="info", publisher=NS.publisher_id, payload={ "message": "Creating Ceph Storage Cluster " "%s" % integration_id })) self.parameters.update({'create_mon_secret': True}) create_ceph(self.parameters) else: Event( Message(job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], priority="info", publisher=NS.publisher_id, payload={ "message": "Creating Gluster Storage " "Cluster %s" % integration_id })) create_gluster(self.parameters) except Exception as ex: # For traceback Event( ExceptionMessage(priority="error", publisher=NS.publisher_id, payload={ "message": ex.message, "exception": ex })) # raising exception to mark job as failed raise ex finally: # releasing nodes if any exception came create_cluster_utils.release_node_lock(self.parameters) return True
def run(self): try: # Lock nodes create_cluster_utils.acquire_node_lock(self.parameters) integration_id = self.parameters['TendrlContext.integration_id'] if integration_id is None: raise FlowExecutionFailedError( "TendrlContext.integration_id cannot be empty") supported_sds = NS.compiled_definitions.get_parsed_defs( )['namespace.tendrl']['supported_sds'] sds_name = self.parameters["TendrlContext.sds_name"] if sds_name not in supported_sds: raise FlowExecutionFailedError("SDS (%s) not supported" % sds_name) ssh_job_ids = [] if "ceph" in sds_name: ssh_job_ids = create_cluster_utils.ceph_create_ssh_setup_jobs( self.parameters) else: ssh_job_ids = \ create_cluster_utils.gluster_create_ssh_setup_jobs( self.parameters, skip_current_node=True ) while True: time.sleep(3) all_status = {} for job_id in ssh_job_ids: all_status[job_id] = NS._int.client.read( "/queue/%s/status" % job_id).value _failed = { _jid: status for _jid, status in all_status.iteritems() if status == "failed" } if _failed: raise FlowExecutionFailedError( "SSH setup failed for jobs %s cluster %s" % (str(_failed), integration_id)) if all( [status == "finished" for status in all_status.values()]): Event( Message(job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], priority="info", publisher=NS.publisher_id, payload={ "message": "SSH setup completed for all " "nodes in cluster %s" % integration_id })) break # SSH setup jobs finished above, now install sds # bits and create cluster if "ceph" in sds_name: Event( Message(job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], priority="info", publisher=NS.publisher_id, payload={ "message": "Expanding ceph cluster %s" % integration_id })) ceph_help.expand_cluster(self.parameters) else: Event( Message(job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], priority="info", publisher=NS.publisher_id, payload={ "message": "Expanding Gluster Storage" " Cluster %s" % integration_id })) gluster_help.expand_gluster(self.parameters) Event( Message( job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], priority="info", publisher=NS.publisher_id, payload={ "message": "SDS install/config completed on newly " "expanded nodes, Please wait while " "tendrl-node-agents detect sds details on the newly " "expanded nodes %s" % self.parameters['Node[]'] })) # Wait till detected cluster in populated for nodes while True: time.sleep(3) all_status = [] detected_cluster = "" different_cluster_id = False dc = "" for node in self.parameters['Node[]']: try: dc = NS._int.client.read( "/nodes/%s/DetectedCluster/detected_cluster_id" % node).value if not detected_cluster: detected_cluster = dc else: if detected_cluster != dc: all_status.append(False) different_cluster_id = True break all_status.append(True) except etcd.EtcdKeyNotFound: all_status.append(False) if different_cluster_id: raise FlowExecutionFailedError( "Seeing different detected cluster id in" " different nodes. %s and %s" % (detected_cluster, dc)) if all_status: if all(all_status): break # Create the params list for import cluster flow new_params = dict() new_params['Node[]'] = self.parameters['Node[]'] new_params['TendrlContext.integration_id'] = integration_id # Get node context for one of the nodes from list sds_pkg_name = NS._int.client.read( "nodes/%s/DetectedCluster/" "sds_pkg_name" % self.parameters['Node[]'][0]).value new_params['import_after_expand'] = True sds_pkg_version = NS._int.client.read( "nodes/%s/DetectedCluster/sds_pkg_" "version" % self.parameters['Node[]'][0]).value new_params['DetectedCluster.sds_pkg_name'] = \ sds_pkg_name new_params['DetectedCluster.sds_pkg_version'] = \ sds_pkg_version tags = [] for node in self.parameters['Node[]']: tags.append("tendrl/node_%s" % node) payload = { "tags": tags, "run": "tendrl.flows.ImportCluster", "status": "new", "parameters": new_params, "parent": self.parameters['job_id'], "type": "node" } _job_id = str(uuid.uuid4()) # release lock before import cluster create_cluster_utils.release_node_lock(self.parameters) Job(job_id=_job_id, status="new", payload=payload).save() Event( Message(job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], priority="info", publisher=NS.publisher_id, payload={ "message": "Please wait while Tendrl imports (" "job_id: %s) newly expanded " "%s storage nodes %s" % (_job_id, sds_pkg_name, integration_id) })) except Exception as ex: Event( ExceptionMessage(priority="error", publisher=NS.publisher_id, payload={ "message": ex.message, "exception": ex })) # raising exception to mark job as failed raise ex finally: # release lock if any exception came create_cluster_utils.release_node_lock(self.parameters)