def run(self, parameters): fsid = manager_utils.get_fsid() attrs = dict(name=parameters['Pool.poolname'], pg_num=parameters['Pool.pg_num'], min_size=parameters['Pool.min_size']) crud = Crud(parameters['manager']) crud.create(fsid, "pool", attrs) return True
def _sync_ec_profiles(self): """Invokes the below CLI commands 1. ```ceph osd erasure-code-profile ls``` and required output format is a list of ec profiles separated with new lines as below ``` default k4m2 ``` 2. ```ceph osd erasure-code-profile get {name}``` and the required output format is '=' separated values in multiple lines ``` k=2 m=1 plugin=jerasure directory={dir} ``` """ required_ec_profiles = [(2, 1), (4, 2), (6, 3), (8, 4)] ec_profile_details = {} commands = ['osd', 'erasure-code-profile', 'ls'] cmd_out = ceph.ceph_command(NS.tendrl_context.cluster_name, commands) if cmd_out['err'] == "": ec_profile_list = [] for item in cmd_out['out'].split('\n'): if item != "": ec_profile_list.append(item) for ec_profile in ec_profile_list: commands = ['osd', 'erasure-code-profile', 'get', ec_profile] cmd_out = ceph.ceph_command(NS.tendrl_context.cluster_name, commands) if cmd_out['err'] == "": info = {} for item in cmd_out['out'].split('\n'): if item != "": info[item.split('=')[0]] = \ item.split('=')[1].strip() ec_profile_details[ec_profile] = info # Ec profile out of band delete handling try: ec_profiles = NS._int.client.read( "clusters/%s/ECProfiles" % (NS.tendrl_context.integration_id)) old_ec_profiles = [] for ec_profile in ec_profiles.leaves: old_ec_profiles.append(ec_profile.key.split("/")[-1]) new_ec_profiles = [] for k, v in ec_profile_details.iteritems(): new_ec_profiles.append(k) delete_ec_profiles = set(old_ec_profiles) - set( new_ec_profiles) for id in delete_ec_profiles: NS._int.client.delete( "clusters/%s/ECProfiles/%s" % (NS.tendrl_context.integration_id, id), recursive=True) except etcd.EtcdKeyNotFound as ex: Event( ExceptionMessage(priority="debug", publisher=NS.publisher_id, payload={ "message": "key not found in etcd", "exception": ex })) available_ec_profiles = [] for k, v in ec_profile_details.iteritems(): NS.ceph.objects.ECProfile( name=k, k=v['k'], m=v['m'], plugin=v.get('plugin'), directory=v.get('directory'), ruleset_failure_domain=v.get('ruleset_failure_domain')).save() available_ec_profiles.append((int(v['k']), int(v['m']))) # Create the missing ec_profile_details missing_ec_profiles = [ item for item in required_ec_profiles if item not in available_ec_profiles ] for item in missing_ec_profiles: attrs = dict(name="k%sm%s" % (item[0], item[1]), k=item[0], m=item[1], plugin='jerasure', directory='/usr/lib/ceph/erasure-code') crud = Crud() crud.create("ec_profile", attrs)
def run(self): attrs = dict(name=self.parameters['Pool.poolname'], pg_num=self.parameters['Pool.pg_num'], min_size=self.parameters.get('Pool.min_size'), size=self.parameters.get('Pool.size'), type=self.parameters.get('Pool.type'), erasure_code_profile=self.parameters.get( 'Pool.erasure_code_profile')) if self.parameters.get('Pool.type', '') == 'erasure': attrs.pop('size') attrs.pop('min_size') if 'Pool.quota_enabled' in self.parameters and \ self.parameters['Pool.quota_enabled'] is True: attrs['quota_max_objects'] = \ self.parameters.get('Pool.quota_max_objects') attrs['quota_max_bytes'] = \ self.parameters.get('Pool.quota_max_bytes') Event( Message( priority="info", publisher=NS.publisher_id, payload={ "message": "Creating pool %s" % self.parameters['Pool.poolname'], }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) crud = Crud() resp = crud.create("pool", attrs) try: crud.sync_request_status(resp['request']) except RequestStateError as ex: Event( Message( priority="error", publisher=NS.publisher_id, payload={ "message": "Failed to create pool %s." " Error: %s" % (self.parameters['Pool.poolname'], ex) }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) return False Event( Message( priority="info", publisher=NS.publisher_id, payload={ "message": "Successfully created pool %s" % self.parameters['Pool.poolname'], }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) return True
def run(self): if 'ECProfile.plugin' in self.parameters: plugin = self.parameters['ECProfile.plugin'] else: plugin = 'jerasure' if 'ECProfile.directory' in self.parameters: directory = self.parameters.get('ECProfile.directory') else: directory = "/usr/lib/ceph/erasure-code" attrs = dict(name=self.parameters['ECProfile.name'], k=self.parameters['ECProfile.k'], m=self.parameters['ECProfile.m'], plugin=plugin, directory=directory ) Event( Message( priority="info", publisher=NS.publisher_id, payload={ "message": "Creating ec-profile %s" % self.parameters['ECProfile.name'], }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, ) ) crud = Crud() resp = crud.create("ec_profile", attrs) try: crud.sync_request_status(resp['request']) except RequestStateError as ex: Event( Message( priority="info", publisher=NS.publisher_id, payload={ "message": "Failed to create ec-profile %s." " Error: %s" % (self.parameters['ECProfile.name'], ex) }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, ) ) return False Event( Message( priority="info", publisher=NS.publisher_id, payload={ "message": "Successfully created ec-profile %s" % self.parameters['ECProfile.name'], }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, ) ) return True
def _sync_ec_profiles(self): """ Invokes the below CLI commands 1. ```ceph osd erasure-code-profile ls``` and required output format is a list of ec profiles separated with new lines as below ``` default k4m2 ``` 2. ```ceph osd erasure-code-profile get {name}``` and the required output format is '=' separated values in multiple lines ``` k=2 m=1 plugin=jerasure directory={dir} ``` """ required_ec_profiles = [(2, 1), (4, 2), (6, 3), (8, 4)] ec_profile_details = {} commands = ['osd', 'erasure-code-profile', 'ls'] cmd_out = ceph.ceph_command(NS.tendrl_context.cluster_name, commands) if cmd_out['err'] == "": ec_profile_list = [] for item in cmd_out['out'].split('\n'): if item != "": ec_profile_list.append(item) for ec_profile in ec_profile_list: commands = ['osd', 'erasure-code-profile', 'get', ec_profile] cmd_out = ceph.ceph_command(NS.tendrl_context.cluster_name, commands) if cmd_out['err'] == "": info = {} for item in cmd_out['out'].split('\n'): if item != "": info[item.split('=')[0]] = \ item.split('=')[1].strip() ec_profile_details[ec_profile] = info available_ec_profiles = [] for k, v in ec_profile_details.iteritems(): NS.ceph.objects.ECProfile( name=k, k=v['k'], m=v['m'], plugin=v.get('plugin'), directory=v.get('directory'), ruleset_failure_domain=v.get('ruleset_failure_domain')).save() available_ec_profiles.append((int(v['k']), int(v['m']))) # Create the missing ec_profile_details missing_ec_profiles = [ item for item in required_ec_profiles if item not in available_ec_profiles ] for item in missing_ec_profiles: attrs = dict(name="k%sm%s" % (item[0], item[1]), k=item[0], m=item[1], plugin='jerasure', directory='/usr/lib/ceph/erasure-code') crud = Crud() crud.create("ec_profile", attrs)
def run(self): attrs = dict(name=self.parameters['Rbd.name'], size=str(self.parameters['Rbd.size']), pool_id=self.parameters.get('Rbd.pool_id')) Event( Message( priority="info", publisher=NS.publisher_id, payload={ "message": "Creating rbd %s on pool %s" % (self.parameters['Rbd.name'], self.parameters['Rbd.pool_id']) }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) crud = Crud() ret_val = crud.create("rbd", attrs) if ret_val['response'] is not None and \ ret_val['response']['error'] is True: Event( Message( priority="info", publisher=NS.publisher_id, payload={ "message": "Failed to create rbd %s." " Error: %s" % (self.parameters['Rbd.name'], ret_val['error_status']) }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) return False Event( Message( priority="info", publisher=NS.publisher_id, payload={ "message": "Successfully created rbd %s on pool %s" % (self.parameters['Rbd.name'], self.parameters['Rbd.pool_id']) }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) pool_name = NS.etcd_orm.client.read( "clusters/%s/Pools/%s/pool_name" % (NS.tendrl_context.integration_id, self.parameters['Rbd.pool_id'])).value rbd_details = NS.state_sync_thread._get_rbds(pool_name) for k, v in rbd_details.iteritems(): NS.ceph.objects.Rbd( name=k, size=v['size'], pool_id=self.parameters['Rbd.pool_id'], flags=v['flags'], provisioned=NS.state_sync_thread._to_bytes(v['provisioned']), used=NS.state_sync_thread._to_bytes(v['used'])).save() return True
def run(self): Event( Message( priority="info", publisher=NS.publisher_id, payload={ "message": "Checking if a new pool has to be created or " "not for rbd creation" }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) if not self.parameters.get('Rbd.pool_id'): # Checking if mandatory parameters for pool creation are present mandatory_pool_params = Set([ "Rbd.pool_poolname", "Rbd.pool_pg_num", "Rbd.pool_size", "Rbd.pool_min_size" ]) missing_params = list( mandatory_pool_params.difference(Set(self.parameters.keys()))) if not missing_params: # Mapping the passed pool parameters into required keys pool_parameters = {} for key, value in self.parameters.items(): if "Rbd.pool_" in key: pool_parameters[key.replace("Rbd.pool_", "Pool.")] =\ value payload = { "integration_id": NS.tendrl_context.integration_id, "run": "ceph.flows.CreatePool", "status": "new", "parameters": pool_parameters, "parent": self.parameters['job_id'], "type": "sds", "tags": ["tendrl/integration/$TendrlContext." "integration_id"] } Event( Message( priority="error", publisher=NS.publisher_id, payload={"message": "Creating job for pool creation"}, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) _job_id = str(uuid.uuid4()) Job(job_id=_job_id, status="new", payload=payload).save() Event( Message( priority="error", publisher=NS.publisher_id, payload={ "message": "Checking for successful pool creation" }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) pool_created = False job_status = "new" while not pool_created: try: job_status = NS._int.client.read("/queue/%s/status" % _job_id).value except etcd.EtcdKeyNotFound: Event( Message( priority="error", publisher=NS.publisher_id, payload={ "message": "Failed to fetch pool " "creation status for rbd " "creation" }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) break if job_status == "finished": pool_created = True elif job_status == "failed": break if pool_created: # Setting pool_id for rbd creation pool_id = self._get_pool_id( self.parameters['Rbd.pool_poolname']) if pool_id: self.parameters['Rbd.pool_id'] = pool_id else: Event( Message( priority="error", publisher=NS.publisher_id, payload={ "message": "Failed to fetch pool_id %s ." "Cannot create rbd without " "pool_id." % pool_id }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) return False else: Event( Message( priority="error", publisher=NS.publisher_id, payload={ "message": "Failed to create pool. " "Cannot proceed with rbd creation." }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) return False else: Event( Message( priority="info", publisher=NS.publisher_id, payload={ "message": "Mandatory parameters %s for pool " "creation not present. Cannot continue" " with rbd creation." % ', '.join(missing_params) }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) return False attrs = dict(name=self.parameters['Rbd.name'], size=str(self.parameters['Rbd.size']), pool_id=self.parameters.get('Rbd.pool_id')) Event( Message( priority="info", publisher=NS.publisher_id, payload={ "message": "Creating rbd %s on pool %s" % (self.parameters['Rbd.name'], self.parameters['Rbd.pool_id']) }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) crud = Crud() resp = crud.create("rbd", attrs) try: crud.sync_request_status(resp['request']) except RequestStateError as ex: Event( Message( priority="info", publisher=NS.publisher_id, payload={ "message": "Failed to create rbd %s." " Error: %s" % (self.parameters['Rbd.name'], ex) }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) return False Event( Message( priority="info", publisher=NS.publisher_id, payload={ "message": "Successfully created rbd %s on pool %s" % (self.parameters['Rbd.name'], self.parameters['Rbd.pool_id']) }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) pool_name = NS._int.client.read("clusters/%s/Pools/%s/pool_name" % (NS.tendrl_context.integration_id, self.parameters['Rbd.pool_id'])).value rbd_details = NS.state_sync_thread._get_rbds(pool_name) for k, v in rbd_details.iteritems(): NS.ceph.objects.Rbd( name=k, size=v['size'], pool_id=self.parameters['Rbd.pool_id'], flags=v['flags'], provisioned=NS.state_sync_thread._to_bytes(v['provisioned']), used=NS.state_sync_thread._to_bytes(v['used'])).save() return True
def run(self): attrs = dict(name=self.parameters['Pool.poolname'], pg_num=self.parameters['Pool.pg_num'], min_size=self.parameters.get('Pool.min_size'), size=self.parameters.get('Pool.size'), type=self.parameters.get('Pool.type'), erasure_code_profile=self.parameters.get( 'Pool.erasure_code_profile')) if 'Pool.quota_enabled' in self.parameters and \ self.parameters['Pool.quota_enabled'] is True: attrs['quota_max_objects'] = \ self.parameters.get('Pool.quota_max_objects') attrs['quota_max_bytes'] = \ self.parameters.get('Pool.quota_max_bytes') Event( Message( priority="info", publisher=NS.publisher_id, payload={ "message": "Creating pool %s" % self.parameters['Pool.poolname'], }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) crud = Crud() ret_val = crud.create("pool", attrs) if ret_val['response'] is not None and \ ret_val['response']['error'] is True: Event( Message( priority="info", publisher=NS.publisher_id, payload={ "message": "Failed to create pool %s." " Error: %s" % (self.parameters['Pool.poolname'], ret_val['error_status']) }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) return False Event( Message( priority="info", publisher=NS.publisher_id, payload={ "message": "Successfully created pool %s" % self.parameters['Pool.poolname'], }, job_id=self.parameters['job_id'], flow_id=self.parameters['flow_id'], cluster_id=NS.tendrl_context.integration_id, )) return True