def create_vre_server(self): """ Create VRE server in ~okeanos """ vre_script_file_name = '' flavor_id = self.get_flavor_id('master') if flavor_id == 0: msg = 'Combination of cpu, ram, disk and disk_template do' \ ' not match an existing id' raise ClientError(msg, error_flavor_id) image_id = self.get_image_id() retval = self.check_all_resources() pub_keys_path = '' vre_script_name = '{0}.sh'.format(filter(lambda l: l.isalpha(), self.opts['os_choice'])) # Create name of VRE server with [orka] prefix vre_server_name = '{0}-{1}'.format('[orka]',self.opts['server_name']) self.opts['server_name'] = vre_server_name task_id = current_task.request.id server_id = db_server_create(self.opts, task_id) self.server_name_postfix_id = '{0}-{1}-vre'.format(self.opts['server_name'], server_id) # Check if a shell script is required to be copied to VRE server if self.vre_req_script: vre_script_file_name = 'scripts/{0}'.format(vre_script_name) # Check if user chose ssh keys or not. ssh_key_selected = self.opts.get('ssh_key_selection','no_ssh_key_selected') if ssh_key_selected == 'no_ssh_key_selected' or ssh_key_selected is None: self.ssh_file = 'no_ssh_key_selected' else: self.ssh_key_file(self.server_name_postfix_id) pub_keys_path = self.ssh_file try: server = self.cyclades.create_server(vre_server_name, flavor_id, image_id, personality=personality('', pub_keys_path,vre_script_file_name), project_id=self.project_id) except ClientError, e: # If no public IP is free, get a new one if e.status == status.HTTP_409_CONFLICT: get_float_network_id(self.net_client, project_id=self.project_id) server = self.cyclades.create_server(vre_server_name, flavor_id, image_id, personality=personality('', pub_keys_path, vre_script_file_name), project_id=self.project_id) else: msg = u'VRE server \"{0}\" creation failed due to error: {1}'.format(self.opts['server_name'], str(e.args[0])) set_server_state(self.opts['token'], server_id, 'Error',status='Failed', error=msg) raise
class YarnCluster(object): """ Class for create hadoop-yarn cluster functionality """ def __init__(self, opts): """Initialization of YarnCluster data attributes""" self.opts = opts # Master VM ip, placeholder value self.HOSTNAME_MASTER_IP = '127.0.0.1' # master VM root password file, placeholder value self.pass_file = 'PLACEHOLDER' self.orka_image_uuid = False # List of cluster VMs self.server_dict = {} if self.opts['disk_template'] == 'Archipelago': self.opts['disk_template'] = 'ext_vlmc' elif self.opts['disk_template'] == 'Standard': self.opts['disk_template'] = 'drbd' # project id of project name given as argument self.project_id = get_project_id(self.opts['token'], self.opts['project_name']) self.status = {} # Instance of an AstakosClient object self.auth = check_credentials(self.opts['token'], self.opts.get('auth_url', auth_url)) # Check if project has actual quota if self.check_project_quota() != 0: msg = 'Project %s exists but you have no quota to request' % \ self.opts['project_name'] raise ClientError(msg, error_project_quota) # ~okeanos endpoints and user id self.endpoints, self.user_id = endpoints_and_user_id(self.auth) # Instance of CycladesClient self.cyclades = init_cyclades(self.endpoints['cyclades'], self.opts['token']) # Instance of CycladesNetworkClient self.net_client = init_cyclades_netclient(self.endpoints['network'], self.opts['token']) # Instance of Plankton/ImageClient self.plankton = init_plankton(self.endpoints['plankton'], self.opts['token']) # Get resources of pending clusters self.pending_quota = retrieve_pending_clusters( self.opts['token'], self.opts['project_name']) self._DispatchCheckers = {} self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_cluster_size_quotas self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_network_quotas self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_ip_quotas self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_cpu_valid self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_ram_valid self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_disk_valid def check_cluster_size_quotas(self): """ Checks if the user quota is enough to create the requested number of VMs. """ dict_quotas = get_user_quota(self.auth) pending_vm = self.pending_quota['VMs'] limit_vm = dict_quotas[self.project_id]['cyclades.vm']['limit'] usage_vm = dict_quotas[self.project_id]['cyclades.vm']['usage'] available_vm = limit_vm - usage_vm - pending_vm if available_vm < self.opts['cluster_size']: msg = 'Cyclades VMs out of limit' raise ClientError(msg, error_quotas_cluster_size) else: return 0 def check_network_quotas(self): """ Checks if the user quota is enough to create a new private network Subtracts the number of networks used and pending from the max allowed number of networks """ dict_quotas = get_user_quota(self.auth) pending_net = self.pending_quota['Network'] limit_net = dict_quotas[ self.project_id]['cyclades.network.private']['limit'] usage_net = dict_quotas[ self.project_id]['cyclades.network.private']['usage'] project_limit_net = dict_quotas[ self.project_id]['cyclades.network.private']['project_limit'] project_usage_net = dict_quotas[ self.project_id]['cyclades.network.private']['project_usage'] available_networks = limit_net - usage_net if (available_networks > (project_limit_net - project_usage_net)): available_networks = project_limit_net - project_usage_net available_networks -= pending_net if available_networks >= 1: logging.log(REPORT, ' Private Network quota is ok') return 0 else: msg = 'Private Network quota exceeded in project: ' + self.opts[ 'project_name'] raise ClientError(msg, error_quotas_network) def check_ip_quotas(self): """Checks user's quota for unattached public ips.""" dict_quotas = get_user_quota(self.auth) list_float_ips = self.net_client.list_floatingips() pending_ips = self.pending_quota['Ip'] limit_ips = dict_quotas[ self.project_id]['cyclades.floating_ip']['limit'] usage_ips = dict_quotas[ self.project_id]['cyclades.floating_ip']['usage'] project_limit_ips = dict_quotas[ self.project_id]['cyclades.floating_ip']['project_limit'] project_usage_ips = dict_quotas[ self.project_id]['cyclades.floating_ip']['project_usage'] available_ips = limit_ips - usage_ips if (available_ips > (project_limit_ips - project_usage_ips)): available_ips = project_limit_ips - project_usage_ips available_ips -= pending_ips for d in list_float_ips: if d['instance_id'] is None and d['port_id'] is None: available_ips += 1 if available_ips > 0: logging.log(REPORT, ' Floating IP quota is ok') return 0 else: msg = 'Floating IP not available in project: ' + self.opts[ 'project_name'] raise ClientError(msg, error_get_ip) def check_cpu_valid(self): """ Checks if the user quota is enough to bind the requested cpu resources. Subtracts the number of cpus used and pending from the max allowed number of cpus. """ dict_quotas = get_user_quota(self.auth) pending_cpu = self.pending_quota['Cpus'] limit_cpu = dict_quotas[self.project_id]['cyclades.cpu']['limit'] usage_cpu = dict_quotas[self.project_id]['cyclades.cpu']['usage'] available_cpu = limit_cpu - usage_cpu - pending_cpu cpu_req = self.opts['cpu_master'] + \ self.opts['cpu_slaves'] * (self.opts['cluster_size'] - 1) if available_cpu < cpu_req: msg = 'Cyclades cpu out of limit' raise ClientError(msg, error_quotas_cpu) else: return 0 def check_ram_valid(self): """ Checks if the user quota is enough to bind the requested ram resources. Subtracts the number of ram used and pending from the max allowed number of ram. """ dict_quotas = get_user_quota(self.auth) pending_ram = self.pending_quota['Ram'] limit_ram = dict_quotas[self.project_id]['cyclades.ram']['limit'] usage_ram = dict_quotas[self.project_id]['cyclades.ram']['usage'] available_ram = (limit_ram - usage_ram) / Bytes_to_MB - pending_ram ram_req = self.opts['ram_master'] + \ self.opts['ram_slaves'] * (self.opts['cluster_size'] - 1) if available_ram < ram_req: msg = 'Cyclades ram out of limit' raise ClientError(msg, error_quotas_ram) else: return 0 def check_disk_valid(self): """ Checks if the requested disk resources are available for the user. Subtracts the number of disk used and pending from the max allowed disk size. """ dict_quotas = get_user_quota(self.auth) pending_cd = self.pending_quota['Disk'] limit_cd = dict_quotas[self.project_id]['cyclades.disk']['limit'] usage_cd = dict_quotas[self.project_id]['cyclades.disk']['usage'] cyclades_disk_req = self.opts['disk_master'] + \ self.opts['disk_slaves'] * (self.opts['cluster_size'] - 1) available_cyclades_disk_GB = (limit_cd - usage_cd) / Bytes_to_GB - pending_cd if available_cyclades_disk_GB < cyclades_disk_req: msg = 'Cyclades disk out of limit' raise ClientError(msg, error_quotas_cyclades_disk) else: return 0 def check_all_resources(self): """ Checks user's quota if every requested resource is available. Returns zero if everything available. """ for checker in [ func for (order, func) in sorted(self._DispatchCheckers.items()) ]: # for k, checker in self._DispatchCheckers.iteritems(): retval = checker() # print checker.__name__ + ":" + str(retval) #debug return retval def get_flavor_id(self, role): """ Return the flavor id for a virtual machine based on cpu,ram,disk_size and disk template. """ try: flavor_list = self.cyclades.list_flavors(True) except ClientError: msg = 'Could not get list of flavors' raise ClientError(msg, error_flavor_list) flavor_id = 0 for flavor in flavor_list: if flavor['ram'] == self.opts['ram_{0}'.format(role)] and \ flavor['SNF:disk_template'] == self.opts['disk_template'] and \ flavor['vcpus'] == self.opts['cpu_{0}'.format(role)] and \ flavor['disk'] == self.opts['disk_{0}'.format(role)]: flavor_id = flavor['id'] return flavor_id def check_project_quota(self): """Checks that for a given project actual quota exist""" dict_quotas = get_user_quota(self.auth) if self.project_id in dict_quotas: return 0 return error_project_quota def ssh_key_file(self, name): """ Creates a file named after the timestamped name of cluster containing the public ssh_key of the user. """ ssh_info = self.ssh_key_list() self.ssh_file = join(os.getcwd(), name.replace(" ", "_") + '_ssh_key') for item in ssh_info: if item['name'] == self.opts['ssh_key_selection']: with open(self.ssh_file, 'w') as f: f.write(item['content']) def ssh_key_list(self): """ Get the ssh_key dictionary of a user """ command = 'curl -X GET -H "Content-Type: application/json" -H "Accept: application/json" -H "X-Auth-Token: ' + self.opts[ 'token'] + '" https://cyclades.okeanos.grnet.gr/userdata/keys' p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) out, err = p.communicate() output = out[2:-2].split('}, {') ssh_dict = list() ssh_counter = 0 for dictionary in output: mydict = dict() new_dictionary = dictionary.replace('"', '') dict1 = new_dictionary.split(', ') for each in dict1: list__keys_values_in_dict = each.split(': ') new_list_of_dict_elements = list() for item in list__keys_values_in_dict: new_list_of_dict_elements.append(item) if len(new_list_of_dict_elements) > 1: for pair in new_list_of_dict_elements: mydict[new_list_of_dict_elements[ 0]] = new_list_of_dict_elements[1] ssh_dict.append(mydict) return ssh_dict def check_user_resources(self): """ Checks user resources before the starting cluster creation. Also, returns the flavor id of master and slave VMs and the id of the image chosen by the user. """ flavor_master = self.get_flavor_id('master') flavor_slaves = self.get_flavor_id('slaves') if flavor_master == 0 or flavor_slaves == 0: msg = 'Combination of cpu, ram, disk and disk_template do' \ ' not match an existing id' raise ClientError(msg, error_flavor_id) retval = self.check_all_resources() # check image metadata in database and pithos and set orka_image_uuid accordingly self.orka_image_uuid = OrkaImage.objects.get( image_name=self.opts['os_choice']).image_pithos_uuid list_current_images = self.plankton.list_public(True, 'default') for image in list_current_images: if self.orka_image_uuid == image['id']: return flavor_master, flavor_slaves, image['id'] msg = 'Image {0} exists on database but cannot be found or has different id' ' on Pithos+'.format(self.opts['os_choice']) raise ClientError(msg, error_flavor_id) def get_image_id(self): """ Return id of given image """ chosen_image = {} list_current_images = self.plankton.list_public(True, 'default') # Check availability of resources # Find image id of the operating system arg given for lst in list_current_images: if lst['name'] == self.opts['os_choice']: chosen_image = lst return chosen_image['id'] if not chosen_image: msg = self.opts['os_choice'] + ' is not a valid image' raise ClientError(msg, error_image_id) def create_vre_server(self): """ Create VRE server in ~okeanos """ flavor_id = self.get_flavor_id('master') image_id = self.get_image_id() retval = self.check_all_resources() pub_keys_path = '' # Create name of VRE server with [orka] prefix vre_server_name = '{0}-{1}'.format('[orka]', self.opts['server_name']) self.opts['server_name'] = vre_server_name task_id = current_task.request.id server_id = db_server_create(self.opts, task_id) self.server_name_postfix_id = '{0}-{1}-vre'.format( self.opts['server_name'], server_id) # Check if user chose ssh keys or not. if self.opts['ssh_key_selection'] is None or self.opts[ 'ssh_key_selection'] == 'no_ssh_key_selected': self.ssh_file = 'no_ssh_key_selected' else: self.ssh_key_file(self.server_name_postfix_id) pub_keys_path = self.ssh_file try: server = self.cyclades.create_server(vre_server_name, flavor_id, image_id, personality=personality( '', pub_keys_path), project_id=self.project_id) except ClientError, e: # If no public IP is free, get a new one if e.status == status.HTTP_409_CONFLICT: get_float_network_id(self.net_client, project_id=self.project_id) server = self.cyclades.create_server( vre_server_name, flavor_id, image_id, personality=personality('', pub_keys_path), project_id=self.project_id) else: raise # Get VRE server root password server_pass = server['adminPass'] # Placeholder for VRe server public IP server_ip = '127.0.0.1' # Update DB with server status as pending new_state = "Started creation of Virtual Research Environment server {0}".format( vre_server_name) set_server_state(self.opts['token'], server_id, new_state, okeanos_server_id=server['id']) new_status = self.cyclades.wait_server(server['id'], max_wait=MAX_WAIT) if new_status == 'ACTIVE': server_details = self.cyclades.get_server_details(server['id']) for attachment in server_details['attachments']: if attachment['OS-EXT-IPS:type'] == 'floating': server_ip = attachment['ipv4'] else: self.cyclades.delete_server(server['id']) set_server_state(self.opts['token'], server_id, 'Error', status='Failed') msg = ' Status for VRE server {0} is {1}'.format( server['name'], new_status) raise ClientError(msg, error_create_server) set_server_state(self.opts['token'], server_id, state='VRE Server created', status='Active', server_IP=server_ip) # Wait for VRE server to be pingable sleep(15) try: vre_image_uuid = VreImage.objects.get( image_name=self.opts['os_choice']).image_pithos_uuid if vre_image_uuid == server['image']['id']: chosen_vre_image = pithos_vre_images_uuids_actions[ vre_image_uuid] start_vre(server_ip, server_pass, self.opts['token'], chosen_vre_image['image']) else: msg = 'Image {0} exists on database but cannot be found or has different id' ' on Pithos+'.format(self.opts['os_choice']) raise ClientError(msg, error_flavor_id) except RuntimeError, e: # Exception is raised if a VRE start command is not executed correctly and informs user of its VRE properties # so user can ssh connect to the VRE server or delete the server from orkaCLI. raise RuntimeError( '{0}. Your VRE server has the following properties id:{1} root_password:{2} server_IP:{3}' .format(e.args[0], server_id, server_pass, server_ip), error_create_server)
start_vre_script(server_ip,server_pass,self.opts['admin_password'], vre_script_name, self.opts['admin_email']) else: msg = u'VRE server \"{0}\" creation failed. Created okeanos VM id does not match image {1} id'.format(self.opts['server_name'], self.opts['os_choice']) set_server_state(self.opts['token'],server_id,'Error',status='Failed',error=msg) self.cyclades.delete_server(server['id']) raise ClientError(msg, error_image_id) except RuntimeError, e: # Exception is raised if a VRE start command is not executed correctly and informs user of its VRE properties # so user can ssh connect to the VRE server or delete the server from orkaCLI. msg = u'VRE server \"{0}\" created but started with errors'.format(self.opts['server_name']) set_server_state(self.opts['token'],server_id,state='VRE Server created but started with errors', status='Active',server_IP=server_ip, error=msg) raise RuntimeError('Your VRE server has the following properties id:{0} root_password:{1} server_IP:{2}' ' but could not be started normally.'.format(server_id,server_pass,server_ip),error_create_server) set_server_state(self.opts['token'],server_id,state='VRE Server created',status='Active',server_IP=server_ip) return server_id, server_pass, server_ip def create_bare_cluster(self): """Creates a bare ~okeanos cluster.""" server_home_path = expanduser('~') server_ssh_keys = join(server_home_path, ".ssh/id_rsa.pub") pub_keys_path = '' logging.log(SUMMARY, 'Authentication verified') current_task.update_state(state="Authenticated") flavor_master, flavor_slaves, image_id = self.check_user_resources() # Create name of cluster with [orka] prefix cluster_name = '%s%s%s' % ('[orka]', '-', self.opts['cluster_name']) self.opts['cluster_name'] = cluster_name