def create_bare_cluster(self): """ This function of our script takes the arguments given and calls the check_quota function. Also, calls get_flavor_id to find the matching flavor_ids from the arguments given and finds the image id of the image given as argument. Then instantiates the Cluster and creates the virtual machine cluster of one master and clustersize-1 slaves. Calls the function to install hadoop to the cluster. """ logging.log(REPORT, ' 1.Credentials and Endpoints') # Finds user public ssh key USER_HOME = expanduser('~') pub_keys_path = join(USER_HOME, ".ssh/id_rsa.pub") auth = check_credentials(self.opts['token'], self.opts['auth_url']) endpoints, user_id = endpoints_and_user_id(auth) cyclades = init_cyclades(endpoints['cyclades'], self.opts['token']) flavor_master = self.get_flavor_id_master(cyclades) flavor_slaves = self.get_flavor_id_slave(cyclades) if flavor_master == 0 or flavor_slaves == 0: logging.error('Combination of cpu, ram, disk and disk_template do' ' not match an existing id') exit(error_flavor_id) # Total cpu,ram and disk needed for cluster cpu = self.opts['cpu_master'] + (self.opts['cpu_slave']) * (self.opts['clustersize'] - 1) ram = self.opts['ram_master'] + (self.opts['ram_slave']) * (self.opts['clustersize'] - 1) cyclades_disk = self.opts['disk_master'] + (self.opts['disk_slave']) * (self.opts['clustersize'] - 1) # The resources requested by user in a dictionary req_quotas = {'cpu': cpu, 'ram': ram, 'cyclades_disk': cyclades_disk, 'vms': self.opts['clustersize']} self.check_quota(auth, req_quotas) plankton = init_plankton(endpoints['plankton'], self.opts['token']) list_current_images = plankton.list_public(True, 'default') # Find image id of the arg given for lst in list_current_images: if lst['name'] == self.opts['image']: chosen_image = lst break else: logging.error(self.opts['image'] + ' is not a valid image option') exit(error_image_id) logging.log(REPORT, ' 2.Create virtual cluster') cluster = Cluster(cyclades, prefix=self.opts['name'], flavor_id_master=flavor_master, flavor_id_slave=flavor_slaves, image_id=chosen_image['id'], size=self.opts['clustersize'], net_client=init_cyclades_netclient(endpoints['network'], self.opts['token']), auth_cl=auth) self.HOSTNAME_MASTER_IP, self.server_dict = cluster.create('', pub_keys_path, '') sleep(15) # wait for the machines to be pingable logging.log(REPORT, ' Bare cluster has been created.') # Return master node ip and server dict return self.HOSTNAME_MASTER_IP, self.server_dict
def __init__(self, opts): """Initialization of YarnCluster data attributes""" self.opts = opts # Master VM ip, placeholder value self.HOSTNAME_MASTER_IP = '127.0.0.1' # master VM root password file, placeholder value self.pass_file = 'PLACEHOLDER' self.orka_image_uuid = False # List of cluster VMs self.server_dict = {} if self.opts['disk_template'] == 'Archipelago': self.opts['disk_template'] = 'ext_vlmc' elif self.opts['disk_template'] == 'Standard': self.opts['disk_template'] = 'drbd' # project id of project name given as argument self.project_id = get_project_id(unmask_token(encrypt_key, self.opts['token']), self.opts['project_name']) self.status = {} # Instance of an AstakosClient object self.auth = check_credentials(unmask_token(encrypt_key, self.opts['token']), self.opts.get('auth_url', auth_url)) # Check if project has actual quota if self.check_project_quota() != 0: msg = 'Project %s exists but you have no quota to request' % \ self.opts['project_name'] raise ClientError(msg, error_project_quota) # ~okeanos endpoints and user id self.endpoints, self.user_id = endpoints_and_user_id(self.auth) # Instance of CycladesClient self.cyclades = init_cyclades(self.endpoints['cyclades'], unmask_token(encrypt_key, self.opts['token'])) # Instance of CycladesNetworkClient self.net_client = init_cyclades_netclient(self.endpoints['network'], unmask_token(encrypt_key, self.opts['token'])) # Instance of Plankton/ImageClient self.plankton = init_plankton(self.endpoints['plankton'], unmask_token(encrypt_key, self.opts['token'])) # Get resources of pending clusters self.pending_quota = retrieve_pending_clusters(unmask_token(encrypt_key, self.opts['token']), self.opts['project_name']) self._DispatchCheckers = {} self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_cluster_size_quotas # Check for private network availability only when cluster is created # and not for Vre server creation if self.opts['cluster_size'] > 1: self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_network_quotas self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_ip_quotas self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_cpu_valid self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_ram_valid self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_disk_valid
def __init__(self, opts): """Initialization of YarnCluster data attributes""" self.opts = opts # Master VM ip, placeholder value self.HOSTNAME_MASTER_IP = '127.0.0.1' # master VM root password file, placeholder value self.pass_file = 'PLACEHOLDER' self.orka_image_uuid = False # List of cluster VMs self.server_dict = {} if self.opts['disk_template'] == 'Archipelago': self.opts['disk_template'] = 'ext_vlmc' elif self.opts['disk_template'] == 'Standard': self.opts['disk_template'] = 'drbd' # project id of project name given as argument self.project_id = get_project_id(self.opts['token'], self.opts['project_name']) self.status = {} # Instance of an AstakosClient object self.auth = check_credentials(self.opts['token'], self.opts.get('auth_url', auth_url)) # Check if project has actual quota if self.check_project_quota() != 0: msg = 'Project %s exists but you have no quota to request' % \ self.opts['project_name'] raise ClientError(msg, error_project_quota) # ~okeanos endpoints and user id self.endpoints, self.user_id = endpoints_and_user_id(self.auth) # Instance of CycladesClient self.cyclades = init_cyclades(self.endpoints['cyclades'], self.opts['token']) # Instance of CycladesNetworkClient self.net_client = init_cyclades_netclient(self.endpoints['network'], self.opts['token']) # Instance of Plankton/ImageClient self.plankton = init_plankton(self.endpoints['plankton'], self.opts['token']) # Get resources of pending clusters self.pending_quota = retrieve_pending_clusters( self.opts['token'], self.opts['project_name']) self._DispatchCheckers = {} self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_cluster_size_quotas self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_network_quotas self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_ip_quotas self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_cpu_valid self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_ram_valid self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_disk_valid
def __init__(self, opts): """Initialization of YarnCluster data attributes""" self.opts = opts # Master VM ip, placeholder value self.HOSTNAME_MASTER_IP = '127.0.0.1' # master VM root password file, placeholder value self.pass_file = 'PLACEHOLDER' self.hadoop_image = False # List of cluster VMs self.server_dict = {} if self.opts['disk_template'] == 'Archipelago': self.opts['disk_template'] = 'ext_vlmc' elif self.opts['disk_template'] == 'Standard': self.opts['disk_template'] = 'drbd' # project id of project name given as argument self.project_id = get_project_id(self.opts['token'], self.opts['project_name']) self.status = {} # Instance of an AstakosClient object self.auth = check_credentials(self.opts['token'], self.opts.get('auth_url', auth_url)) # Check if project has actual quota if self.check_project_quota() != 0: msg = 'Project %s exists but you have no quota to request' % \ self.opts['project_name'] raise ClientError(msg, error_project_quota) # ~okeanos endpoints and user id self.endpoints, self.user_id = endpoints_and_user_id(self.auth) # Instance of CycladesClient self.cyclades = init_cyclades(self.endpoints['cyclades'], self.opts['token']) # Instance of CycladesNetworkClient self.net_client = init_cyclades_netclient(self.endpoints['network'], self.opts['token']) # Instance of Plankton/ImageClient self.plankton = init_plankton(self.endpoints['plankton'], self.opts['token']) # Get resources of pending clusters self.pending_quota = retrieve_pending_clusters(self.opts['token'], self.opts['project_name']) # check escienceconf flag and set hadoop_image accordingly list_current_images = self.plankton.list_public(True, 'default') for image in list_current_images: if self.opts['os_choice'] == image['name']: try: if image['properties']['escienceconf']: image_metadata = json.loads(image['properties']['escienceconf']) if image_metadata['hadoop'] == 'True' and image_metadata['hue'] == 'True': self.hadoop_image = 'hue' elif image_metadata['hadoop'] == 'False': self.hadoop_image = 'debianbase' else: self.hadoop_image = 'hadoopbase' except: # if property hasn't been set then hadoop_image flag is false self.hadoop_image = 'debianbase' self._DispatchCheckers = {} self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_cluster_size_quotas self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_network_quotas self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_ip_quotas self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_cpu_valid self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_ram_valid self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\ self.check_disk_valid