Пример #1
0
    def bind_okeanos_resources(self):
        '''
        Create every available public ip for the user running the test, call
        bind_floating_ip to attach the ips and return whatever is needed
        for the selenium test.
        '''
        auth = check_credentials(self.token)
        endpoints, user_id = endpoints_and_user_id(auth)
        net_client = init_cyclades_netclient(endpoints['network'], self.token)

        dict_quotas = auth.get_quotas()
        # Find and create available public ips
        limit_ips = dict_quotas[
            self.project_id]['cyclades.floating_ip']['limit']
        usage_ips = dict_quotas[
            self.project_id]['cyclades.floating_ip']['usage']
        pending_ips = dict_quotas[
            self.project_id]['cyclades.floating_ip']['pending']
        available_ips = limit_ips - (usage_ips + pending_ips)

        if available_ips > 0:
            for i in range(available_ips):
                # Create all available public ips
                status = self.get_flo_net_id(net_client)
                if status != 0:
                    logging.error('Error in creating float ip')
                    sys.exit(error_get_ip)
        # Call bind_floating_ip to attach every unused ip
        float_ids, port_ids = self.bind_floating_ip(net_client)
        return float_ids, port_ids, net_client
Пример #2
0
    def bind_okeanos_resources(self):
        '''
        Binds all available private networks in ~okeanos
        for the user running the test.
        '''
        auth = check_credentials(self.token)
        endpoints, user_id = endpoints_and_user_id(auth)
        net_client = init_cyclades_netclient(endpoints['network'], self.token)
        dict_quotas = auth.get_quotas()
	project_id = get_project_id()
        limit_net = dict_quotas[project_id]['cyclades.network.private']['limit']
        usage_net = dict_quotas[project_id]['cyclades.network.private']['usage']
        pending_net = dict_quotas[project_id]['cyclades.network.private']['pending']
        available_networks = limit_net - usage_net - pending_net
        network_ids = []
        if available_networks >= 1:
            logging.info(' Private Network quota is ok')
            try:
                for i in range(available_networks):
                    new_network = net_client.create_network('MAC_FILTERED',
                                                            'mycluster '
                                                            + str(i),
                                                            project_id=project_id)
                    network_ids.append(new_network['id'])
                return net_client, network_ids
            except Exception:
                logging.exception('Error in creating network')
                sys.exit(error_create_network)
        else:
            logging.error('Private Network quota exceeded')
            return error_quotas_network
Пример #3
0
    def bind_okeanos_resources(self):
        '''
        Create every available public ip for the user running the test, call
        bind_floating_ip to attach the ips and return whatever is needed
        for the selenium test.
        '''
        auth = check_credentials(self.token)
        endpoints, user_id = endpoints_and_user_id(auth)
        net_client = init_cyclades_netclient(endpoints['network'],
                                             self.token)

        dict_quotas = auth.get_quotas()
        # Find and create available public ips
        limit_ips = dict_quotas[self.project_id]['cyclades.floating_ip']['limit']
        usage_ips = dict_quotas[self.project_id]['cyclades.floating_ip']['usage']
        pending_ips = dict_quotas[self.project_id]['cyclades.floating_ip']['pending']
        available_ips = limit_ips - (usage_ips + pending_ips)

        if available_ips > 0:
            for i in range(available_ips):
                # Create all available public ips
                status = self.get_flo_net_id(net_client)
                if status != 0:
                    logging.error('Error in creating float ip')
                    sys.exit(error_get_ip)
        # Call bind_floating_ip to attach every unused ip
        float_ids, port_ids = self.bind_floating_ip(net_client)
        return float_ids, port_ids, net_client
Пример #4
0
 def bind_okeanos_resources(self):
     '''
     Binds all available private networks in ~okeanos
     for the user running the test.
     '''
     auth = check_credentials(self.token)
     endpoints, user_id = endpoints_and_user_id(auth)
     net_client = init_cyclades_netclient(endpoints['network'], self.token)
     dict_quotas = auth.get_quotas()
     limit_net = dict_quotas[self.project_id]['cyclades.network.private']['limit']
     usage_net = dict_quotas[self.project_id]['cyclades.network.private']['usage']
     pending_net = dict_quotas[self.project_id]['cyclades.network.private']['pending']
     available_networks = limit_net - usage_net - pending_net
     network_ids = []
     if available_networks >= 1:
         logging.info(' Private Network quota is ok')
         try:
             for i in range(available_networks):
                 new_network = net_client.create_network('MAC_FILTERED',
                                                         'mycluster '
                                                         + str(i),
                                                         project_id=self.project_id)
                 network_ids.append(new_network['id'])
             return net_client, network_ids
         except Exception:
             logging.exception('Error in creating network')
             sys.exit(error_create_network)
     else:
         logging.error('Private Network quota exceeded')
         return error_quotas_network
Пример #5
0
    def create_bare_cluster(self):
        """
        This function of our script takes the arguments given and calls the
        check_quota function. Also, calls get_flavor_id to find the matching
        flavor_ids from the arguments given and finds the image id of the
        image given as argument. Then instantiates the Cluster and creates
        the virtual machine cluster of one master and clustersize-1 slaves.
        Calls the function to install hadoop to the cluster.
        """
        logging.log(REPORT, ' 1.Credentials  and  Endpoints')
        # Finds user public ssh key
        USER_HOME = expanduser('~')
        pub_keys_path = join(USER_HOME, ".ssh/id_rsa.pub")
        auth = check_credentials(self.opts['token'], self.opts['auth_url'])
        endpoints, user_id = endpoints_and_user_id(auth)
        cyclades = init_cyclades(endpoints['cyclades'], self.opts['token'])
        flavor_master = self.get_flavor_id_master(cyclades)
        flavor_slaves = self.get_flavor_id_slave(cyclades)
        if flavor_master == 0 or flavor_slaves == 0:
            logging.error('Combination of cpu, ram, disk and disk_template do'
                          ' not match an existing id')

            exit(error_flavor_id)
        # Total cpu,ram and disk needed for cluster
        cpu = self.opts['cpu_master'] + (self.opts['cpu_slave']) * (self.opts['clustersize'] - 1)
        ram = self.opts['ram_master'] + (self.opts['ram_slave']) * (self.opts['clustersize'] - 1)
        cyclades_disk = self.opts['disk_master'] + (self.opts['disk_slave']) * (self.opts['clustersize'] - 1)
        # The resources requested by user in a dictionary
        req_quotas = {'cpu': cpu, 'ram': ram, 'cyclades_disk': cyclades_disk,
                      'vms': self.opts['clustersize']}
        self.check_quota(auth, req_quotas)
        plankton = init_plankton(endpoints['plankton'], self.opts['token'])
        list_current_images = plankton.list_public(True, 'default')
        # Find image id of the arg given
        for lst in list_current_images:
            if lst['name'] == self.opts['image']:
                chosen_image = lst
                break
        else:
            logging.error(self.opts['image'] + ' is not a valid image option')
            exit(error_image_id)

        logging.log(REPORT, ' 2.Create  virtual  cluster')
        cluster = Cluster(cyclades,
                          prefix=self.opts['name'],
                          flavor_id_master=flavor_master,
                          flavor_id_slave=flavor_slaves,
                          image_id=chosen_image['id'],
                          size=self.opts['clustersize'],
                          net_client=init_cyclades_netclient(endpoints['network'],
                                                             self.opts['token']),
                          auth_cl=auth)

        self.HOSTNAME_MASTER_IP, self.server_dict = cluster.create('', pub_keys_path, '')
        sleep(15)
        # wait for the machines to be pingable
        logging.log(REPORT, ' Bare cluster has been created.')
        # Return master node ip and server dict
        return self.HOSTNAME_MASTER_IP, self.server_dict
Пример #6
0
    def __init__(self, opts):
        """Initialization of YarnCluster data attributes"""
        self.opts = opts
        # Master VM ip, placeholder value
        self.HOSTNAME_MASTER_IP = '127.0.0.1'
        # master VM root password file, placeholder value
        self.pass_file = 'PLACEHOLDER'
        self.orka_image_uuid = False
        # List of cluster VMs
        self.server_dict = {}
        if self.opts['disk_template'] == 'Archipelago':
            self.opts['disk_template'] = 'ext_vlmc'
        elif self.opts['disk_template'] == 'Standard':
            self.opts['disk_template'] = 'drbd'
        # project id of project name given as argument
        self.project_id = get_project_id(unmask_token(encrypt_key, self.opts['token']),
                                         self.opts['project_name'])
        self.status = {}
        # Instance of an AstakosClient object
        self.auth = check_credentials(unmask_token(encrypt_key, self.opts['token']),
                                      self.opts.get('auth_url',
                                                    auth_url))
        # Check if project has actual quota
        if self.check_project_quota() != 0:
            msg = 'Project %s exists but you have no quota to request' % \
                self.opts['project_name']
            raise ClientError(msg, error_project_quota)
        # ~okeanos endpoints and user id
        self.endpoints, self.user_id = endpoints_and_user_id(self.auth)

        # Instance of CycladesClient
        self.cyclades = init_cyclades(self.endpoints['cyclades'],
                                      unmask_token(encrypt_key, self.opts['token']))
        # Instance of CycladesNetworkClient
        self.net_client = init_cyclades_netclient(self.endpoints['network'],
                                                  unmask_token(encrypt_key, self.opts['token']))
        # Instance of Plankton/ImageClient
        self.plankton = init_plankton(self.endpoints['plankton'],
                                      unmask_token(encrypt_key, self.opts['token']))
        # Get resources of pending clusters
        self.pending_quota = retrieve_pending_clusters(unmask_token(encrypt_key, self.opts['token']),
                                                       self.opts['project_name'])
        self._DispatchCheckers = {}
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_cluster_size_quotas
        # Check for private network availability only when cluster is created
        # and not for Vre server creation
        if self.opts['cluster_size'] > 1:
            self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
                self.check_network_quotas
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_ip_quotas
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_cpu_valid
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_ram_valid
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_disk_valid
Пример #7
0
    def __init__(self, opts):
        """Initialization of YarnCluster data attributes"""
        self.opts = opts
        # Master VM ip, placeholder value
        self.HOSTNAME_MASTER_IP = '127.0.0.1'
        # master VM root password file, placeholder value
        self.pass_file = 'PLACEHOLDER'
        self.orka_image_uuid = False
        # List of cluster VMs
        self.server_dict = {}
        if self.opts['disk_template'] == 'Archipelago':
            self.opts['disk_template'] = 'ext_vlmc'
        elif self.opts['disk_template'] == 'Standard':
            self.opts['disk_template'] = 'drbd'
        # project id of project name given as argument
        self.project_id = get_project_id(self.opts['token'],
                                         self.opts['project_name'])
        self.status = {}
        # Instance of an AstakosClient object
        self.auth = check_credentials(self.opts['token'],
                                      self.opts.get('auth_url', auth_url))
        # Check if project has actual quota
        if self.check_project_quota() != 0:
            msg = 'Project %s exists but you have no quota to request' % \
                self.opts['project_name']
            raise ClientError(msg, error_project_quota)
        # ~okeanos endpoints and user id
        self.endpoints, self.user_id = endpoints_and_user_id(self.auth)

        # Instance of CycladesClient
        self.cyclades = init_cyclades(self.endpoints['cyclades'],
                                      self.opts['token'])
        # Instance of CycladesNetworkClient
        self.net_client = init_cyclades_netclient(self.endpoints['network'],
                                                  self.opts['token'])
        # Instance of Plankton/ImageClient
        self.plankton = init_plankton(self.endpoints['plankton'],
                                      self.opts['token'])
        # Get resources of pending clusters
        self.pending_quota = retrieve_pending_clusters(
            self.opts['token'], self.opts['project_name'])
        self._DispatchCheckers = {}
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_cluster_size_quotas
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_network_quotas
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_ip_quotas
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_cpu_valid
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_ram_valid
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_disk_valid
Пример #8
0
 def check_ip_quotas(self):
     dict_quotas = self.auth.get_quotas()
     endpoints, user_id = endpoints_and_user_id(self.auth)
     net_client = init_cyclades_netclient(endpoints['network'], self.opts['token'])
     list_float_ips = net_client.list_floatingips()
     limit_ips = dict_quotas['system']['cyclades.floating_ip']['limit']
     usage_ips = dict_quotas['system']['cyclades.floating_ip']['usage']
     pending_ips = dict_quotas['system']['cyclades.floating_ip']['pending']
     available_ips = limit_ips - (usage_ips + pending_ips)
     for d in list_float_ips:
         if d['instance_id'] is None:
             available_ips += 1
     if available_ips > 0:
         return 0
     else:
         logging.error('Floating IP not available')
         return error_get_ip
Пример #9
0
    def __init__(self, opts):
        """Initialization of YarnCluster data attributes"""
        self.opts = opts
        # Master VM ip, placeholder value
        self.HOSTNAME_MASTER_IP = '127.0.0.1'
        # master VM root password file, placeholder value
        self.pass_file = 'PLACEHOLDER'
        self.hadoop_image = False
        # List of cluster VMs
        self.server_dict = {}
        if self.opts['disk_template'] == 'Archipelago':
            self.opts['disk_template'] = 'ext_vlmc'
        elif self.opts['disk_template'] == 'Standard':
            self.opts['disk_template'] = 'drbd'
        # project id of project name given as argument
        self.project_id = get_project_id(self.opts['token'],
                                         self.opts['project_name'])
        self.status = {}
        # Instance of an AstakosClient object
        self.auth = check_credentials(self.opts['token'],
                                      self.opts.get('auth_url',
                                                    auth_url))
        # Check if project has actual quota
        if self.check_project_quota() != 0:
            msg = 'Project %s exists but you have no quota to request' % \
                self.opts['project_name']
            raise ClientError(msg, error_project_quota)
        # ~okeanos endpoints and user id
        self.endpoints, self.user_id = endpoints_and_user_id(self.auth)

        # Instance of CycladesClient
        self.cyclades = init_cyclades(self.endpoints['cyclades'],
                                      self.opts['token'])
        # Instance of CycladesNetworkClient
        self.net_client = init_cyclades_netclient(self.endpoints['network'],
                                                  self.opts['token'])
        # Instance of Plankton/ImageClient
        self.plankton = init_plankton(self.endpoints['plankton'],
                                      self.opts['token'])
        # Get resources of pending clusters
        self.pending_quota = retrieve_pending_clusters(self.opts['token'],
                                                       self.opts['project_name'])
        # check escienceconf flag and set hadoop_image accordingly
        list_current_images = self.plankton.list_public(True, 'default')
        for image in list_current_images:
            if self.opts['os_choice'] == image['name']:
                try:
                    if image['properties']['escienceconf']:
                        image_metadata = json.loads(image['properties']['escienceconf'])
                        if image_metadata['hadoop'] == 'True' and image_metadata['hue'] == 'True':
                            self.hadoop_image = 'hue'
                        elif image_metadata['hadoop'] == 'False':
                            self.hadoop_image = 'debianbase'
                        else:
                            self.hadoop_image = 'hadoopbase'

                except:
                    # if property hasn't been set then hadoop_image flag is false
                    self.hadoop_image = 'debianbase'
                        
        self._DispatchCheckers = {}
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_cluster_size_quotas
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_network_quotas
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_ip_quotas
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_cpu_valid
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_ram_valid
        self._DispatchCheckers[len(self._DispatchCheckers) + 1] =\
            self.check_disk_valid