Ejemplo n.º 1
0
    def bind_okeanos_resources(self, remaining_ram, ram_list):
        '''
        Create a bare cluster in ~okeanos with two vms. The ram depend
        on remaining_ram argument.
        '''
        if remaining_ram == 0:
	    opts = {"name": self.name,
                              "clustersize": 2,
                              "cpu_master": 1, "ram_master": 1024, "disk_master": 5,
                              "disk_template":'Archipelago', "cpu_slave": 1,
                              "ram_slave": 1024, "disk_slave": 5, "token": self.token,
                              "image": 'Debian Base', "project_name": self.project_name}
	    c_yarn_cluster = YarnCluster(opts)
            return c_yarn_cluster.create_bare_cluster()

        else:
            for ram in ram_list:
                if ram >= remaining_ram:
                    remaining_ram = ram
                    opts = {"name": self.name,
                              "clustersize": 2,
                              "cpu_master": 1, "ram_master": remaining_ram, "disk_master": 5,
                              "disk_template":'Archipelago', "cpu_slave": 1,
                              "ram_slave": remaining_ram, "disk_slave": 5, "token": self.token,
                              "image": 'Debian Base', "project_name": self.project_name}
	            c_yarn_cluster = YarnCluster(opts)
                    return c_yarn_cluster.create_bare_cluster()
Ejemplo n.º 2
0
    def bind_okeanos_resources(self, remaining_cpu, cpu_list):
        '''
        Create a bare cluster in ~okeanos with two vms. The cpus depend
        on remaining_cpu argument.
        '''
        if remaining_cpu == 0:
            opts = {"name": self.name,
                              "clustersize": 2,
                              "cpu_master": 1, "ram_master": 1024, "disk_master": 5,
                              "disk_template":'Standard', "cpu_slave": 1,
                              "ram_slave": 1024, "disk_slave": 5, "token": self.token,
                              "image": 'Debian Base', "project_name": self.project_name}
	    c_yarn_cluster = YarnCluster(opts)
            return c_yarn_cluster.create_bare_cluster()

        else:
            for cpu in cpu_list:
                if cpu >= remaining_cpu:
                    remaining_cpu = cpu
                    opts = {"name": self.name,
                              "clustersize": 2,
                              "cpu_master": remaining_cpu, "ram_master": 1024, "disk_master": 5,
                              "disk_template":'Standard', "cpu_slave": remaining_cpu,
                              "ram_slave": 1024, "disk_slave": 5, "token": self.token,
                              "image": 'Debian Base', "project_name": self.project_name}
	            c_yarn_cluster = YarnCluster(opts)
                    return c_yarn_cluster.create_bare_cluster()
Ejemplo n.º 3
0
def create_server_async(choices):
    """
    Asynchronous create VRE server task.
    """
    new_vre_server = YarnCluster(choices)
    server_id, server_pass, server_ip = new_vre_server.create_vre_server()
    task_result = {"server_IP": server_ip, "VRE_VM_password": server_pass, "server_id": server_id}
    return task_result
Ejemplo n.º 4
0
 def test_check_network_quotas_sufficient(self):
     # arrange
     c_yarn_cluster = YarnCluster(self.opts)
     expected = 0 # success
     # act
     returned = c_yarn_cluster.check_network_quotas()
     #assert
     self.assertEqual(expected, returned)
Ejemplo n.º 5
0
 def test_check_all_resources(self):
     # arrange
     c_yarn_cluster = YarnCluster(self.opts)
     expected = 0
     # act
     returned = c_yarn_cluster.check_all_resources()
     # assert
     self.assertTrue(True) # temporarily short-ciruit
Ejemplo n.º 6
0
 def test_create_bare_cluster(self):
     # arrange
     expected_masterip = '127.0.0.1'
     expected_vm_dict = {1: 'f vm'}
     c_yarn_cluster = YarnCluster(self.opts)
     # act
     returned_masterip, returned_vm_dict = c_yarn_cluster.create_bare_cluster()
     # assert
     self.assertTupleEqual((expected_masterip, expected_vm_dict), (returned_masterip, returned_vm_dict))
Ejemplo n.º 7
0
def create_cluster_async(choices):
    """
    Asynchronous create cluster task.
    """
    new_yarn_cluster = YarnCluster(choices)
    MASTER_IP, servers, password = new_yarn_cluster.create_yarn_cluster()
    task_result = {"master_IP": MASTER_IP, "master_VM_password": password}

    return task_result
Ejemplo n.º 8
0
 def test_check_cpu_valid_exceeded(self):
     # arrange
     prev_clustersize = self.opts['clustersize']
     self.opts['clustersize'] = 3
     c_yarn_cluster = YarnCluster(self.opts)
     expected = error_quotas_cpu # usage: 5, limit: 9 (4 remaining), requested: 2 + 4, expect result error
     # act
     returned = c_yarn_cluster.check_cpu_valid()
     self.opts['clustersize'] = prev_clustersize
     #assert
     self.assertEqual(expected, returned)
Ejemplo n.º 9
0
 def test_check_cpu_valid_sufficient(self):
     # arrange
     prev_clustersize = self.opts['clustersize']
     self.opts['clustersize'] = 2
     c_yarn_cluster = YarnCluster(self.opts)
     expected = 0 # usage: 5, limit: 9 (4 remaining), requested: 4, expected result success
     # act
     returned = c_yarn_cluster.check_cpu_valid()
     self.opts['clustersize'] = prev_clustersize
     #assert
     self.assertEqual(expected, returned)
Ejemplo n.º 10
0
 def test_check_clustersize_quotas_exceeded(self):
     # arrange
     prev_clustersize = self.opts['clustersize']
     self.opts['clustersize'] = 3
     c_yarn_cluster = YarnCluster(self.opts)
     expected = error_quotas_clustersize # usage: 4, limit: 6 (2 remaining), requested: 3, expect error
     # act
     returned = c_yarn_cluster.check_clustersize_quotas()
     self.opts['clustersize'] = prev_clustersize
     #assert
     self.assertEqual(expected, returned)
Ejemplo n.º 11
0
def create_server_async(choices):
    """
    Asynchronous create VRE server task.
    """
    new_vre_server = YarnCluster(choices)
    server_id, server_pass, server_ip = new_vre_server.create_vre_server()
    task_result = {
        "server_IP": server_ip,
        "VRE_VM_password": server_pass,
        "server_id": server_id
    }
    return task_result
Ejemplo n.º 12
0
 def test_check_all_resources(self):
     # arrange
     opts = {'name': 'Test', 'clustersize': 2, 'cpu_master': 2,
             'ram_master': 4096, 'disk_master': 5, 'cpu_slave': 2,
             'ram_slave': 2048, 'disk_slave': 5, 'token': self.token,
             'disk_template': 'ext_vlmc', 'image': 'Debian Base',
             'auth_url': self.auth_url}
     c_yarn_cluster = YarnCluster(opts)
     expected = 0
     # act
     returned = c_yarn_cluster.check_all_resources()
     # assert
     self.assertTrue(True) # temporarily short-ciruit
Ejemplo n.º 13
0
    def bind_okeanos_resources(self):
        '''
        Create a bare cluster with two vms, so we can bind the
        resources in ~okeanos
        '''
	opts = {"name": self.name,
                              "clustersize": 2,
                              "cpu_master": 1, "ram_master": 1024, "disk_master": 5,
                              "disk_template":'Standard', "cpu_slave": 1,
                              "ram_slave": 1024, "disk_slave": 5, "token": self.token,
                              "image": 'Debian Base', "project_name": self.project_name}
	c_yarn_cluster = YarnCluster(opts)
        return c_yarn_cluster.create_bare_cluster()
Ejemplo n.º 14
0
    def bind_okeanos_resources(self):
        '''
        Create a bare cluster with two vms, so we can bind the
        resources in ~okeanos
        '''
	opts = {"name": self.name,
                              "clustersize": 2,
                              "cpu_master": 1, "ram_master": 1024, "disk_master": 5,
                              "disk_template":'Archipelago', "cpu_slave": 1,
                              "ram_slave": 1024, "disk_slave": 5, "token": self.token,
                              "image": 'Debian Base', "project_name": self.project_name}
	c_yarn_cluster = YarnCluster(opts)
        return c_yarn_cluster.create_bare_cluster()
Ejemplo n.º 15
0
 def test_create_yarn_cluster(self):
     # arrange
     opts = {'name': 'Test', 'clustersize': 2, 'cpu_master': 2,
             'ram_master': 4096, 'disk_master': 5, 'cpu_slave': 2,
             'ram_slave': 2048, 'disk_slave': 5, 'token': self.token,
             'disk_template': 'ext_vlmc', 'image': 'Debian Base',
             'auth_url': self.auth_url, 'yarn': True}
     expected_masterip = '127.0.0.1'
     expected_vm_dict = {1: 'f vm'}
     c_yarn_cluster = YarnCluster(opts)
     # act
     returned_masterip, returned_vm_dict = c_yarn_cluster.create_yarn_cluster()
     # assert
     self.assertTupleEqual((expected_masterip, expected_vm_dict), (returned_masterip, returned_vm_dict))
Ejemplo n.º 16
0
def create_cluster_async(choices):
    """
    Asynchronous create cluster task.
    """
    new_yarn_cluster = YarnCluster(choices)
    MASTER_IP, servers, password, cluster_id = new_yarn_cluster.create_yarn_cluster(
    )
    task_result = {
        "master_IP": MASTER_IP,
        "master_VM_password": password,
        "cluster_id": cluster_id
    }

    return task_result
Ejemplo n.º 17
0
    def put(self, request, *args, **kwargs):
        """
        Handles ember requests with user's cluster creation parameters.
        Check the parameters with HadoopCluster object from create_cluster
        script.
        """
        self.resource_name = 'clusterchoice'
        self.serializer_class = ClusterchoicesSerializer
        serializer = self.serializer_class(data=request.DATA)
        if serializer.is_valid():
            user_token = Token.objects.get(key=request.auth)
            user = UserInfo.objects.get(user_id=user_token.user.user_id)
            if serializer.data['hadoop_status']:
                try:
                    cluster_action = hadoop_cluster_action_async.delay(
                        serializer.data['id'],
                        serializer.data['hadoop_status'])
                    task_id = cluster_action.id
                    return Response({
                        "id": 1,
                        "task_id": task_id
                    },
                                    status=status.HTTP_202_ACCEPTED)
                except Exception, e:
                    return Response({"status": str(e.args[0])})

            # Dictionary of YarnCluster arguments
            choices = dict()
            choices = serializer.data.copy()
            choices.update({'token': user.okeanos_token})
            try:
                YarnCluster(choices).check_user_resources()
            except ClientError, e:
                return Response({"id": 1, "message": e.message})
Ejemplo n.º 18
0
    def bind_okeanos_resources(self, remaining_disk, disk_list):
        '''
        Create a bare cluster in ~okeanos with two vms. The disk size depend
        on remaining_disk argument.
        '''
        if remaining_disk == 0:
            opts = {
                "name": self.name,
                "clustersize": 2,
                "cpu_master": 1,
                "ram_master": 1024,
                "disk_master": 5,
                "disk_template": 'Archipelago',
                "cpu_slave": 1,
                "ram_slave": 1024,
                "disk_slave": 5,
                "token": self.token,
                "image": 'Debian Base',
                "project_name": self.project_name
            }
            c_yarn_cluster = YarnCluster(opts)
            return c_yarn_cluster.create_bare_cluster()

        else:
            for disk in disk_list:
                if disk >= remaining_disk:
                    remaining_disk = disk
                    opts = {
                        "name": self.name,
                        "clustersize": 2,
                        "cpu_master": 1,
                        "ram_master": 1024,
                        "disk_master": remaining_disk,
                        "disk_template": 'Archipelago',
                        "cpu_slave": 1,
                        "ram_slave": 1024,
                        "disk_slave": remaining_disk,
                        "token": self.token,
                        "image": 'Debian Base',
                        "project_name": self.project_name
                    }
                    c_yarn_cluster = YarnCluster(opts)
                    return c_yarn_cluster.create_bare_cluster()