def bind_okeanos_resources(self, remaining_ram, ram_list):
'''
Create a bare cluster in ~okeanos with two vms. The ram depend
on remaining_ram argument.
'''
if remaining_ram == 0:
opts = {"name": self.name,
"clustersize": 2,
"cpu_master": 1, "ram_master": 1024, "disk_master": 5,
"disk_template":'Archipelago', "cpu_slave": 1,
"ram_slave": 1024, "disk_slave": 5, "token": self.token,
"image": 'Debian Base', "project_name": self.project_name}
c_yarn_cluster = YarnCluster(opts)
return c_yarn_cluster.create_bare_cluster()
else:
for ram in ram_list:
if ram >= remaining_ram:
remaining_ram = ram
opts = {"name": self.name,
"clustersize": 2,
"cpu_master": 1, "ram_master": remaining_ram, "disk_master": 5,
"disk_template":'Archipelago', "cpu_slave": 1,
"ram_slave": remaining_ram, "disk_slave": 5, "token": self.token,
"image": 'Debian Base', "project_name": self.project_name}
c_yarn_cluster = YarnCluster(opts)
return c_yarn_cluster.create_bare_cluster()
def bind_okeanos_resources(self, remaining_cpu, cpu_list):
'''
Create a bare cluster in ~okeanos with two vms. The cpus depend
on remaining_cpu argument.
'''
if remaining_cpu == 0:
opts = {"name": self.name,
"clustersize": 2,
"cpu_master": 1, "ram_master": 1024, "disk_master": 5,
"disk_template":'Standard', "cpu_slave": 1,
"ram_slave": 1024, "disk_slave": 5, "token": self.token,
"image": 'Debian Base', "project_name": self.project_name}
c_yarn_cluster = YarnCluster(opts)
return c_yarn_cluster.create_bare_cluster()
else:
for cpu in cpu_list:
if cpu >= remaining_cpu:
remaining_cpu = cpu
opts = {"name": self.name,
"clustersize": 2,
"cpu_master": remaining_cpu, "ram_master": 1024, "disk_master": 5,
"disk_template":'Standard', "cpu_slave": remaining_cpu,
"ram_slave": 1024, "disk_slave": 5, "token": self.token,
"image": 'Debian Base', "project_name": self.project_name}
c_yarn_cluster = YarnCluster(opts)
return c_yarn_cluster.create_bare_cluster()
def create_server_async(choices):
"""
Asynchronous create VRE server task.
"""
new_vre_server = YarnCluster(choices)
server_id, server_pass, server_ip = new_vre_server.create_vre_server()
task_result = {"server_IP": server_ip, "VRE_VM_password": server_pass, "server_id": server_id}
return task_result
def test_check_network_quotas_sufficient(self):
# arrange
c_yarn_cluster = YarnCluster(self.opts)
expected = 0 # success
# act
returned = c_yarn_cluster.check_network_quotas()
#assert
self.assertEqual(expected, returned)
def test_check_all_resources(self):
# arrange
c_yarn_cluster = YarnCluster(self.opts)
expected = 0
# act
returned = c_yarn_cluster.check_all_resources()
# assert
self.assertTrue(True) # temporarily short-ciruit
def test_create_bare_cluster(self):
# arrange
expected_masterip = '127.0.0.1'
expected_vm_dict = {1: 'f vm'}
c_yarn_cluster = YarnCluster(self.opts)
# act
returned_masterip, returned_vm_dict = c_yarn_cluster.create_bare_cluster()
# assert
self.assertTupleEqual((expected_masterip, expected_vm_dict), (returned_masterip, returned_vm_dict))
def create_cluster_async(choices):
"""
Asynchronous create cluster task.
"""
new_yarn_cluster = YarnCluster(choices)
MASTER_IP, servers, password = new_yarn_cluster.create_yarn_cluster()
task_result = {"master_IP": MASTER_IP, "master_VM_password": password}
return task_result
def test_check_cpu_valid_exceeded(self):
# arrange
prev_clustersize = self.opts['clustersize']
self.opts['clustersize'] = 3
c_yarn_cluster = YarnCluster(self.opts)
expected = error_quotas_cpu # usage: 5, limit: 9 (4 remaining), requested: 2 + 4, expect result error
# act
returned = c_yarn_cluster.check_cpu_valid()
self.opts['clustersize'] = prev_clustersize
#assert
self.assertEqual(expected, returned)
def test_check_cpu_valid_sufficient(self):
# arrange
prev_clustersize = self.opts['clustersize']
self.opts['clustersize'] = 2
c_yarn_cluster = YarnCluster(self.opts)
expected = 0 # usage: 5, limit: 9 (4 remaining), requested: 4, expected result success
# act
returned = c_yarn_cluster.check_cpu_valid()
self.opts['clustersize'] = prev_clustersize
#assert
self.assertEqual(expected, returned)
def test_check_clustersize_quotas_exceeded(self):
# arrange
prev_clustersize = self.opts['clustersize']
self.opts['clustersize'] = 3
c_yarn_cluster = YarnCluster(self.opts)
expected = error_quotas_clustersize # usage: 4, limit: 6 (2 remaining), requested: 3, expect error
# act
returned = c_yarn_cluster.check_clustersize_quotas()
self.opts['clustersize'] = prev_clustersize
#assert
self.assertEqual(expected, returned)
def create_server_async(choices):
"""
Asynchronous create VRE server task.
"""
new_vre_server = YarnCluster(choices)
server_id, server_pass, server_ip = new_vre_server.create_vre_server()
task_result = {
"server_IP": server_ip,
"VRE_VM_password": server_pass,
"server_id": server_id
}
return task_result
def test_check_all_resources(self):
# arrange
opts = {'name': 'Test', 'clustersize': 2, 'cpu_master': 2,
'ram_master': 4096, 'disk_master': 5, 'cpu_slave': 2,
'ram_slave': 2048, 'disk_slave': 5, 'token': self.token,
'disk_template': 'ext_vlmc', 'image': 'Debian Base',
'auth_url': self.auth_url}
c_yarn_cluster = YarnCluster(opts)
expected = 0
# act
returned = c_yarn_cluster.check_all_resources()
# assert
self.assertTrue(True) # temporarily short-ciruit
def bind_okeanos_resources(self):
'''
Create a bare cluster with two vms, so we can bind the
resources in ~okeanos
'''
opts = {"name": self.name,
"clustersize": 2,
"cpu_master": 1, "ram_master": 1024, "disk_master": 5,
"disk_template":'Standard', "cpu_slave": 1,
"ram_slave": 1024, "disk_slave": 5, "token": self.token,
"image": 'Debian Base', "project_name": self.project_name}
c_yarn_cluster = YarnCluster(opts)
return c_yarn_cluster.create_bare_cluster()
def bind_okeanos_resources(self):
'''
Create a bare cluster with two vms, so we can bind the
resources in ~okeanos
'''
opts = {"name": self.name,
"clustersize": 2,
"cpu_master": 1, "ram_master": 1024, "disk_master": 5,
"disk_template":'Archipelago', "cpu_slave": 1,
"ram_slave": 1024, "disk_slave": 5, "token": self.token,
"image": 'Debian Base', "project_name": self.project_name}
c_yarn_cluster = YarnCluster(opts)
return c_yarn_cluster.create_bare_cluster()
def test_create_yarn_cluster(self):
# arrange
opts = {'name': 'Test', 'clustersize': 2, 'cpu_master': 2,
'ram_master': 4096, 'disk_master': 5, 'cpu_slave': 2,
'ram_slave': 2048, 'disk_slave': 5, 'token': self.token,
'disk_template': 'ext_vlmc', 'image': 'Debian Base',
'auth_url': self.auth_url, 'yarn': True}
expected_masterip = '127.0.0.1'
expected_vm_dict = {1: 'f vm'}
c_yarn_cluster = YarnCluster(opts)
# act
returned_masterip, returned_vm_dict = c_yarn_cluster.create_yarn_cluster()
# assert
self.assertTupleEqual((expected_masterip, expected_vm_dict), (returned_masterip, returned_vm_dict))
def create_cluster_async(choices):
"""
Asynchronous create cluster task.
"""
new_yarn_cluster = YarnCluster(choices)
MASTER_IP, servers, password, cluster_id = new_yarn_cluster.create_yarn_cluster(
)
task_result = {
"master_IP": MASTER_IP,
"master_VM_password": password,
"cluster_id": cluster_id
}
return task_result
def put(self, request, *args, **kwargs):
"""
Handles ember requests with user's cluster creation parameters.
Check the parameters with HadoopCluster object from create_cluster
script.
"""
self.resource_name = 'clusterchoice'
self.serializer_class = ClusterchoicesSerializer
serializer = self.serializer_class(data=request.DATA)
if serializer.is_valid():
user_token = Token.objects.get(key=request.auth)
user = UserInfo.objects.get(user_id=user_token.user.user_id)
if serializer.data['hadoop_status']:
try:
cluster_action = hadoop_cluster_action_async.delay(
serializer.data['id'],
serializer.data['hadoop_status'])
task_id = cluster_action.id
return Response({
"id": 1,
"task_id": task_id
},
status=status.HTTP_202_ACCEPTED)
except Exception, e:
return Response({"status": str(e.args[0])})
# Dictionary of YarnCluster arguments
choices = dict()
choices = serializer.data.copy()
choices.update({'token': user.okeanos_token})
try:
YarnCluster(choices).check_user_resources()
except ClientError, e:
return Response({"id": 1, "message": e.message})
def bind_okeanos_resources(self, remaining_disk, disk_list):
'''
Create a bare cluster in ~okeanos with two vms. The disk size depend
on remaining_disk argument.
'''
if remaining_disk == 0:
opts = {
"name": self.name,
"clustersize": 2,
"cpu_master": 1,
"ram_master": 1024,
"disk_master": 5,
"disk_template": 'Archipelago',
"cpu_slave": 1,
"ram_slave": 1024,
"disk_slave": 5,
"token": self.token,
"image": 'Debian Base',
"project_name": self.project_name
}
c_yarn_cluster = YarnCluster(opts)
return c_yarn_cluster.create_bare_cluster()
else:
for disk in disk_list:
if disk >= remaining_disk:
remaining_disk = disk
opts = {
"name": self.name,
"clustersize": 2,
"cpu_master": 1,
"ram_master": 1024,
"disk_master": remaining_disk,
"disk_template": 'Archipelago',
"cpu_slave": 1,
"ram_slave": 1024,
"disk_slave": remaining_disk,
"token": self.token,
"image": 'Debian Base',
"project_name": self.project_name
}
c_yarn_cluster = YarnCluster(opts)
return c_yarn_cluster.create_bare_cluster()