def login(version=1):
req=api.TrinityAPI(request)
if req.has_access:
response.status=200
return {'token': req.token}
else:
response.status=401
return
def show_node(node,version=1):
req=api.TrinityAPI(request)
# We do an authentication here because the object method is
# need by non-admin calls too
req.authenticate()
if req.is_admin:
return req.node_info(node)
else:
return {'error':req.not_admin}
def show_cluster(cluster,version=1):
req=api.TrinityAPI(request)
if not (req.is_admin or req.tenant==cluster):
ret={'error':req.no_access}
return ret
hc_overview=req.detailed_overview()
c_overview={'hardware':{}}
for hardware in hc_overview['hardware']:
amount=len(set(hc_overview['cluster'][cluster]).intersection(set(hc_overview['hardware'][hardware])))
c_overview['hardware'][hardware]=amount
return c_overview
def cluster_overview(version=1):
req=api.TrinityAPI(request)
if not req.is_admin:
ret={'error':req.not_admin}
return ret
hc_overview=req.detailed_overview()
c_overview={}
for cluster in hc_overview['cluster']:
c_overview[cluster]={}
c_overview[cluster]['hardware']={}
for hardware in hc_overview['hardware']:
amount=len(set(hc_overview['cluster'][cluster]).intersection(set(hc_overview['hardware'][hardware])))
c_overview[cluster]['hardware'][hardware]=amount
return c_overview
def show_hardware(hardware,version=1):
req=api.TrinityAPI(request)
if not req.is_admin:
ret={'error':req.not_admin}
return ret
hc_overview=req.detailed_overview()
c_nodes=set()
for cluster in hc_overview['cluster']:
c_nodes=c_nodes.union(set(hc_overview['cluster'][cluster]))
h_overview={}
h_overview={}
h_overview['total']=len(hc_overview['hardware'][hardware])
h_overview['list_unallocated']=list(set(hc_overview['hardware'][hardware])-c_nodes)
h_overview['unallocated']=len(h_overview['list_unallocated'])
h_overview['allocated']=h_overview['total']-h_overview['unallocated']
return h_overview
def modify_cluster(self, cluster, version=1):
global network_map
api.state_has_changed = True
req = api.TrinityAPI(request)
ret = {}
ret['statusOK'] = True
clusters = req.detailed_overview()['cluster'].keys()
if cluster in clusters:
cluster_exists = True
ret = update_cluster(req, cluster)
slurm_needs_update = False
if ret['statusOK']:
if ret['change']:
slurm_needs_update = True
else:
cluster_exists = False
ret = create_cluster(req, cluster)
if ret['statusOK']:
# Create the cluster home directories
vc_cluster = req.vc + cluster
cluster_home = os.path.join(req.cluster_path, vc_cluster)
if not os.path.isdir(cluster_home):
os.makedirs(cluster_home)
src_root = req.template_dir
vc_cluster = req.vc + cluster
dest_root = os.path.join(req.cluster_path, vc_cluster)
excludes = []
copy_with_excludes(src_root, dest_root, excludes)
# create munge user on the physical node it does not exist
# then create a munge key
subprocess.call('! id munge && useradd -u 1002 -U munge',
shell=True)
munge_dir_path = os.path.join(req.cluster_path, vc_cluster,
req.munge_key_dir)
munge_key_path = os.path.join(munge_dir_path,
req.munge_key_file)
if os.path.isfile(munge_key_path):
os.remove(munge_key_path)
if not os.path.isdir(munge_dir_path):
os.makedirs(munge_dir_path)
subprocess.call('dd if=/dev/urandom bs=1 count=1024 > ' +
munge_key_path,
shell=True)
subprocess.call('chown munge:munge ' + munge_key_path,
shell=True)
subprocess.call('chmod u=r,go= ' + munge_key_path, shell=True)
subprocess.call('chown munge:munge ' + munge_dir_path,
shell=True)
subprocess.call('chmod u=rwx,go= ' + munge_dir_path,
shell=True)
subprocess.call('chmod u=rwx,go=rx ' + req.cluster_path + '/' +
vc_cluster + '/etc/slurm',
shell=True)
subprocess.call('chmod u=rw,go=rx ' + req.cluster_path + '/' +
vc_cluster + '/etc/slurm/slurm.conf',
shell=True)
subprocess.call('chmod u=rw,go=r ' + req.cluster_path + '/' +
vc_cluster + '/etc/slurm/slurm-nodes.conf',
shell=True)
subprocess.call('chmod ug=rw,o=r ' + req.cluster_path + '/' +
vc_cluster + '/etc/slurm/slurm-user.conf',
shell=True)
slurm_needs_update = True
apps = os.path.join(req.cluster_path, vc_cluster, 'apps')
modulefiles = os.path.join(req.cluster_path, vc_cluster,
'modulefiles')
if not os.path.isdir(apps):
os.makedirs(apps)
if not os.path.isdir(modulefiles):
os.makedirs(modulefiles)
if slurm_needs_update:
vc_cluster = req.vc + cluster
slurm = os.path.join(req.cluster_path, vc_cluster,
req.slurm_node_file)
@retry(stop_max_delay=10000, wait_fixed=1000)
def try_write():
fop = open(slurm, 'w')
nodes = sorted(ret['nodeList'])
for cont in nodes:
node_name = cont
cpu_count = api.all_nodes_info[cont]['cpucount']
slurm_string = 'NodeName=' + node_name + ' CPUS=' + cpu_count + ' State=UNKNOWN'
fop.write(slurm_string + '\n')
cont_string = ','.join(nodes)
part_string = 'PartitionName=' + req.cont_part + ' Nodes=' + cont_string + ' Default=Yes MaxTime=INFINITE State=UP'
fop.write(part_string + '\n')
fop.close()
try_write()
subprocess.call('echo slurm-nodes.conf was written', shell=True)
##---------------------------------------------------------------------
## In this part we update makehosts, makedns etc for the cluster
##---------------------------------------------------------------------
vc_cluster = req.vc + cluster
vc_net = 'vc_' + cluster + '_net'
login_cluster = 'login-' + cluster
# Get the network info
path = "/tables/networks/rows"
xcat_networks = requests.get(xcat_host + path,
verify=False,
params=req.query,
headers=req.headers).json()["networks"]
network_map = {}
for network in xcat_networks:
if "domain" in network and network["domain"].startswith(req.vc):
network_map.update(
{network["domain"]: network["net"].split(".")[1]})
if vc_cluster in network_map.keys():
second_octet = network_map[vc_cluster]
else:
for second_octet_int in range(16, 32):
second_octet = str(second_octet_int)
if second_octet not in network_map.values():
break
if not cluster_exists:
# create vc-<cluster> entry in the hosts table
# The verb is PUT because the nodes already exist
verb = 'PUT'
path = '/tables/hosts/rows/node=' + vc_cluster
payload = {
"ip": "|\D+(\d+)$|172." + second_octet +
".((($1-1)/255)).(($1-1)%255+1)|",
"hostnames": "|\D+(\d+)$|c($1)|"
}
req.xcat(verb=verb, path=path, payload=payload)
verb = 'PUT'
# create login-<cluster> entry in the hosts table
path = '/tables/hosts/rows/node=' + login_cluster
payload = {
"ip": "172." + second_octet + ".255.254",
"hostnames": "login." + vc_cluster
}
req.xcat(verb=verb, path=path, payload=payload)
# create the entry in the networks table
verb = 'POST'
path = '/networks/' + vc_net
payload = {
"domain": vc_cluster,
"gateway": "<xcatmaster>",
"mask": "255.255.0.0",
"mgtifname": req.xcat_mgtifname,
"net": "172." + second_octet + ".0.0"
}
req.xcat(verb=verb, path=path, payload=payload)
# create a login node entry in the xCATdb
verb = 'POST'
path = '/nodes/' + login_cluster
payload = {"groups": "login"}
req.xcat(verb=verb, path=path, payload=payload)
# makehost and makedns for login node
verb = 'POST'
payload = {}
path = '/nodes/' + login_cluster + '/host'
req.xcat(verb=verb, path=path, payload=payload)
path = '/nodes/' + login_cluster + '/dns'
req.xcat(verb=verb, path=path, payload=payload)
# makehost and makedns for cluster
cont_subs_list = []
node_subs_list = []
if 'error' not in ret:
for cont in ret["subsList"]:
cont_subs_list.append(cont)
subtracted_node = cont.replace(req.cont_pref, req.node_pref, 1)
node_subs_list.append(subtracted_node)
cont_subs_string = ",".join(cont_subs_list)
node_subs_string = ",".join(node_subs_list)
if cont_subs_list:
verb = 'DELETE'
payload = {}
path = '/nodes/' + cont_subs_string + '/dns'
r = requests.delete(req.xcat_host + path,
verify=False,
params=req.query)
verb = 'POST'
payload = {
"command": ["docker stop trinity; docker rm trinity"]
}
path = '/nodes/' + node_subs_string + '/nodeshell'
req.xcat(verb=verb, path=path, payload=payload)
cont_adds_list = []
node_adds_list = []
if 'error' not in ret:
for cont in ret["addsList"]:
cont_adds_list.append(cont)
added_node = cont.replace(req.cont_pref, req.node_pref, 1)
node_adds_list.append(added_node)
cont_adds_string = ",".join(cont_adds_list)
node_adds_string = ",".join(node_adds_list)
if cont_adds_list:
verb = 'POST'
payload = {}
path = '/nodes/' + cont_adds_string + '/host'
req.xcat(verb=verb, path=path, payload=payload)
path = '/nodes/' + cont_adds_string + '/dns'
req.xcat(verb=verb, path=path, payload=payload)
verb = 'POST'
payload = {
"command": [
"docker stop trinity; docker rm trinity; service trinity restart"
]
}
path = '/nodes/' + node_adds_string + '/nodeshell'
req.xcat(verb=verb, path=path, payload=payload)
api.state_has_changed = True
#----------------------------------------------------------------------
# Now create the login node
#----------------------------------------------------------------------
login_ip = "172." + second_octet + ".255.254"
nova = novaclient.Client('2',
keystone_admin_user,
keystone_admin_pass,
'admin',
req.keystone_host,
connection_pool=True)
if not nova.servers.list(search_opts={
'all_tenants': 1,
'name': 'login-' + cluster
}):
login_pool = login_cluster
path = req.nova_host + '/' + req.tenant_id + '/os-floating-ips-bulk'
headers = {"X-Auth-Project-Id": "admin", "X-Auth-Token": req.token}
headers.update(req.headers)
payload = {
"floating_ips_bulk_create": {
"ip_range": "172." + second_octet + ".255.254",
"pool": login_pool
}
}
r = requests.post(path, headers=headers, data=json.dumps(payload))
path = req.keystone_admin + '/OS-KSADM/roles'
headers = {"X-Auth-Token": req.token}
headers.update(req.headers)
r = requests.get(path, headers=headers)
for role in r.json()["roles"]:
if role["name"] == "_member_":
role_id = role["id"]
break
path = req.keystone_admin + '/users'
headers = {"X-Auth-Token": req.token}
headers.update(req.headers)
r = requests.get(path, headers=headers)
for user in r.json()["users"]:
if user["username"] == "trinity":
user_id = user["id"]
break
path = req.keystone_admin + '/tenants'
headers = {"X-Auth-Token": req.token}
headers.update(req.headers)
r = requests.get(path, headers=headers)
for tenant in r.json()["tenants"]:
if tenant["name"] == cluster:
tenant_id = tenant["id"]
break
path = req.keystone_admin + '/tenants/' + tenant_id + '/users/' + user_id + '/roles/OS-KSADM/' + role_id
headers = {"X-Auth-Token": req.token}
headers.update(req.headers)
r = requests.put(path, headers=headers)
path = req.keystone_host + '/tokens'
headers = req.headers
payload = {
"auth": {
"tenantName": cluster,
"passwordCredentials": {
"username": '******',
"password": '******'
}
}
}
r = requests.post(path, data=json.dumps(payload), headers=headers)
tenant_token = r.json()["access"]["token"]["id"]
path = req.nova_host + '/' + tenant_id + '/os-floating-ips'
headers = {
"X-Auth-Project-Id": cluster,
"X-Auth-Token": tenant_token
}
headers.update(req.headers)
payload = {"pool": login_pool}
r = requests.post(path, data=json.dumps(payload), headers=headers)
path = req.nova_host + '/' + tenant_id + '/os-security-groups'
headers = {
"X-Auth-Project-Id": cluster,
"X-Auth-Token": tenant_token
}
headers.update(req.headers)
r = requests.get(path, headers=headers)
for security_group in r.json()["security_groups"]:
if security_group["name"] == "default":
default_id = security_group["id"]
break
path = req.nova_host + '/' + tenant_id + '/os-security-group-rules'
headers = {
"X-Auth-Project-Id": cluster,
"X-Auth-Token": tenant_token
}
headers.update(req.headers)
payload = {
"security_group_rule": {
"ip_protocol": "tcp",
"parent_group_id": default_id,
"from_port": 1,
"to_port": 65535,
"cidr": "0.0.0.0/0",
"group_id": None
}
}
r = requests.post(path, data=json.dumps(payload), headers=headers)
payload = {
"security_group_rule": {
"ip_protocol": "icmp",
"parent_group_id": default_id,
"from_port": -1,
"to_port": -1,
"cidr": "0.0.0.0/0",
"group_id": None
}
}
r = requests.post(path, data=json.dumps(payload), headers=headers)
path = req.nova_host + '/' + tenant_id + '/images'
headers = {
"X-Auth-Project-Id": cluster,
"X-Auth-Token": tenant_token
}
headers.update(req.headers)
r = requests.get(path, headers=headers)
for image in r.json()["images"]:
if image["name"] == "login":
image_id = image["id"]
# For now assume that we are using flavor = 2 (small)
fop = open(req.login_conf)
login_data = fop.read()
fop.close()
replacements = {
"vc-a": vc_cluster,
"UTC": tzlocal.get_localzone().zone or 'UTC'
}
for i, j in replacements.iteritems():
print i, j
login_data = login_data.replace(i, j)
login_data_encoded = base64.b64encode(login_data)
path = req.nova_host + '/' + tenant_id + '/servers'
headers = {
"X-Auth-Project-Id": cluster,
"X-Auth-Token": tenant_token
}
headers.update(req.headers)
payload = {
"server": {
"name": login_cluster,
"imageRef": image_id,
"flavorRef": "2",
"user_data": login_data_encoded,
"security_groups": [{
"name": "default"
}],
"max_count": 1,
"min_count": 1
}
}
r = requests.post(path, data=json.dumps(payload), headers=headers)
instance_id = r.json()["server"]["id"]
# This is added to add a small delay between creating the instance
# and associating a floating ip, otherwise we get the following message
# "No nw_info cache associated with instance"
time.sleep(5)
path = req.nova_host + '/' + tenant_id + '/servers/' + instance_id + '/action'
headers = {
"X-Auth-Project-Id": cluster,
"X-Auth-Token": tenant_token
}
headers.update(req.headers)
payload = {
"addFloatingIp": {
"address": "172." + second_octet + ".255.254"
}
}
r = requests.post(path, data=json.dumps(payload), headers=headers)
path = req.keystone_admin + '/tenants/' + tenant_id + '/users/' + user_id + '/roles/OS-KSADM/' + role_id
headers = {"X-Auth-Token": req.token}
headers.update(req.headers)
r = requests.delete(path, headers=headers)
return ret
def list_clusters(version=1):
req=api.TrinityAPI(request)
clusters=req.detailed_overview()['cluster'].keys()
return {'statusOK': True, 'clusters': clusters}
def total_overview(version=1):
req=api.TrinityAPI(request)
if not req.is_admin:
ret={'error':req.not_admin}
return ret
return req.detailed_overview()
def show_monitoring_info(version=1): req=api.TrinityAPI(request) return req.mon_info()
def show_hardware_details(cluster,version=1): req=api.TrinityAPI(request) return req.cluster_details(cluster)
def list_nodes(version=1): req=api.TrinityAPI(request) return req.nodes()
def list_hardwares(version=1):
req=api.TrinityAPI(request)
hardwares=req.detailed_overview()['hardware'].keys()
return {'statusOK': True, 'hardwares': hardwares}
