def update(self, old_sfa_record, new_sfa_record, hrn, new_key):
type = new_sfa_record['type']
# new_key implemented for users only
if new_key and type not in ['user']:
raise UnknownSfaType(type)
elif type == "slice":
# can update project manager and description
name = hrn_to_os_slicename(hrn)
researchers = sfa_record.get('researchers', [])
pis = sfa_record.get('pis', [])
project_manager = None
description = sfa_record.get('description', None)
if pis:
project_manager = Xrn(pis[0], 'user').get_leaf()
elif researchers:
project_manager = Xrn(researchers[0], 'user').get_leaf()
self.shell.auth_manager.modify_project(name, project_manager,
description)
elif type == "user":
# can techinally update access_key and secret_key,
# but that is not in our scope, so we do nothing.
pass
return True
def register_slice(self, sfa_record, hrn):
# add slice description, name, researchers, PI
name = hrn_to_os_tenant_name(hrn)
description = sfa_record.get('description', None)
self.shell.auth_manager.tenants.create(name, description)
tenant = self.shell.auth_manager.tenants.find(name=name)
auth_hrn = OSXrn(xrn=hrn, type='slice').get_authority_hrn()
parent_tenant_name = OSXrn(xrn=auth_hrn,
type='slice').get_tenant_name()
parent_tenant = self.shell.auth_manager.tenants.find(
name=parent_tenant_name)
researchers = sfa_record.get('researchers', [])
for researcher in researchers:
name = Xrn(researcher).get_leaf()
user = self.shell.auth_manager.users.find(name=name)
self.shell.auth_manager.roles.add_user_role(user, 'Member', tenant)
self.shell.auth_manager.roles.add_user_role(user, 'user', tenant)
pis = sfa_record.get('pis', [])
for pi in pis:
name = Xrn(pi).get_leaf()
user = self.shell.auth_manager.users.find(name=name)
self.shell.auth_manager.roles.add_user_role(user, 'pi', tenant)
self.shell.auth_manager.roles.add_user_role(
user, 'pi', parent_tenant)
return tenant
def test_void(self):
void = Xrn(xrn='', type=None)
expected = 'urn:publicid:IDN++'
self.assertEqual(void.get_hrn(), '')
self.assertEqual(void.get_type(), None)
self.assertEqual(void.get_urn(), expected)
loop = Xrn(xrn=expected)
self.assertEqual(loop.get_hrn(), '')
# xxx - this is not quite right as the first object has type None
self.assertEqual(loop.get_type(), '')
def __hrn(self, h, t, exp_urn):
if verbose: print 'testing (', h, t, ') expecting', exp_urn
xrn = Xrn(h, type=t)
if verbose: print xrn.dump_string()
urn = xrn.get_urn()
(h1, t1) = Xrn(urn).get_hrn_type()
if h1 != h or t1 != t or urn != exp_urn:
print "hrn->urn->hrn : MISMATCH with in=(%s,%s) -- out=(%s,%s) -- urn=%s" % (
h, t, h1, t1, urn)
self.assertEqual(h1, h)
self.assertEqual(t1, t)
self.assertEqual(urn, exp_urn)
def check_sliver_credentials(self, creds, urns):
# build list of cred object hrns
slice_cred_names = []
for cred in creds:
slice_cred_hrn = Credential(cred=cred).get_gid_object().get_hrn()
slice_cred_names.append(
DummyXrn(xrn=slice_cred_hrn).dummy_slicename())
# look up slice name of slivers listed in urns arg
slice_ids = []
for urn in urns:
sliver_id_parts = Xrn(xrn=urn).get_sliver_id_parts()
try:
slice_ids.append(int(sliver_id_parts[0]))
except ValueError:
pass
if not slice_ids:
raise Forbidden("sliver urn not provided")
slices = self.shell.GetSlices({'slice_ids': slice_ids})
sliver_names = [slice['slice_name'] for slice in slices]
# make sure we have a credential for every specified sliver ierd
for sliver_name in sliver_names:
if sliver_name not in slice_cred_names:
msg = "Valid credential not found for target: %s" % sliver_name
raise Forbidden(msg)
def allocate(self, urn, rspec_string, expiration, options={}):
xrn = Xrn(urn)
aggregate = DummyAggregate(self)
slices = DummySlices(self)
slice_record = None
users = options.get('geni_users', [])
if users:
slice_record = users[0].get('slice_record', {})
# parse rspec
rspec = RSpec(rspec_string)
requested_attributes = rspec.version.get_slice_attributes()
# ensure slice record exists
slice = slices.verify_slice(xrn.hrn,
slice_record,
expiration=expiration,
options=options)
# ensure person records exists
#persons = slices.verify_persons(xrn.hrn, slice, users, peer, sfa_peer, options=options)
# add/remove slice from nodes
request_nodes = rspec.version.get_nodes_with_slivers()
nodes = slices.verify_slice_nodes(urn, slice, request_nodes)
return aggregate.describe([xrn.get_urn()], version=rspec.version)
def describe(self, urns, version=None, options={}):
# update nova connection
tenant_name = OSXrn(xrn=urns[0], type='slice').get_tenant_name()
self.driver.shell.nova_manager.connect(tenant=tenant_name)
instances = self.get_instances(urns)
# lookup the sliver allocations
sliver_ids = [sliver['sliver_id'] for sliver in slivers]
constraint = SliverAllocation.sliver_id.in_(sliver_ids)
sliver_allocations = self.driver.api.dbsession().query(SliverAllocation).filter(constraint)
sliver_allocation_dict = {}
for sliver_allocation in sliver_allocations:
sliver_allocation_dict[sliver_allocation.sliver_id] = sliver_allocation
geni_slivers = []
rspec_nodes = []
for instance in instances:
rspec_nodes.append(self.instance_to_rspec_node(instance))
geni_sliver = self.instance_to_geni_sliver(instance, sliver_sllocation_dict)
geni_slivers.append(geni_sliver)
version_manager = VersionManager()
version = version_manager.get_version(version)
rspec_version = version_manager._get_version(version.type, version.version, 'manifest')
rspec = RSpec(version=rspec_version, user_options=options)
rspec.xml.set('expires', datetime_to_string(utcparse(time.time())))
rspec.version.add_nodes(rspec_nodes)
result = {'geni_urn': Xrn(urns[0]).get_urn(),
'geni_rspec': rspec.toxml(),
'geni_slivers': geni_slivers}
return result
def create_instance_key(self, slice_hrn, user):
slice_name = Xrn(slice_hrn).leaf
user_name = Xrn(user['urn']).leaf
key_name = "%s_%s" % (slice_name, user_name)
pubkey = user['keys'][0]
key_found = False
existing_keys = self.driver.shell.nova_manager.keypairs.findall(name=key_name)
for existing_key in existing_keys:
if existing_key.public_key != pubkey:
self.driver.shell.nova_manager.keypairs.delete(existing_key)
elif existing_key.public_key == pubkey:
key_found = True
if not key_found:
self.driver.shell.nova_manager.keypairs.create(key_name, pubkey)
return key_name
def get_aggregate_nodes(self):
zones = self.get_availability_zones()
# available sliver/instance/vm types
instances = self.driver.shell.nova_manager.flavors.list()
if isinstance(instances, dict):
instances = instances.values()
# available images
images = self.driver.shell.image_manager.get_images_detailed()
disk_images = [image_to_rspec_disk_image(img) for img in images if img['container_format'] in ['ami', 'ovf']]
rspec_nodes = []
for zone in zones:
rspec_node = Node()
xrn = OSXrn(zone, type='node')
rspec_node['component_id'] = xrn.urn
rspec_node['component_name'] = xrn.name
rspec_node['component_manager_id'] = Xrn(self.driver.hrn, 'authority+cm').get_urn()
rspec_node['exclusive'] = 'false'
rspec_node['hardware_types'] = [HardwareType({'name': 'plos-pc'}),
HardwareType({'name': 'pc'})]
slivers = []
for instance in instances:
sliver = self.instance_to_sliver(instance)
sliver['disk_image'] = disk_images
slivers.append(sliver)
rspec_node['available'] = 'true'
rspec_node['slivers'] = slivers
rspec_nodes.append(rspec_node)
return rspec_nodes
def check_sliver_credentials(self, creds, urns):
# build list of cred object hrns
slice_cred_names = []
for cred in creds:
slice_cred_hrn = Credential(cred=cred).get_gid_object().get_hrn()
slice_cred_names.append(OSXrn(xrn=slice_cred_hrn).get_slicename())
# look up slice name of slivers listed in urns arg
slice_ids = []
for urn in urns:
sliver_id_parts = Xrn(xrn=urn).get_sliver_id_parts()
slice_ids.append(sliver_id_parts[0])
if not slice_ids:
raise Forbidden("sliver urn not provided")
sliver_names = []
for slice_id in slice_ids:
slice = self.shell.auth_manager.tenants.find(slice_id)
sliver_names.append(slice['name'])
# make sure we have a credential for every specified sliver ierd
for sliver_name in sliver_names:
if sliver_name not in slice_cred_names:
msg = "Valid credential not found for target: %s" % sliver_name
raise Forbidden(msg)
def urn_to_properties(value):
# urn:publicid:IDN+omf:paris.fit-nitos.fr+node+node38
# node38.paris.fit-nitos.fr
#import pdb
#pdb.set_trace()
urn = value['component_id']
hrn = Xrn(urn).get_hrn()
t_urn = urn.split('+')
authority = t_urn[1]
if ':' in authority:
t_authority = authority.split(':')
authority = t_authority[1]
host = t_urn[-1]
hostname = host + '.' + authority
if value['type'] == 'node' and 'available.now' in value:
if value['available.now'] == 'true':
boot = "boot"
else:
boot = "disabled"
return {
'exclusive': True,
'hostname': hostname,
'hrn': hrn,
'urn': urn,
'boot_state': boot
}
else:
return {
'exclusive': True,
'hostname': hostname,
'hrn': hrn,
'urn': urn
}
def delete(self, urns, options={}):
# collect sliver ids so we can update sliver allocation states after
# we remove the slivers.
aggregate = OSAggregate(self)
instances = aggregate.get_instances(urns)
sliver_ids = []
for instance in instances:
sliver_hrn = "%s.%s" % (self.driver.hrn, instance.id)
sliver_ids.append(Xrn(sliver_hrn, type='sliver').urn)
# delete the instance
aggregate.delete_instance(instance)
# delete sliver allocation states
dbsession = self.api.dbsession()
SliverAllocation.delete_allocations(sliver_ids, dbsession)
# return geni_slivers
geni_slivers = []
for sliver_id in sliver_ids:
geni_slivers.append({
'geni_sliver_urn': sliver['sliver_id'],
'geni_allocation_status': 'geni_unallocated',
'geni_expires': None
})
return geni_slivers
def register_user(self, sfa_record, hrn, pub_key):
# add person roles, projects and keys
email = sfa_record.get('email', None)
xrn = Xrn(hrn)
name = xrn.get_leaf()
auth_hrn = xrn.get_authority_hrn()
tenant_name = OSXrn(xrn=auth_hrn, type='authority').get_tenant_name()
tenant = self.shell.auth_manager.tenants.find(name=tenant_name)
self.shell.auth_manager.users.create(name,
email=email,
tenant_id=tenant.id)
user = self.shell.auth_manager.users.find(name=name)
slices = sfa_records.get('slices', [])
for slice in projects:
slice_tenant_name = OSXrn(xrn=slice,
type='slice').get_tenant_name()
slice_tenant = self.shell.auth_manager.tenants.find(
name=slice_tenant_name)
self.shell.auth_manager.roles.add_user_role(
user, slice_tenant, 'user')
keys = sfa_records.get('keys', [])
for key in keys:
keyname = OSXrn(xrn=hrn, type='user').get_slicename()
self.shell.nova_client.keypairs.create(keyname, key)
return user
def allocate(self, urn, rspec_string, expiration, options={}):
xrn = Xrn(urn)
aggregate = OSAggregate(self)
# assume first user is the caller and use their context
# for the ec2/euca api connection. Also, use the first users
# key as the project key.
key_name = None
if len(users) > 1:
key_name = aggregate.create_instance_key(xrn.get_hrn(), users[0])
# collect public keys
users = options.get('geni_users', [])
pubkeys = []
for user in users:
pubkeys.extend(user['keys'])
rspec = RSpec(rspec_string)
instance_name = hrn_to_os_slicename(slice_hrn)
tenant_name = OSXrn(xrn=slice_hrn, type='slice').get_tenant_name()
slivers = aggregate.run_instances(instance_name, tenant_name, \
rspec_string, key_name, pubkeys)
# update all sliver allocation states setting then to geni_allocated
sliver_ids = [sliver.id for sliver in slivers]
dbsession = self.api.dbsession()
SliverAllocation.set_allocations(sliver_ids, 'geni_provisioned',
dbsession)
return aggregate.describe(urns=[urn], version=rspec.version)
def instance_to_geni_sliver(self, instance, sliver_allocations = {}):
sliver_hrn = '%s.%s' % (self.driver.hrn, instance.id)
sliver_id = Xrn(sliver_hrn, type='sliver').urn
# set sliver allocation and operational status
sliver_allocation = sliver_allocations[sliver_id]
if sliver_allocation:
allocation_status = sliver_allocation.allocation_state
if allocation_status == 'geni_allocated':
op_status = 'geni_pending_allocation'
elif allocation_status == 'geni_provisioned':
state = instance.state.lower()
if state == 'active':
op_status = 'geni_ready'
elif state == 'building':
op_status = 'geni_notready'
elif state == 'failed':
op_status =' geni_failed'
else:
op_status = 'geni_unknown'
else:
allocation_status = 'geni_unallocated'
# required fields
geni_sliver = {'geni_sliver_urn': sliver_id,
'geni_expires': None,
'geni_allocation_status': allocation_status,
'geni_operational_status': op_status,
'geni_error': None,
'plos_created_at': datetime_to_string(utcparse(instance.created)),
'plos_sliver_type': self.shell.nova_manager.flavors.find(id=instance.flavor['id']).name,
}
return geni_sliver
def GetVersion(self, api, options):
# peers explicitly in aggregates.xml
peers =dict ([ (peername,interface.get_url()) for (peername,interface) in api.aggregates.iteritems()
if peername != api.hrn])
version_manager = VersionManager()
ad_rspec_versions = []
request_rspec_versions = []
cred_types = [{'geni_type': 'geni_sfa', 'geni_version': str(i)} for i in range(4)[-2:]]
for rspec_version in version_manager.versions:
if rspec_version.content_type in ['*', 'ad']:
ad_rspec_versions.append(rspec_version.to_dict())
if rspec_version.content_type in ['*', 'request']:
request_rspec_versions.append(rspec_version.to_dict())
xrn=Xrn(api.hrn, 'authority+sm')
version_more = {
'interface':'slicemgr',
'sfa': 2,
'geni_api': 3,
'geni_api_versions': {'3': 'http://%s:%s' % (api.config.SFA_SM_HOST, api.config.SFA_SM_PORT)},
'hrn' : xrn.get_hrn(),
'urn' : xrn.get_urn(),
'peers': peers,
'geni_single_allocation': 0, # Accept operations that act on as subset of slivers in a given state.
'geni_allocate': 'geni_many',# Multiple slivers can exist and be incrementally added, including those which connect or overlap in some way.
'geni_credential_types': cred_types,
}
sm_version=version_core(version_more)
# local aggregate if present needs to have localhost resolved
if api.hrn in api.aggregates:
local_am_url=api.aggregates[api.hrn].get_url()
sm_version['peers'][api.hrn]=local_am_url.replace('localhost',sm_version['hostname'])
return sm_version
def _process_node(cls, node):
node['type'] = 'node'
node['network_hrn'] = Xrn(node['component_id']).authority[0] # network ? XXX
node['hrn'] = urn_to_hrn(node['component_id'])[0]
node['urn'] = node['component_id']
node['hostname'] = node['component_name']
node['initscripts'] = node.pop('pl_initscripts')
# All Labora nodes are exclusive = true
node['exclusive'] = 'true'
if 'granularity' in node:
node['granularity'] = node['granularity']['grain']
# XXX This should use a MAP as before
if 'position' in node: # iotlab
node['x'] = node['position']['posx']
node['y'] = node['position']['posy']
node['z'] = node['position']['posz']
del node['position']
if 'location' in node:
if node['location']:
node['latitude'] = node['location']['latitude']
node['longitude'] = node['location']['longitude']
del node['location']
# Flatten tags
if 'tags' in node:
if node['tags']:
for tag in node['tags']:
node[tag['tagname']] = tag['value']
del node['tags']
return node
def sliver_to_slice_xrn(self, xrn):
sliver_id_parts = Xrn(xrn).get_sliver_id_parts()
slice = self.shell.auth_manager.tenants.find(id=sliver_id_parts[0])
if not slice:
raise Forbidden("Unable to locate slice record for sliver: %s" %
xrn)
slice_xrn = OSXrn(name=slice.name, type='slice')
return slice_xrn
def provision(self, xrn):
"""Provision slivers"""
xrn = Xrn(xrn, 'slice')
slice_urn = xrn.get_urn()
options = {}
manifest = self.api.manager.provision(self.api, [slice_urn], [],
options)
print manifest
def allocate(self, xrn, rspec):
"""Allocate slivers"""
xrn = Xrn(xrn, 'slice')
slice_urn = xrn.get_urn()
rspec_string = open(rspec).read()
options = {}
manifest = self.api.manager.Allocate(self.api, slice_urn, [],
rspec_string, options)
print manifest
def xrn_to_hostname(xrn):
"""Returns a node's hostname from its xrn.
:param xrn: The nodes xrn identifier.
:type xrn: Xrn (from sfa.util.xrn)
:returns: node's hostname.
:rtype: string
"""
return Xrn.unescape(Xrn(xrn=xrn, type='node').get_leaf())
def GetVersion(self, api, options):
peers = dict ( [ (hrn,interface.get_url()) for (hrn,interface) in api.registries.iteritems()
if hrn != api.hrn])
xrn=Xrn(api.hrn)
return version_core({'interface':'registry',
'sfa': 2,
'geni_api': 2,
'hrn':xrn.get_hrn(),
'urn':xrn.get_urn(),
'peers':peers})
def type_of_urn(urn):
"""
Returns the type of the given urn (node, slice, sliver)
:param urn: URN to discover the type
:type string
:returns type of the URN (node, slice, sliver, user, authority)
"""
return Xrn(urn).get_type()
def remove(self, sfa_record):
type = sfa_record['type']
if type == 'user':
name = Xrn(sfa_record['hrn']).get_leaf()
if self.shell.auth_manager.get_user(name):
self.shell.auth_manager.delete_user(name)
elif type == 'slice':
name = hrn_to_os_slicename(sfa_record['hrn'])
if self.shell.auth_manager.get_project(name):
self.shell.auth_manager.delete_project(name)
return True
def hrn_to_authname (hrn):
"""
Gets the authority name from an HRN
:param hrn: HRN whose authority is got
:type string
:returns: authority name name.
:rtype: string
"""
return Xrn(xrn=hrn).get_authority_hrn()
def CreateGid(self, api, xrn, cert):
# get the authority
authority = Xrn(xrn=xrn).get_authority_hrn()
auth_info = api.auth.get_auth_info(authority)
if not cert:
pkey = Keypair(create=True)
else:
certificate = Certificate(string=cert)
pkey = certificate.get_pubkey()
gid = api.auth.hierarchy.create_gid(xrn, create_uuid(), pkey)
return gid.save_to_string(save_parents=True)
def _record_dict(self,
xrn=None,
type=None,
url=None,
description=None,
email='',
key=None,
slices=[],
researchers=[],
pis=[],
extras={}):
record_dict = {}
if xrn:
if type:
xrn = Xrn(xrn, type)
else:
xrn = Xrn(xrn)
record_dict['urn'] = xrn.get_urn()
record_dict['hrn'] = xrn.get_hrn()
record_dict['type'] = xrn.get_type()
if url:
record_dict['url'] = url
if description:
record_dict['description'] = description
if key:
try:
pubkey = open(key, 'r').read()
except IOError:
pubkey = key
record_dict['keys'] = [pubkey]
if slices:
record_dict['slices'] = slices
if researchers:
record_dict['researcher'] = researchers
if email:
record_dict['email'] = email
if pis:
record_dict['pi'] = pis
if extras:
record_dict.update(extras)
return record_dict
def instance_to_sliver(self, instance, xrn=None):
if xrn:
sliver_hrn = '%s.%s' % (self.driver.hrn, instance.id)
sliver_id = Xrn(sliver_hrn, type='sliver').urn
sliver = Sliver({'sliver_id': sliver_id,
'name': instance.name,
'type': instance.name,
'cpus': str(instance.vcpus),
'memory': str(instance.ram),
'storage': str(instance.disk)})
return sliver
def add_leases(xml, leases, channels):
network_elems = xml.xpath('//network')
if len(network_elems) > 0:
network_elem = network_elems[0]
elif len(leases) > 0:
network_urn = Xrn(
leases[0]['component_id']).get_authority_urn().split(':')[0]
network_elem = xml.add_element('network', name=network_urn)
else:
network_elem = xml
# group the leases by slice and timeslots
grouped_leases = []
while leases:
slice_id = leases[0]['slice_id']
start_time = leases[0]['start_time']
duration = leases[0]['duration']
group = []
for lease in leases:
if slice_id == lease['slice_id'] and start_time == lease[
'start_time'] and duration == lease['duration']:
group.append(lease)
grouped_leases.append(group)
for lease1 in group:
leases.remove(lease1)
lease_elems = []
for lease in grouped_leases:
#lease[0]['start_time'] = datetime_to_string(utcparse(lease[0]['start_time']))
lease_fields = ['slice_id', 'start_time', 'duration']
lease_elem = network_elem.add_instance('lease', lease[0],
lease_fields)
lease_elems.append(lease_elem)
# add nodes of this lease
for node in lease:
lease_elem.add_instance('node', node, ['component_id'])
# add reserved channels of this lease
#channels = [{'channel_id': 1}, {'channel_id': 2}]
for channel in channels:
#channel['start_time'] = datetime_to_string(utcparse(channel['start_time']))
if channel['slice_id'] == lease[0]['slice_id'] and channel[
'start_time'] == lease[0]['start_time'] and channel[
'duration'] == lease[0]['duration']:
lease_elem.add_instance('channel', channel,
['component_id'])
def verify_slice_nodes(self, slice_urn, slice, rspec_nodes):
slivers = {}
for node in rspec_nodes:
hostname = node.get('component_name')
client_id = node.get('client_id')
component_id = node.get('component_id').strip()
if hostname:
hostname = hostname.strip()
elif component_id:
hostname = xrn_to_hostname(component_id)
if hostname:
slivers[hostname] = {
'client_id': client_id,
'component_id': component_id
}
nodes = self.driver.shell.GetNodes(
slice['node_ids'], ['node_id', 'hostname', 'interface_ids'])
current_slivers = [node['hostname'] for node in nodes]
# remove nodes not in rspec
deleted_nodes = list(set(current_slivers).difference(slivers.keys()))
# add nodes from rspec
added_nodes = list(set(slivers.keys()).difference(current_slivers))
try:
self.driver.shell.AddSliceToNodes(slice['name'], added_nodes)
self.driver.shell.DeleteSliceFromNodes(slice['name'],
deleted_nodes)
except:
logger.log_exc('Failed to add/remove slice from nodes')
slices = self.driver.shell.GetSlices(slice['name'], ['node_ids'])
resulting_nodes = self.driver.shell.GetNodes(slices[0]['node_ids'])
# update sliver allocations
for node in resulting_nodes:
client_id = slivers[node['hostname']]['client_id']
component_id = slivers[node['hostname']]['component_id']
sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'],
node['node_id'])
sliver_id = Xrn(sliver_hrn, type='sliver').urn
record = SliverAllocation(sliver_id=sliver_id,
client_id=client_id,
component_id=component_id,
slice_urn=slice_urn,
allocation_state='geni_allocated')
record.sync(self.driver.api.dbsession())
return resulting_nodes
