def run (self, options):
config = Config ()
interface_hrn = config.SFA_INTERFACE_HRN
root_auth = config.SFA_REGISTRY_ROOT_AUTH
shell = NitosShell (config)
######## retrieve all existing SFA objects
all_records = global_dbsession.query(RegRecord).all()
# create hash by (type,hrn)
# we essentially use this to know if a given record is already known to SFA
self.records_by_type_hrn = \
dict ( [ ( (record.type, record.hrn) , record ) for record in all_records ] )
# create hash by (type,pointer)
self.records_by_type_pointer = \
dict ( [ ( (record.type, record.pointer) , record ) for record in all_records
if record.pointer != -1] )
# initialize record.stale to True by default, then mark stale=False on the ones that are in use
for record in all_records: record.stale=True
######## retrieve NITOS data
# Get site info
# retrieve only required stuf
site = shell.getTestbedInfo()
sites = [site]
# create a hash of sites by login_base
# # sites_by_login_base = dict ( [ ( site['login_base'], site ) for site in sites ] )
# Get all NITOS users
users = shell.getUsers()
# create a hash of users by user_id
users_by_id = dict ( [ ( user['user_id'], user) for user in users ] )
# Get all NITOS public keys
# accumulate key ids for keys retrieval
# key_ids = []
# for person in persons:
# key_ids.extend(person['key_ids'])
# keys = shell.GetKeys( {'peer_id': None, 'key_id': key_ids,
# 'key_type': 'ssh'} )
# # create a hash of keys by key_id
# keys_by_id = dict ( [ ( key['key_id'], key ) for key in keys ] )
# create a dict user_id -> [ (nitos)keys ]
keys_by_user_id = dict ( [ ( user['user_id'], user['keys']) for user in users ] )
# Get all nitos nodes
nodes = shell.getNodes({}, [])
# create hash by node_id
nodes_by_id = dict ( [ (node['node_id'], node) for node in nodes ] )
# Get all nitos slices
slices = shell.getSlices({}, [])
# create hash by slice_id
slices_by_id = dict ( [ (slice['slice_id'], slice) for slice in slices ] )
# start importing
for site in sites:
#for i in [0]:
site_hrn = _get_site_hrn(interface_hrn, site)
# import if hrn is not in list of existing hrns or if the hrn exists
# but its not a site record
site_record=self.locate_by_type_hrn ('authority', site_hrn)
if not site_record:
try:
urn = hrn_to_urn(site_hrn, 'authority')
if not self.auth_hierarchy.auth_exists(urn):
self.auth_hierarchy.create_auth(urn)
auth_info = self.auth_hierarchy.get_auth_info(urn)
site_record = RegAuthority(hrn=site_hrn, gid=auth_info.get_gid_object(),
pointer=0,
authority=get_authority(site_hrn))
site_record.just_created()
global_dbsession.add(site_record)
global_dbsession.commit()
self.logger.info("NitosImporter: imported authority (site) : %s" % site_record)
self.remember_record (site_record)
except:
# if the site import fails then there is no point in trying to import the
# site's child records (node, slices, persons), so skip them.
self.logger.log_exc("NitosImporter: failed to import site. Skipping child records")
continue
else:
# xxx update the record ...
pass
site_record.stale=False
# import node records
for node in nodes:
site_auth = get_authority(site_hrn)
site_name = site['name']
node_hrn = hostname_to_hrn(site_auth, site_name, node['hostname'])
# xxx this sounds suspicious
if len(node_hrn) > 64: node_hrn = node_hrn[:64]
node_record = self.locate_by_type_hrn ( 'node', node_hrn )
if not node_record:
try:
pkey = Keypair(create=True)
urn = hrn_to_urn(node_hrn, 'node')
node_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
node_record = RegNode (hrn=node_hrn, gid=node_gid,
pointer =node['node_id'],
authority=get_authority(node_hrn))
node_record.just_created()
global_dbsession.add(node_record)
global_dbsession.commit()
self.logger.info("NitosImporter: imported node: %s" % node_record)
self.remember_record (node_record)
except:
self.logger.log_exc("NitosImporter: failed to import node")
else:
# xxx update the record ...
pass
node_record.stale=False
# import users
for user in users:
user_hrn = username_to_hrn(interface_hrn, site['name'], user['username'])
# xxx suspicious again
if len(user_hrn) > 64: user_hrn = user_hrn[:64]
user_urn = hrn_to_urn(user_hrn, 'user')
user_record = self.locate_by_type_hrn ( 'user', user_hrn)
# return a tuple pubkey (a nitos key object) and pkey (a Keypair object)
def init_user_key (user):
pubkey = None
pkey = None
if user['keys']:
# randomly pick first key in set
for key in user['keys']:
pubkey = key
try:
pkey = convert_public_key(pubkey)
break
except:
continue
if not pkey:
self.logger.warn('NitosImporter: unable to convert public key for %s' % user_hrn)
pkey = Keypair(create=True)
else:
# the user has no keys. Creating a random keypair for the user's gid
self.logger.warn("NitosImporter: user %s does not have a NITOS public key"%user_hrn)
pkey = Keypair(create=True)
return (pubkey, pkey)
# new user
try:
if not user_record:
(pubkey,pkey) = init_user_key (user)
user_gid = self.auth_hierarchy.create_gid(user_urn, create_uuid(), pkey)
user_gid.set_email(user['email'])
user_record = RegUser (hrn=user_hrn, gid=user_gid,
pointer=user['user_id'],
authority=get_authority(user_hrn),
email=user['email'])
if pubkey:
user_record.reg_keys=[RegKey (pubkey)]
else:
self.logger.warning("No key found for user %s"%user_record)
user_record.just_created()
global_dbsession.add (user_record)
global_dbsession.commit()
self.logger.info("NitosImporter: imported user: %s" % user_record)
self.remember_record ( user_record )
else:
# update the record ?
# if user's primary key has changed then we need to update the
# users gid by forcing an update here
sfa_keys = user_record.reg_keys
def sfa_key_in_list (sfa_key,nitos_user_keys):
for nitos_key in nitos_user_keys:
if nitos_key==sfa_key: return True
return False
# are all the SFA keys known to nitos ?
new_keys=False
if not sfa_keys and user['keys']:
new_keys = True
else:
for sfa_key in sfa_keys:
if not sfa_key_in_list (sfa_key.key,user['keys']):
new_keys = True
if new_keys:
(pubkey,pkey) = init_user_key (user)
user_gid = self.auth_hierarchy.create_gid(user_urn, create_uuid(), pkey)
if not pubkey:
user_record.reg_keys=[]
else:
user_record.reg_keys=[ RegKey (pubkey)]
user_record.gid = user_gid
user_record.just_updated()
self.logger.info("NitosImporter: updated user: %s" % user_record)
user_record.email = user['email']
global_dbsession.commit()
user_record.stale=False
except:
self.logger.log_exc("NitosImporter: failed to import user %s %s"%(user['user_id'],user['email']))
# import slices
for slice in slices:
slice_hrn = slicename_to_hrn(interface_hrn, site['name'], slice['slice_name'])
slice_record = self.locate_by_type_hrn ('slice', slice_hrn)
if not slice_record:
try:
pkey = Keypair(create=True)
urn = hrn_to_urn(slice_hrn, 'slice')
slice_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
slice_record = RegSlice (hrn=slice_hrn, gid=slice_gid,
pointer=slice['slice_id'],
authority=get_authority(slice_hrn))
slice_record.just_created()
global_dbsession.add(slice_record)
global_dbsession.commit()
self.logger.info("NitosImporter: imported slice: %s" % slice_record)
self.remember_record ( slice_record )
except:
self.logger.log_exc("NitosImporter: failed to import slice")
else:
# xxx update the record ...
self.logger.warning ("Slice update not yet implemented")
pass
# record current users affiliated with the slice
slice_record.reg_researchers = \
[ self.locate_by_type_pointer ('user',int(user_id)) for user_id in slice['user_ids'] ]
global_dbsession.commit()
slice_record.stale=False
### remove stale records
# special records must be preserved
system_hrns = [interface_hrn, root_auth, interface_hrn + '.slicemanager']
for record in all_records:
if record.hrn in system_hrns:
record.stale=False
if record.peer_authority:
record.stale=False
for record in all_records:
try: stale=record.stale
except:
stale=True
self.logger.warning("stale not found with %s"%record)
if stale:
self.logger.info("NitosImporter: deleting stale record: %s" % record)
global_dbsession.delete(record)
global_dbsession.commit()
def __init__ (self, config):
Driver.__init__ (self, config)
self.shell = NitosShell (config)
self.cache=None
self.testbedInfo = self.shell.getTestbedInfo()
def __init__(self, api):
Driver.__init__(self, api)
config = api.config
self.shell = NitosShell(config)
self.cache = None
self.testbedInfo = self.shell.getTestbedInfo()
class NitosDriver (Driver):
# the cache instance is a class member so it survives across incoming requests
cache = None
def __init__ (self, config):
Driver.__init__ (self, config)
self.shell = NitosShell (config)
self.cache=None
self.testbedInfo = self.shell.getTestbedInfo()
# un-comment below lines to enable caching
# if config.SFA_AGGREGATE_CACHING:
# if NitosDriver.cache is None:
# NitosDriver.cache = Cache()
# self.cache = NitosDriver.cache
###########################################
########## utility methods for NITOS driver
###########################################
def filter_nitos_results (self, listo, filters_dict):
"""
the Nitos scheduler API does not provide a get result filtring so we do it here
"""
mylist = []
mylist.extend(listo)
for dicto in mylist:
for filter in filters_dict:
if filter not in dicto or dicto[filter] != filters_dict[filter]:
listo.remove(dicto)
break
return listo
def convert_id (self, list_of_dict):
"""
convert object id retrived in string format to int format
"""
for dicto in list_of_dict:
for key in dicto:
if key in ['node_id', 'slice_id', 'user_id', 'channel_id', 'reservation_id'] and isinstance(dicto[key], str):
dicto[key] = int(dicto[key])
elif key in ['user_ids']:
user_ids2 = []
for user_id in dicto['user_ids']:
user_ids2.append(int(user_id))
dicto['user_ids'] = user_ids2
return list_of_dict
########################################
########## registry oriented
########################################
def augment_records_with_testbed_info (self, sfa_records):
return self.fill_record_info (sfa_records)
##########
def register (self, sfa_record, hrn, pub_key):
type = sfa_record['type']
nitos_record = self.sfa_fields_to_nitos_fields(type, hrn, sfa_record)
if type == 'authority':
pointer = -1
elif type == 'slice':
slices = self.shell.getSlices()
# filter slices
for slice in slices:
if slice['slice_name'] == nitos_record['name']:
slice_id = slice['slice_id']
break
if not slice_id:
pointer = self.shell.addSlice({'slice_name' : nitos_record['name']})
else:
pointer = slice_id
elif type == 'user':
users = self.shell.getUsers()
# filter users
for user in users:
if user['user_name'] == nitos_record['name']:
user_id = user['user_id']
break
if not user_id:
pointer = self.shell.addUser({'username' : nitos_record['name'], 'email' : nitos_record['email']})
else:
pointer = user_id
# Add the user's key
if pub_key:
self.shell.addUserKey({'user_id' : pointer,'key' : pub_key})
elif type == 'node':
nodes = self.shell.GetNodes({}, [])
# filter nodes
for node in nodes:
if node['hostname'] == nitos_record['name']:
node_id = node['node_id']
break
if not node_id:
pointer = self.shell.addNode(nitos_record)
else:
pointer = node_id
return pointer
##########
def update (self, old_sfa_record, new_sfa_record, hrn, new_key):
pointer = old_sfa_record['pointer']
type = old_sfa_record['type']
new_nitos_record = self.sfa_fields_to_nitos_fields(type, hrn, new_sfa_record)
# new_key implemented for users only
if new_key and type not in [ 'user' ]:
raise UnknownSfaType(type)
if type == "slice":
if 'name' in new_sfa_record:
self.shell.updateSlice({'slice_id': pointer, 'fields': {'slice_name': new_sfa_record['name']}})
elif type == "user":
update_fields = {}
if 'name' in new_sfa_record:
update_fields['username'] = new_sfa_record['name']
if 'email' in new_sfa_record:
update_fields['email'] = new_sfa_record['email']
self.shell.updateUser({'user_id': pointer, 'fields': update_fields})
if new_key:
# needs to be improved
self.shell.addUserKey({'user_id': pointer, 'key': new_key})
elif type == "node":
self.shell.updateNode({'node_id': pointer, 'fields': new_sfa_record})
return True
##########
def remove (self, sfa_record):
type=sfa_record['type']
pointer=sfa_record['pointer']
if type == 'user':
self.shell.deleteUser({'user_id': pointer})
elif type == 'slice':
self.shell.deleteSlice({'slice_id': pointer})
elif type == 'node':
self.shell.deleteNode({'node_id': pointer})
return True
##
# Convert SFA fields to NITOS fields for use when registering or updating
# registry record in the NITOS Scheduler database
#
def sfa_fields_to_nitos_fields(self, type, hrn, sfa_record):
nitos_record = {}
if type == "slice":
nitos_record["slice_name"] = hrn_to_nitos_slicename(hrn)
elif type == "node":
if "hostname" not in sfa_record:
raise MissingSfaInfo("hostname")
nitos_record["node_name"] = sfa_record["hostname"]
return nitos_record
####################
def fill_record_info(self, records):
"""
Given a (list of) SFA record, fill in the NITOS specific
and SFA specific fields in the record.
"""
if not isinstance(records, list):
records = [records]
self.fill_record_nitos_info(records)
self.fill_record_hrns(records)
self.fill_record_sfa_info(records)
return records
def fill_record_nitos_info(self, records):
"""
Fill in the nitos specific fields of a SFA record. This
involves calling the appropriate NITOS API method to retrieve the
database record for the object.
@param record: record to fill in field (in/out param)
"""
# get ids by type
node_ids, slice_ids = [], []
user_ids, key_ids = [], []
type_map = {'node': node_ids, 'slice': slice_ids, 'user': user_ids}
for record in records:
for type in type_map:
if type == record['type']:
type_map[type].append(record['pointer'])
# get nitos records
nodes, slices, users, keys = {}, {}, {}, {}
if node_ids:
all_nodes = self.convert_id(self.shell.getNodes({}, []))
node_list = [node for node in all_nodes if node['node_id'] in node_ids]
nodes = list_to_dict(node_list, 'node_id')
if slice_ids:
all_slices = self.convert_id(self.shell.getSlices({}, []))
slice_list = [slice for slice in all_slices if slice['slice_id'] in slice_ids]
slices = list_to_dict(slice_list, 'slice_id')
if user_ids:
all_users = self.convert_id(self.shell.getUsers())
user_list = [user for user in all_users if user['user_id'] in user_ids]
users = list_to_dict(user_list, 'user_id')
nitos_records = {'node': nodes, 'slice': slices, 'user': users}
# fill record info
for record in records:
if record['pointer'] == -1:
continue
for type in nitos_records:
if record['type'] == type:
if record['pointer'] in nitos_records[type]:
record.update(nitos_records[type][record['pointer']])
break
# fill in key info
if record['type'] == 'user':
if record['pointer'] in nitos_records['user']:
record['keys'] = nitos_records['user'][record['pointer']]['keys']
return records
def fill_record_hrns(self, records):
"""
convert nitos ids to hrns
"""
# get ids
slice_ids, user_ids, node_ids = [], [], []
for record in records:
if 'user_ids' in record:
user_ids.extend(record['user_ids'])
if 'slice_ids' in record:
slice_ids.extend(record['slice_ids'])
if 'node_ids' in record:
node_ids.extend(record['node_ids'])
# get nitos records
slices, users, nodes = {}, {}, {}
if node_ids:
all_nodes = self.convert_id(self.shell.getNodes({}, []))
node_list = [node for node in all_nodes if node['node_id'] in node_ids]
nodes = list_to_dict(node_list, 'node_id')
if slice_ids:
all_slices = self.convert_id(self.shell.getSlices({}, []))
slice_list = [slice for slice in all_slices if slice['slice_id'] in slice_ids]
slices = list_to_dict(slice_list, 'slice_id')
if user_ids:
all_users = self.convert_id(self.shell.getUsers())
user_list = [user for user in all_users if user['user_id'] in user_ids]
users = list_to_dict(user_list, 'user_id')
# convert ids to hrns
for record in records:
# get all relevant data
type = record['type']
pointer = record['pointer']
auth_hrn = self.hrn
testbed_name = self.testbedInfo['name']
if pointer == -1:
continue
if 'user_ids' in record:
usernames = [users[user_id]['username'] for user_id in record['user_ids'] \
if user_id in users]
user_hrns = [".".join([auth_hrn, testbed_name, username]) for username in usernames]
record['users'] = user_hrns
if 'slice_ids' in record:
slicenames = [slices[slice_id]['slice_name'] for slice_id in record['slice_ids'] \
if slice_id in slices]
slice_hrns = [slicename_to_hrn(auth_hrn, slicename) for slicename in slicenames]
record['slices'] = slice_hrns
if 'node_ids' in record:
hostnames = [nodes[node_id]['hostname'] for node_id in record['node_ids'] \
if node_id in nodes]
node_hrns = [hostname_to_hrn(auth_hrn, login_base, hostname) for hostname in hostnames]
record['nodes'] = node_hrns
if 'expires' in record:
date = utcparse(record['expires'])
datestring = datetime_to_string(date)
record['expires'] = datestring
return records
def fill_record_sfa_info(self, records):
def startswith(prefix, values):
return [value for value in values if value.startswith(prefix)]
# get user ids
user_ids = []
for record in records:
user_ids.extend(record.get("user_ids", []))
# get the registry records
user_list, users = [], {}
user_list = dbsession.query(RegRecord).filter(RegRecord.pointer.in_(user_ids)).all()
# create a hrns keyed on the sfa record's pointer.
# Its possible for multiple records to have the same pointer so
# the dict's value will be a list of hrns.
users = defaultdict(list)
for user in user_list:
users[user.pointer].append(user)
# get the nitos records
nitos_user_list, nitos_users = [], {}
nitos_all_users = self.convert_id(self.shell.getUsers())
nitos_user_list = [user for user in nitos_all_users if user['user_id'] in user_ids]
nitos_users = list_to_dict(nitos_user_list, 'user_id')
# fill sfa info
for record in records:
if record['pointer'] == -1:
continue
sfa_info = {}
type = record['type']
logger.info("fill_record_sfa_info - incoming record typed %s"%type)
if (type == "slice"):
# all slice users are researchers
record['geni_urn'] = hrn_to_urn(record['hrn'], 'slice')
record['researcher'] = []
for user_id in record.get('user_ids', []):
hrns = [user.hrn for user in users[user_id]]
record['researcher'].extend(hrns)
elif (type == "node"):
sfa_info['dns'] = record.get("hostname", "")
# xxx TODO: URI, LatLong, IP, DNS
elif (type == "user"):
logger.info('setting user.email')
sfa_info['email'] = record.get("email", "")
sfa_info['geni_urn'] = hrn_to_urn(record['hrn'], 'user')
sfa_info['geni_certificate'] = record['gid']
# xxx TODO: PostalAddress, Phone
record.update(sfa_info)
####################
def update_relation (self, subject_type, target_type, relation_name, subject_id, target_ids):
if subject_type =='slice' and target_type == 'user' and relation_name == 'researcher':
subject=self.shell.getSlices ({'slice_id': subject_id}, [])[0]
current_target_ids = subject['user_ids']
add_target_ids = list ( set (target_ids).difference(current_target_ids))
del_target_ids = list ( set (current_target_ids).difference(target_ids))
logger.debug ("subject_id = %s (type=%s)"%(subject_id,type(subject_id)))
for target_id in add_target_ids:
self.shell.addUserToSlice ({'user_id': target_id, 'slice_id': subject_id})
logger.debug ("add_target_id = %s (type=%s)"%(target_id,type(target_id)))
for target_id in del_target_ids:
logger.debug ("del_target_id = %s (type=%s)"%(target_id,type(target_id)))
self.shell.deleteUserFromSlice ({'user_id': target_id, 'slice_id': subject_id})
else:
logger.info('unexpected relation %s to maintain, %s -> %s'%(relation_name,subject_type,target_type))
########################################
########## aggregate oriented
########################################
def testbed_name (self): return "nitos"
# 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
def aggregate_version (self):
version_manager = VersionManager()
ad_rspec_versions = []
request_rspec_versions = []
for rspec_version in version_manager.versions:
if rspec_version.content_type in ['*', 'ad']:
ad_rspec_versions.append(rspec_version.to_dict())
if rspec_version.content_type in ['*', 'request']:
request_rspec_versions.append(rspec_version.to_dict())
return {
'testbed':self.testbed_name(),
'geni_request_rspec_versions': request_rspec_versions,
'geni_ad_rspec_versions': ad_rspec_versions,
}
def list_slices (self, creds, options):
# look in cache first
if self.cache:
slices = self.cache.get('slices')
if slices:
logger.debug("NitosDriver.list_slices returns from cache")
return slices
# get data from db
slices = self.shell.getSlices({}, [])
testbed_name = self.testbedInfo['name']
slice_hrns = [slicename_to_hrn(self.hrn, testbed_name, slice['slice_name']) for slice in slices]
slice_urns = [hrn_to_urn(slice_hrn, 'slice') for slice_hrn in slice_hrns]
# cache the result
if self.cache:
logger.debug ("NitosDriver.list_slices stores value in cache")
self.cache.add('slices', slice_urns)
return slice_urns
# first 2 args are None in case of resource discovery
def list_resources (self, slice_urn, slice_hrn, creds, options):
cached_requested = options.get('cached', True)
version_manager = VersionManager()
# get the rspec's return format from options
#rspec_version = version_manager.get_version(options.get('geni_rspec_version'))
# rspec's return format for nitos aggregate is version NITOS 1
rspec_version = version_manager.get_version('NITOS 1')
version_string = "rspec_%s" % (rspec_version)
#panos adding the info option to the caching key (can be improved)
if options.get('info'):
version_string = version_string + "_"+options.get('info', 'default')
# Adding the list_leases option to the caching key
if options.get('list_leases'):
version_string = version_string + "_"+options.get('list_leases', 'default')
# Adding geni_available to caching key
if options.get('geni_available'):
version_string = version_string + "_" + str(options.get('geni_available'))
# look in cache first
if cached_requested and self.cache and not slice_hrn:
rspec = self.cache.get(version_string)
if rspec:
logger.debug("NitosDriver.ListResources: returning cached advertisement")
return rspec
#panos: passing user-defined options
#print "manager options = ",options
aggregate = NitosAggregate(self)
rspec = aggregate.get_rspec(slice_xrn=slice_urn, version=rspec_version,
options=options)
# cache the result
if self.cache and not slice_hrn:
logger.debug("NitosDriver.ListResources: stores advertisement in cache")
self.cache.add(version_string, rspec)
return rspec
def sliver_status (self, slice_urn, slice_hrn):
# find out where this slice is currently running
slicename = hrn_to_nitos_slicename(slice_hrn)
slices = self.shell.getSlices({}, [])
# filter slicename
if len(slices) == 0:
raise SliverDoesNotExist("%s (used %s as slicename internally)" % (slice_hrn, slicename))
for slice in slices:
if slice['slice_name'] == slicename:
user_slice = slice
break
if not user_slice:
raise SliverDoesNotExist("%s (used %s as slicename internally)" % (slice_hrn, slicename))
# report about the reserved nodes only
reserved_nodes = self.shell.getReservedNodes({}, [])
nodes = self.shell.getNodes({}, [])
slice_reserved_nodes = []
for r_node in reserved_nodes:
if r_node['slice_id'] == slice['slice_id']:
for node in nodes:
if node['node_id'] == r_node['node_id']:
slice_reserved_nodes.append(node)
if len(slice_reserved_nodes) == 0:
raise SliverDoesNotExist("You have not allocated any slivers here")
##### continue from here
# get login info
user = {}
keys = []
if slice['user_ids']:
users = self.shell.getUsers()
# filter users on slice['user_ids']
for usr in users:
if usr['user_id'] in slice['user_ids']:
keys.extend(usr['keys'])
user.update({'urn': slice_urn,
'login': slice['slice_name'],
'protocol': ['ssh'],
'port': ['22'],
'keys': keys})
result = {}
top_level_status = 'unknown'
if slice_reserved_nodes:
top_level_status = 'ready'
result['geni_urn'] = slice_urn
result['nitos_gateway_login'] = slice['slice_name']
#result['pl_expires'] = datetime_to_string(utcparse(slice['expires']))
#result['geni_expires'] = datetime_to_string(utcparse(slice['expires']))
resources = []
for node in slice_reserved_nodes:
res = {}
res['nitos_hostname'] = node['hostname']
sliver_id = Xrn(slice_urn, type='slice', id=node['node_id']).urn
res['geni_urn'] = sliver_id
res['geni_status'] = 'ready'
res['geni_error'] = ''
res['users'] = [user]
resources.append(res)
result['geni_status'] = top_level_status
result['geni_resources'] = resources
return result
def create_sliver (self, slice_urn, slice_hrn, creds, rspec_string, users, options):
aggregate = NitosAggregate(self)
slices = NitosSlices(self)
sfa_peer = slices.get_sfa_peer(slice_hrn)
slice_record=None
if users:
slice_record = users[0].get('slice_record', {})
# parse rspec
rspec = RSpec(rspec_string, version='NITOS 1')
# ensure slice record exists
slice = slices.verify_slice(slice_hrn, slice_record, sfa_peer, options=options)
# ensure user records exists
users = slices.verify_users(slice_hrn, slice, users, sfa_peer, options=options)
# add/remove leases (nodes and channels)
# a lease in Nitos RSpec case is a reservation of nodes and channels grouped by (slice,timeslot)
rspec_requested_nodes, rspec_requested_channels = rspec.version.get_leases()
nodes = slices.verify_slice_leases_nodes(slice, rspec_requested_nodes)
channels = slices.verify_slice_leases_channels(slice, rspec_requested_channels)
return aggregate.get_rspec(slice_xrn=slice_urn, version=rspec.version)
def delete_sliver (self, slice_urn, slice_hrn, creds, options):
slicename = hrn_to_nitos_slicename(slice_hrn)
slices = self.filter_nitos_results(self.shell.getSlices({}, []), {'slice_name': slicename})
if not slices:
return 1
slice = slices[0]
slice_reserved_nodes = self.filter_nitos_results(self.shell.getReservedNodes({}, []), {'slice_id': slice['slice_id'] })
slice_reserved_channels = self.filter_nitos_results(self.shell.getReservedChannels(), {'slice_id': slice['slice_id'] })
slice_reserved_nodes_ids = [node['reservation_id'] for node in slice_reserved_nodes]
slice_reserved_channels_ids = [channel['reservation_id'] for channel in slice_reserved_channels]
# release all reserved nodes and channels for that slice
try:
released_nodes = self.shell.releaseNodes({'reservation_ids': slice_reserved_nodes_ids})
released_channels = self.shell.releaseChannels({'reservation_ids': slice_reserved_channels_ids})
except:
pass
return 1
def renew_sliver (self, slice_urn, slice_hrn, creds, expiration_time, options):
slicename = hrn_to_nitos_slicename(slice_hrn)
slices = self.shell.GetSlices({'slicename': slicename}, ['slice_id'])
if not slices:
raise RecordNotFound(slice_hrn)
slice = slices[0]
requested_time = utcparse(expiration_time)
record = {'expires': int(datetime_to_epoch(requested_time))}
try:
self.shell.UpdateSlice(slice['slice_id'], record)
return True
except:
return False
# xxx this code is quite old and has not run for ages
# it is obviously totally broken and needs a rewrite
def get_ticket (self, slice_urn, slice_hrn, creds, rspec_string, options):
raise SfaNotImplemented,"NitosDriver.get_ticket needs a rewrite"
class NitosDriver(Driver):
# the cache instance is a class member so it survives across incoming requests
cache = None
def __init__(self, api):
Driver.__init__(self, api)
config = api.config
self.shell = NitosShell(config)
self.cache = None
self.testbedInfo = self.shell.getTestbedInfo()
# un-comment below lines to enable caching
# if config.SFA_AGGREGATE_CACHING:
# if NitosDriver.cache is None:
# NitosDriver.cache = Cache()
# self.cache = NitosDriver.cache
###########################################
########## utility methods for NITOS driver
###########################################
def filter_nitos_results(self, listo, filters_dict):
"""
the Nitos scheduler API does not provide a get result filtring so we do it here
"""
mylist = []
mylist.extend(listo)
for dicto in mylist:
for filter in filters_dict:
if filter not in dicto or dicto[filter] != filters_dict[filter]:
listo.remove(dicto)
break
return listo
def convert_id(self, list_of_dict):
"""
convert object id retrived in string format to int format
"""
for dicto in list_of_dict:
for key in dicto:
if key in [
'node_id', 'slice_id', 'user_id', 'channel_id',
'reservation_id'
] and isinstance(dicto[key], str):
dicto[key] = int(dicto[key])
elif key in ['user_ids']:
user_ids2 = []
for user_id in dicto['user_ids']:
user_ids2.append(int(user_id))
dicto['user_ids'] = user_ids2
return list_of_dict
########################################
########## registry oriented
########################################
def augment_records_with_testbed_info(self, sfa_records):
return self.fill_record_info(sfa_records)
##########
def register(self, sfa_record, hrn, pub_key):
type = sfa_record['type']
nitos_record = self.sfa_fields_to_nitos_fields(type, hrn, sfa_record)
if type == 'authority':
pointer = -1
elif type == 'slice':
slices = self.shell.getSlices()
# filter slices
for slice in slices:
if slice['slice_name'] == nitos_record['name']:
slice_id = slice['slice_id']
break
if not slice_id:
pointer = self.shell.addSlice(
{'slice_name': nitos_record['name']})
else:
pointer = slice_id
elif type == 'user':
users = self.shell.getUsers()
# filter users
for user in users:
if user['user_name'] == nitos_record['name']:
user_id = user['user_id']
break
if not user_id:
pointer = self.shell.addUser({
'username': nitos_record['name'],
'email': nitos_record['email']
})
else:
pointer = user_id
# Add the user's key
if pub_key:
self.shell.addUserKey({'user_id': pointer, 'key': pub_key})
elif type == 'node':
nodes = self.shell.GetNodes({}, [])
# filter nodes
for node in nodes:
if node['hostname'] == nitos_record['name']:
node_id = node['node_id']
break
if not node_id:
pointer = self.shell.addNode(nitos_record)
else:
pointer = node_id
return pointer
##########
def update(self, old_sfa_record, new_sfa_record, hrn, new_key):
pointer = old_sfa_record['pointer']
type = old_sfa_record['type']
new_nitos_record = self.sfa_fields_to_nitos_fields(
type, hrn, new_sfa_record)
# new_key implemented for users only
if new_key and type not in ['user']:
raise UnknownSfaType(type)
if type == "slice":
if 'name' in new_sfa_record:
self.shell.updateSlice({
'slice_id': pointer,
'fields': {
'slice_name': new_sfa_record['name']
}
})
elif type == "user":
update_fields = {}
if 'name' in new_sfa_record:
update_fields['username'] = new_sfa_record['name']
if 'email' in new_sfa_record:
update_fields['email'] = new_sfa_record['email']
self.shell.updateUser({
'user_id': pointer,
'fields': update_fields
})
if new_key:
# needs to be improved
self.shell.addUserKey({'user_id': pointer, 'key': new_key})
elif type == "node":
self.shell.updateNode({
'node_id': pointer,
'fields': new_sfa_record
})
return True
##########
def remove(self, sfa_record):
type = sfa_record['type']
pointer = sfa_record['pointer']
if type == 'user':
self.shell.deleteUser({'user_id': pointer})
elif type == 'slice':
self.shell.deleteSlice({'slice_id': pointer})
elif type == 'node':
self.shell.deleteNode({'node_id': pointer})
return True
##
# Convert SFA fields to NITOS fields for use when registering or updating
# registry record in the NITOS Scheduler database
#
def sfa_fields_to_nitos_fields(self, type, hrn, sfa_record):
nitos_record = {}
if type == "slice":
nitos_record["slice_name"] = hrn_to_nitos_slicename(hrn)
elif type == "node":
if "hostname" not in sfa_record:
raise MissingSfaInfo("hostname")
nitos_record["node_name"] = sfa_record["hostname"]
return nitos_record
####################
def fill_record_info(self, records):
"""
Given a (list of) SFA record, fill in the NITOS specific
and SFA specific fields in the record.
"""
if not isinstance(records, list):
records = [records]
self.fill_record_nitos_info(records)
self.fill_record_hrns(records)
self.fill_record_sfa_info(records)
return records
def fill_record_nitos_info(self, records):
"""
Fill in the nitos specific fields of a SFA record. This
involves calling the appropriate NITOS API method to retrieve the
database record for the object.
@param record: record to fill in field (in/out param)
"""
# get ids by type
node_ids, slice_ids = [], []
user_ids, key_ids = [], []
type_map = {'node': node_ids, 'slice': slice_ids, 'user': user_ids}
for record in records:
for type in type_map:
if type == record['type']:
type_map[type].append(record['pointer'])
# get nitos records
nodes, slices, users, keys = {}, {}, {}, {}
if node_ids:
all_nodes = self.convert_id(self.shell.getNodes({}, []))
node_list = [
node for node in all_nodes if node['node_id'] in node_ids
]
nodes = list_to_dict(node_list, 'node_id')
if slice_ids:
all_slices = self.convert_id(self.shell.getSlices({}, []))
slice_list = [
slice for slice in all_slices if slice['slice_id'] in slice_ids
]
slices = list_to_dict(slice_list, 'slice_id')
if user_ids:
all_users = self.convert_id(self.shell.getUsers())
user_list = [
user for user in all_users if user['user_id'] in user_ids
]
users = list_to_dict(user_list, 'user_id')
nitos_records = {'node': nodes, 'slice': slices, 'user': users}
# fill record info
for record in records:
if record['pointer'] == -1:
continue
for type in nitos_records:
if record['type'] == type:
if record['pointer'] in nitos_records[type]:
record.update(nitos_records[type][record['pointer']])
break
# fill in key info
if record['type'] == 'user':
if record['pointer'] in nitos_records['user']:
record['keys'] = nitos_records['user'][
record['pointer']]['keys']
return records
def fill_record_hrns(self, records):
"""
convert nitos ids to hrns
"""
# get ids
slice_ids, user_ids, node_ids = [], [], []
for record in records:
if 'user_ids' in record:
user_ids.extend(record['user_ids'])
if 'slice_ids' in record:
slice_ids.extend(record['slice_ids'])
if 'node_ids' in record:
node_ids.extend(record['node_ids'])
# get nitos records
slices, users, nodes = {}, {}, {}
if node_ids:
all_nodes = self.convert_id(self.shell.getNodes({}, []))
node_list = [
node for node in all_nodes if node['node_id'] in node_ids
]
nodes = list_to_dict(node_list, 'node_id')
if slice_ids:
all_slices = self.convert_id(self.shell.getSlices({}, []))
slice_list = [
slice for slice in all_slices if slice['slice_id'] in slice_ids
]
slices = list_to_dict(slice_list, 'slice_id')
if user_ids:
all_users = self.convert_id(self.shell.getUsers())
user_list = [
user for user in all_users if user['user_id'] in user_ids
]
users = list_to_dict(user_list, 'user_id')
# convert ids to hrns
for record in records:
# get all relevant data
type = record['type']
pointer = record['pointer']
auth_hrn = self.hrn
testbed_name = self.testbedInfo['name']
if pointer == -1:
continue
if 'user_ids' in record:
usernames = [users[user_id]['username'] for user_id in record['user_ids'] \
if user_id in users]
user_hrns = [
".".join([auth_hrn, testbed_name, username])
for username in usernames
]
record['users'] = user_hrns
if 'slice_ids' in record:
slicenames = [slices[slice_id]['slice_name'] for slice_id in record['slice_ids'] \
if slice_id in slices]
slice_hrns = [
slicename_to_hrn(auth_hrn, slicename)
for slicename in slicenames
]
record['slices'] = slice_hrns
if 'node_ids' in record:
hostnames = [nodes[node_id]['hostname'] for node_id in record['node_ids'] \
if node_id in nodes]
node_hrns = [
hostname_to_hrn(auth_hrn, login_base, hostname)
for hostname in hostnames
]
record['nodes'] = node_hrns
if 'expires' in record:
date = utcparse(record['expires'])
datestring = datetime_to_string(date)
record['expires'] = datestring
return records
def fill_record_sfa_info(self, records):
def startswith(prefix, values):
return [value for value in values if value.startswith(prefix)]
# get user ids
user_ids = []
for record in records:
user_ids.extend(record.get("user_ids", []))
# get the registry records
user_list, users = [], {}
user_list = self.api.dbsession().query(RegRecord).filter(
RegRecord.pointer.in_(user_ids)).all()
# create a hrns keyed on the sfa record's pointer.
# Its possible for multiple records to have the same pointer so
# the dict's value will be a list of hrns.
users = defaultdict(list)
for user in user_list:
users[user.pointer].append(user)
# get the nitos records
nitos_user_list, nitos_users = [], {}
nitos_all_users = self.convert_id(self.shell.getUsers())
nitos_user_list = [
user for user in nitos_all_users if user['user_id'] in user_ids
]
nitos_users = list_to_dict(nitos_user_list, 'user_id')
# fill sfa info
for record in records:
if record['pointer'] == -1:
continue
sfa_info = {}
type = record['type']
logger.info("fill_record_sfa_info - incoming record typed %s" %
type)
if (type == "slice"):
# all slice users are researchers
record['geni_urn'] = hrn_to_urn(record['hrn'], 'slice')
record['researcher'] = []
for user_id in record.get('user_ids', []):
hrns = [user.hrn for user in users[user_id]]
record['researcher'].extend(hrns)
elif (type == "node"):
sfa_info['dns'] = record.get("hostname", "")
# xxx TODO: URI, LatLong, IP, DNS
elif (type == "user"):
logger.info('setting user.email')
sfa_info['email'] = record.get("email", "")
sfa_info['geni_urn'] = hrn_to_urn(record['hrn'], 'user')
sfa_info['geni_certificate'] = record['gid']
# xxx TODO: PostalAddress, Phone
record.update(sfa_info)
####################
def update_relation(self, subject_type, target_type, relation_name,
subject_id, target_ids):
if subject_type == 'slice' and target_type == 'user' and relation_name == 'researcher':
subject = self.shell.getSlices({'slice_id': subject_id}, [])[0]
current_target_ids = subject['user_ids']
add_target_ids = list(
set(target_ids).difference(current_target_ids))
del_target_ids = list(
set(current_target_ids).difference(target_ids))
logger.debug("subject_id = %s (type=%s)" %
(subject_id, type(subject_id)))
for target_id in add_target_ids:
self.shell.addUserToSlice({
'user_id': target_id,
'slice_id': subject_id
})
logger.debug("add_target_id = %s (type=%s)" %
(target_id, type(target_id)))
for target_id in del_target_ids:
logger.debug("del_target_id = %s (type=%s)" %
(target_id, type(target_id)))
self.shell.deleteUserFromSlice({
'user_id': target_id,
'slice_id': subject_id
})
else:
logger.info('unexpected relation %s to maintain, %s -> %s' %
(relation_name, subject_type, target_type))
########################################
########## aggregate oriented
########################################
def testbed_name(self):
return "nitos"
# 'geni_request_rspec_versions' and 'geni_ad_rspec_versions' are mandatory
def aggregate_version(self):
version_manager = VersionManager()
ad_rspec_versions = []
request_rspec_versions = []
for rspec_version in version_manager.versions:
if rspec_version.content_type in ['*', 'ad']:
ad_rspec_versions.append(rspec_version.to_dict())
if rspec_version.content_type in ['*', 'request']:
request_rspec_versions.append(rspec_version.to_dict())
return {
'testbed': self.testbed_name(),
'geni_request_rspec_versions': request_rspec_versions,
'geni_ad_rspec_versions': ad_rspec_versions,
}
def list_slices(self, creds, options):
# look in cache first
if self.cache:
slices = self.cache.get('slices')
if slices:
logger.debug("NitosDriver.list_slices returns from cache")
return slices
# get data from db
slices = self.shell.getSlices({}, [])
testbed_name = self.testbedInfo['name']
slice_hrns = [
slicename_to_hrn(self.hrn, testbed_name, slice['slice_name'])
for slice in slices
]
slice_urns = [
hrn_to_urn(slice_hrn, 'slice') for slice_hrn in slice_hrns
]
# cache the result
if self.cache:
logger.debug("NitosDriver.list_slices stores value in cache")
self.cache.add('slices', slice_urns)
return slice_urns
# first 2 args are None in case of resource discovery
def list_resources(self, slice_urn, slice_hrn, creds, options):
cached_requested = options.get('cached', True)
version_manager = VersionManager()
# get the rspec's return format from options
#rspec_version = version_manager.get_version(options.get('geni_rspec_version'))
# rspec's return format for nitos aggregate is version NITOS 1
rspec_version = version_manager.get_version('NITOS 1')
version_string = "rspec_%s" % (rspec_version)
#panos adding the info option to the caching key (can be improved)
if options.get('info'):
version_string = version_string + "_" + options.get(
'info', 'default')
# Adding the list_leases option to the caching key
if options.get('list_leases'):
version_string = version_string + "_" + options.get(
'list_leases', 'default')
# Adding geni_available to caching key
if options.get('geni_available'):
version_string = version_string + "_" + str(
options.get('geni_available'))
# look in cache first
if cached_requested and self.cache and not slice_hrn:
rspec = self.cache.get(version_string)
if rspec:
logger.debug(
"NitosDriver.ListResources: returning cached advertisement"
)
return rspec
#panos: passing user-defined options
#print "manager options = ",options
aggregate = NitosAggregate(self)
rspec = aggregate.get_rspec(slice_xrn=slice_urn,
version=rspec_version,
options=options)
# cache the result
if self.cache and not slice_hrn:
logger.debug(
"NitosDriver.ListResources: stores advertisement in cache")
self.cache.add(version_string, rspec)
return rspec
def sliver_status(self, slice_urn, slice_hrn):
# find out where this slice is currently running
slicename = hrn_to_nitos_slicename(slice_hrn)
slices = self.shell.getSlices({}, [])
# filter slicename
if len(slices) == 0:
raise SliverDoesNotExist("%s (used %s as slicename internally)" %
(slice_hrn, slicename))
for slice in slices:
if slice['slice_name'] == slicename:
user_slice = slice
break
if not user_slice:
raise SliverDoesNotExist("%s (used %s as slicename internally)" %
(slice_hrn, slicename))
# report about the reserved nodes only
reserved_nodes = self.shell.getReservedNodes({}, [])
nodes = self.shell.getNodes({}, [])
slice_reserved_nodes = []
for r_node in reserved_nodes:
if r_node['slice_id'] == slice['slice_id']:
for node in nodes:
if node['node_id'] == r_node['node_id']:
slice_reserved_nodes.append(node)
if len(slice_reserved_nodes) == 0:
raise SliverDoesNotExist("You have not allocated any slivers here")
##### continue from here
# get login info
user = {}
keys = []
if slice['user_ids']:
users = self.shell.getUsers()
# filter users on slice['user_ids']
for usr in users:
if usr['user_id'] in slice['user_ids']:
keys.extend(usr['keys'])
user.update({
'urn': slice_urn,
'login': slice['slice_name'],
'protocol': ['ssh'],
'port': ['22'],
'keys': keys
})
result = {}
top_level_status = 'unknown'
if slice_reserved_nodes:
top_level_status = 'ready'
result['geni_urn'] = slice_urn
result['nitos_gateway_login'] = slice['slice_name']
#result['pl_expires'] = datetime_to_string(utcparse(slice['expires']))
#result['geni_expires'] = datetime_to_string(utcparse(slice['expires']))
resources = []
for node in slice_reserved_nodes:
res = {}
res['nitos_hostname'] = node['hostname']
sliver_id = Xrn(slice_urn, type='slice', id=node['node_id']).urn
res['geni_urn'] = sliver_id
res['geni_status'] = 'ready'
res['geni_error'] = ''
res['users'] = [user]
resources.append(res)
result['geni_status'] = top_level_status
result['geni_resources'] = resources
return result
def create_sliver(self, slice_urn, slice_hrn, creds, rspec_string, users,
options):
aggregate = NitosAggregate(self)
slices = NitosSlices(self)
sfa_peer = slices.get_sfa_peer(slice_hrn)
slice_record = None
if users:
slice_record = users[0].get('slice_record', {})
# parse rspec
rspec = RSpec(rspec_string, version='NITOS 1')
# ensure slice record exists
slice = slices.verify_slice(slice_hrn,
slice_record,
sfa_peer,
options=options)
# ensure user records exists
users = slices.verify_users(slice_hrn,
slice,
users,
sfa_peer,
options=options)
# add/remove leases (nodes and channels)
# a lease in Nitos RSpec case is a reservation of nodes and channels grouped by (slice,timeslot)
rspec_requested_leases = rspec.version.get_leases()
rspec_requested_nodes = []
rspec_requested_channels = []
for lease in rspec_requested_leases:
if lease['type'] == 'node':
lease.pop('type', None)
rspec_requested_nodes.append(lease)
else:
lease.pop('type', None)
rspec_requested_channels.append(lease)
nodes = slices.verify_slice_leases_nodes(slice, rspec_requested_nodes)
channels = slices.verify_slice_leases_channels(
slice, rspec_requested_channels)
return aggregate.get_rspec(slice_xrn=slice_urn, version=rspec.version)
def delete_sliver(self, slice_urn, slice_hrn, creds, options):
slicename = hrn_to_nitos_slicename(slice_hrn)
slices = self.filter_nitos_results(self.shell.getSlices({}, []),
{'slice_name': slicename})
if not slices:
return 1
slice = slices[0]
slice_reserved_nodes = self.filter_nitos_results(
self.shell.getReservedNodes({}, []),
{'slice_id': slice['slice_id']})
slice_reserved_channels = self.filter_nitos_results(
self.shell.getReservedChannels(), {'slice_id': slice['slice_id']})
slice_reserved_nodes_ids = [
node['reservation_id'] for node in slice_reserved_nodes
]
slice_reserved_channels_ids = [
channel['reservation_id'] for channel in slice_reserved_channels
]
# release all reserved nodes and channels for that slice
try:
released_nodes = self.shell.releaseNodes(
{'reservation_ids': slice_reserved_nodes_ids})
released_channels = self.shell.releaseChannels(
{'reservation_ids': slice_reserved_channels_ids})
except:
pass
return 1
def renew_sliver(self, slice_urn, slice_hrn, creds, expiration_time,
options):
slicename = hrn_to_nitos_slicename(slice_hrn)
slices = self.shell.GetSlices({'slicename': slicename}, ['slice_id'])
if not slices:
raise RecordNotFound(slice_hrn)
slice = slices[0]
requested_time = utcparse(expiration_time)
record = {'expires': int(datetime_to_epoch(requested_time))}
try:
self.shell.UpdateSlice(slice['slice_id'], record)
return True
except:
return False
# xxx this code is quite old and has not run for ages
# it is obviously totally broken and needs a rewrite
def get_ticket(self, slice_urn, slice_hrn, creds, rspec_string, options):
raise SfaNotImplemented, "NitosDriver.get_ticket needs a rewrite"
