def list_slices(self, creds, options):
# look in cache first
if self.cache:
slices = self.cache.get('slices')
if slices:
logger.debug("NitosDriver.list_slices returns from cache")
return slices
# get data from db
slices = self.shell.getSlices({}, [])
testbed_name = self.testbedInfo['name']
slice_hrns = [
slicename_to_hrn(self.hrn, testbed_name, slice['slice_name'])
for slice in slices
]
slice_urns = [
hrn_to_urn(slice_hrn, 'slice') for slice_hrn in slice_hrns
]
# cache the result
if self.cache:
logger.debug("NitosDriver.list_slices stores value in cache")
self.cache.add('slices', slice_urns)
return slice_urns
def list_slices (self, creds, options):
# look in cache first
if self.cache:
slices = self.cache.get('slices')
if slices:
logger.debug("NitosDriver.list_slices returns from cache")
return slices
# get data from db
slices = self.shell.getSlices({}, [])
testbed_name = self.testbedInfo['name']
slice_hrns = [slicename_to_hrn(self.hrn, testbed_name, slice['slice_name']) for slice in slices]
slice_urns = [hrn_to_urn(slice_hrn, 'slice') for slice_hrn in slice_hrns]
# cache the result
if self.cache:
logger.debug ("NitosDriver.list_slices stores value in cache")
self.cache.add('slices', slice_urns)
return slice_urns
def get_leases_and_channels(self, slice=None, slice_xrn=None, options={}):
slices = self.driver.shell.getSlices({}, [])
nodes = self.driver.shell.getNodes({}, [])
leases = self.driver.shell.getReservedNodes({}, [])
channels = self.driver.shell.getChannels({}, [])
reserved_channels = self.driver.shell.getReservedChannels()
grain = self.driver.testbedInfo['grain']
if slice_xrn and not slice:
return ([], [])
if slice:
all_leases = []
all_leases.extend(leases)
all_reserved_channels = []
all_reserved_channels.extend(reserved_channels)
for lease in all_leases:
if lease['slice_id'] != slice['slice_id']:
leases.remove(lease)
for channel in all_reserved_channels:
if channel['slice_id'] != slice['slice_id']:
reserved_channels.remove(channel)
rspec_channels = []
for channel in reserved_channels:
rspec_channel = {}
#retrieve channel number
for chl in channels:
if chl['channel_id'] == channel['channel_id']:
channel_number = chl['channel']
break
rspec_channel['channel_num'] = channel_number
rspec_channel['start_time'] = channel['start_time']
rspec_channel['duration'] = (int(channel['end_time']) - int(
channel['start_time'])) / int(grain)
rspec_channel['component_id'] = channel_to_urn(
self.driver.hrn, self.driver.testbedInfo['name'],
channel_number)
# retreive slicename
for slc in slices:
if slc['slice_id'] == channel['slice_id']:
slicename = slc['slice_name']
break
if slice_xrn:
slice_urn = slice_xrn
slice_hrn = urn_to_hrn(slice_urn)
else:
slice_hrn = slicename_to_hrn(self.driver.hrn,
self.driver.testbedInfo['name'],
slicename)
slice_urn = hrn_to_urn(slice_hrn, 'slice')
rspec_channel['slice_id'] = slice_urn
rspec_channels.append(rspec_channel)
rspec_leases = []
for lease in leases:
rspec_lease = Lease()
rspec_lease['lease_id'] = lease['reservation_id']
# retreive node name
for node in nodes:
if node['node_id'] == lease['node_id']:
nodename = node['hostname']
break
rspec_lease['component_id'] = hostname_to_urn(
self.driver.hrn, self.driver.testbedInfo['name'], nodename)
# retreive slicename
for slc in slices:
if slc['slice_id'] == lease['slice_id']:
slicename = slc['slice_name']
break
if slice_xrn:
slice_urn = slice_xrn
slice_hrn = urn_to_hrn(slice_urn)
else:
slice_hrn = slicename_to_hrn(self.driver.hrn,
self.driver.testbedInfo['name'],
slicename)
slice_urn = hrn_to_urn(slice_hrn, 'slice')
rspec_lease['slice_id'] = slice_urn
rspec_lease['start_time'] = lease['start_time']
rspec_lease['duration'] = (int(lease['end_time']) -
int(lease['start_time'])) / int(grain)
rspec_leases.append(rspec_lease)
return (rspec_leases, rspec_channels)
def get_leases_and_channels(self, slice=None, slice_xrn=None, options={}):
slices = self.driver.shell.getSlices({}, [])
nodes = self.driver.shell.getNodes({}, [])
leases = self.driver.shell.getReservedNodes({}, [])
channels = self.driver.shell.getChannels({}, [])
reserved_channels = self.driver.shell.getReservedChannels()
grain = self.driver.testbedInfo['grain']
if slice_xrn and not slice:
return ([], [])
if slice:
all_leases = []
all_leases.extend(leases)
all_reserved_channels = []
all_reserved_channels.extend(reserved_channels)
for lease in all_leases:
if lease['slice_id'] != slice['slice_id']:
leases.remove(lease)
for channel in all_reserved_channels:
if channel['slice_id'] != slice['slice_id']:
reserved_channels.remove(channel)
rspec_channels = []
for channel in reserved_channels:
rspec_channel = {}
#retrieve channel number
for chl in channels:
if chl['channel_id'] == channel['channel_id']:
channel_number = chl['channel']
break
rspec_channel['channel_num'] = channel_number
rspec_channel['start_time'] = channel['start_time']
rspec_channel['duration'] = (int(channel['end_time']) - int(channel['start_time'])) / int(grain)
rspec_channel['component_id'] = channel_to_urn(self.driver.hrn, self.driver.testbedInfo['name'], channel_number)
# retreive slicename
for slc in slices:
if slc['slice_id'] == channel['slice_id']:
slicename = slc['slice_name']
break
if slice_xrn:
slice_urn = slice_xrn
slice_hrn = urn_to_hrn(slice_urn)
else:
slice_hrn = slicename_to_hrn(self.driver.hrn, self.driver.testbedInfo['name'], slicename)
slice_urn = hrn_to_urn(slice_hrn, 'slice')
rspec_channel['slice_id'] = slice_urn
rspec_channels.append(rspec_channel)
rspec_leases = []
for lease in leases:
rspec_lease = Lease()
rspec_lease['lease_id'] = lease['reservation_id']
# retreive node name
for node in nodes:
if node['node_id'] == lease['node_id']:
nodename = node['hostname']
break
rspec_lease['component_id'] = hostname_to_urn(self.driver.hrn, self.driver.testbedInfo['name'], nodename)
# retreive slicename
for slc in slices:
if slc['slice_id'] == lease['slice_id']:
slicename = slc['slice_name']
break
if slice_xrn:
slice_urn = slice_xrn
slice_hrn = urn_to_hrn(slice_urn)
else:
slice_hrn = slicename_to_hrn(self.driver.hrn, self.driver.testbedInfo['name'], slicename)
slice_urn = hrn_to_urn(slice_hrn, 'slice')
rspec_lease['slice_id'] = slice_urn
rspec_lease['start_time'] = lease['start_time']
rspec_lease['duration'] = (int(lease['end_time']) - int(lease['start_time'])) / int(grain)
rspec_leases.append(rspec_lease)
return (rspec_leases, rspec_channels)
def run (self, options):
config = Config ()
interface_hrn = config.SFA_INTERFACE_HRN
root_auth = config.SFA_REGISTRY_ROOT_AUTH
shell = NitosShell (config)
######## retrieve all existing SFA objects
all_records = global_dbsession.query(RegRecord).all()
# create hash by (type,hrn)
# we essentially use this to know if a given record is already known to SFA
self.records_by_type_hrn = \
dict ( [ ( (record.type, record.hrn) , record ) for record in all_records ] )
# create hash by (type,pointer)
self.records_by_type_pointer = \
dict ( [ ( (record.type, record.pointer) , record ) for record in all_records
if record.pointer != -1] )
# initialize record.stale to True by default, then mark stale=False on the ones that are in use
for record in all_records: record.stale=True
######## retrieve NITOS data
# Get site info
# retrieve only required stuf
site = shell.getTestbedInfo()
sites = [site]
# create a hash of sites by login_base
# # sites_by_login_base = dict ( [ ( site['login_base'], site ) for site in sites ] )
# Get all NITOS users
users = shell.getUsers()
# create a hash of users by user_id
users_by_id = dict ( [ ( user['user_id'], user) for user in users ] )
# Get all NITOS public keys
# accumulate key ids for keys retrieval
# key_ids = []
# for person in persons:
# key_ids.extend(person['key_ids'])
# keys = shell.GetKeys( {'peer_id': None, 'key_id': key_ids,
# 'key_type': 'ssh'} )
# # create a hash of keys by key_id
# keys_by_id = dict ( [ ( key['key_id'], key ) for key in keys ] )
# create a dict user_id -> [ (nitos)keys ]
keys_by_user_id = dict ( [ ( user['user_id'], user['keys']) for user in users ] )
# Get all nitos nodes
nodes = shell.getNodes({}, [])
# create hash by node_id
nodes_by_id = dict ( [ (node['node_id'], node) for node in nodes ] )
# Get all nitos slices
slices = shell.getSlices({}, [])
# create hash by slice_id
slices_by_id = dict ( [ (slice['slice_id'], slice) for slice in slices ] )
# start importing
for site in sites:
#for i in [0]:
site_hrn = _get_site_hrn(interface_hrn, site)
# import if hrn is not in list of existing hrns or if the hrn exists
# but its not a site record
site_record=self.locate_by_type_hrn ('authority', site_hrn)
if not site_record:
try:
urn = hrn_to_urn(site_hrn, 'authority')
if not self.auth_hierarchy.auth_exists(urn):
self.auth_hierarchy.create_auth(urn)
auth_info = self.auth_hierarchy.get_auth_info(urn)
site_record = RegAuthority(hrn=site_hrn, gid=auth_info.get_gid_object(),
pointer=0,
authority=get_authority(site_hrn))
site_record.just_created()
global_dbsession.add(site_record)
global_dbsession.commit()
self.logger.info("NitosImporter: imported authority (site) : %s" % site_record)
self.remember_record (site_record)
except:
# if the site import fails then there is no point in trying to import the
# site's child records (node, slices, persons), so skip them.
self.logger.log_exc("NitosImporter: failed to import site. Skipping child records")
continue
else:
# xxx update the record ...
pass
site_record.stale=False
# import node records
for node in nodes:
site_auth = get_authority(site_hrn)
site_name = site['name']
node_hrn = hostname_to_hrn(site_auth, site_name, node['hostname'])
# xxx this sounds suspicious
if len(node_hrn) > 64: node_hrn = node_hrn[:64]
node_record = self.locate_by_type_hrn ( 'node', node_hrn )
if not node_record:
try:
pkey = Keypair(create=True)
urn = hrn_to_urn(node_hrn, 'node')
node_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
node_record = RegNode (hrn=node_hrn, gid=node_gid,
pointer =node['node_id'],
authority=get_authority(node_hrn))
node_record.just_created()
global_dbsession.add(node_record)
global_dbsession.commit()
self.logger.info("NitosImporter: imported node: %s" % node_record)
self.remember_record (node_record)
except:
self.logger.log_exc("NitosImporter: failed to import node")
else:
# xxx update the record ...
pass
node_record.stale=False
# import users
for user in users:
user_hrn = username_to_hrn(interface_hrn, site['name'], user['username'])
# xxx suspicious again
if len(user_hrn) > 64: user_hrn = user_hrn[:64]
user_urn = hrn_to_urn(user_hrn, 'user')
user_record = self.locate_by_type_hrn ( 'user', user_hrn)
# return a tuple pubkey (a nitos key object) and pkey (a Keypair object)
def init_user_key (user):
pubkey = None
pkey = None
if user['keys']:
# randomly pick first key in set
for key in user['keys']:
pubkey = key
try:
pkey = convert_public_key(pubkey)
break
except:
continue
if not pkey:
self.logger.warn('NitosImporter: unable to convert public key for %s' % user_hrn)
pkey = Keypair(create=True)
else:
# the user has no keys. Creating a random keypair for the user's gid
self.logger.warn("NitosImporter: user %s does not have a NITOS public key"%user_hrn)
pkey = Keypair(create=True)
return (pubkey, pkey)
# new user
try:
if not user_record:
(pubkey,pkey) = init_user_key (user)
user_gid = self.auth_hierarchy.create_gid(user_urn, create_uuid(), pkey)
user_gid.set_email(user['email'])
user_record = RegUser (hrn=user_hrn, gid=user_gid,
pointer=user['user_id'],
authority=get_authority(user_hrn),
email=user['email'])
if pubkey:
user_record.reg_keys=[RegKey (pubkey)]
else:
self.logger.warning("No key found for user %s"%user_record)
user_record.just_created()
global_dbsession.add (user_record)
global_dbsession.commit()
self.logger.info("NitosImporter: imported user: %s" % user_record)
self.remember_record ( user_record )
else:
# update the record ?
# if user's primary key has changed then we need to update the
# users gid by forcing an update here
sfa_keys = user_record.reg_keys
def sfa_key_in_list (sfa_key,nitos_user_keys):
for nitos_key in nitos_user_keys:
if nitos_key==sfa_key: return True
return False
# are all the SFA keys known to nitos ?
new_keys=False
if not sfa_keys and user['keys']:
new_keys = True
else:
for sfa_key in sfa_keys:
if not sfa_key_in_list (sfa_key.key,user['keys']):
new_keys = True
if new_keys:
(pubkey,pkey) = init_user_key (user)
user_gid = self.auth_hierarchy.create_gid(user_urn, create_uuid(), pkey)
if not pubkey:
user_record.reg_keys=[]
else:
user_record.reg_keys=[ RegKey (pubkey)]
user_record.gid = user_gid
user_record.just_updated()
self.logger.info("NitosImporter: updated user: %s" % user_record)
user_record.email = user['email']
global_dbsession.commit()
user_record.stale=False
except:
self.logger.log_exc("NitosImporter: failed to import user %s %s"%(user['user_id'],user['email']))
# import slices
for slice in slices:
slice_hrn = slicename_to_hrn(interface_hrn, site['name'], slice['slice_name'])
slice_record = self.locate_by_type_hrn ('slice', slice_hrn)
if not slice_record:
try:
pkey = Keypair(create=True)
urn = hrn_to_urn(slice_hrn, 'slice')
slice_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey)
slice_record = RegSlice (hrn=slice_hrn, gid=slice_gid,
pointer=slice['slice_id'],
authority=get_authority(slice_hrn))
slice_record.just_created()
global_dbsession.add(slice_record)
global_dbsession.commit()
self.logger.info("NitosImporter: imported slice: %s" % slice_record)
self.remember_record ( slice_record )
except:
self.logger.log_exc("NitosImporter: failed to import slice")
else:
# xxx update the record ...
self.logger.warning ("Slice update not yet implemented")
pass
# record current users affiliated with the slice
slice_record.reg_researchers = \
[ self.locate_by_type_pointer ('user',int(user_id)) for user_id in slice['user_ids'] ]
global_dbsession.commit()
slice_record.stale=False
### remove stale records
# special records must be preserved
system_hrns = [interface_hrn, root_auth, interface_hrn + '.slicemanager']
for record in all_records:
if record.hrn in system_hrns:
record.stale=False
if record.peer_authority:
record.stale=False
for record in all_records:
try: stale=record.stale
except:
stale=True
self.logger.warning("stale not found with %s"%record)
if stale:
self.logger.info("NitosImporter: deleting stale record: %s" % record)
global_dbsession.delete(record)
global_dbsession.commit()
def fill_record_hrns(self, records):
"""
convert nitos ids to hrns
"""
# get ids
slice_ids, user_ids, node_ids = [], [], []
for record in records:
if 'user_ids' in record:
user_ids.extend(record['user_ids'])
if 'slice_ids' in record:
slice_ids.extend(record['slice_ids'])
if 'node_ids' in record:
node_ids.extend(record['node_ids'])
# get nitos records
slices, users, nodes = {}, {}, {}
if node_ids:
all_nodes = self.convert_id(self.shell.getNodes({}, []))
node_list = [node for node in all_nodes if node['node_id'] in node_ids]
nodes = list_to_dict(node_list, 'node_id')
if slice_ids:
all_slices = self.convert_id(self.shell.getSlices({}, []))
slice_list = [slice for slice in all_slices if slice['slice_id'] in slice_ids]
slices = list_to_dict(slice_list, 'slice_id')
if user_ids:
all_users = self.convert_id(self.shell.getUsers())
user_list = [user for user in all_users if user['user_id'] in user_ids]
users = list_to_dict(user_list, 'user_id')
# convert ids to hrns
for record in records:
# get all relevant data
type = record['type']
pointer = record['pointer']
auth_hrn = self.hrn
testbed_name = self.testbedInfo['name']
if pointer == -1:
continue
if 'user_ids' in record:
usernames = [users[user_id]['username'] for user_id in record['user_ids'] \
if user_id in users]
user_hrns = [".".join([auth_hrn, testbed_name, username]) for username in usernames]
record['users'] = user_hrns
if 'slice_ids' in record:
slicenames = [slices[slice_id]['slice_name'] for slice_id in record['slice_ids'] \
if slice_id in slices]
slice_hrns = [slicename_to_hrn(auth_hrn, slicename) for slicename in slicenames]
record['slices'] = slice_hrns
if 'node_ids' in record:
hostnames = [nodes[node_id]['hostname'] for node_id in record['node_ids'] \
if node_id in nodes]
node_hrns = [hostname_to_hrn(auth_hrn, login_base, hostname) for hostname in hostnames]
record['nodes'] = node_hrns
if 'expires' in record:
date = utcparse(record['expires'])
datestring = datetime_to_string(date)
record['expires'] = datestring
return records
def fill_record_hrns(self, records):
"""
convert nitos ids to hrns
"""
# get ids
slice_ids, user_ids, node_ids = [], [], []
for record in records:
if 'user_ids' in record:
user_ids.extend(record['user_ids'])
if 'slice_ids' in record:
slice_ids.extend(record['slice_ids'])
if 'node_ids' in record:
node_ids.extend(record['node_ids'])
# get nitos records
slices, users, nodes = {}, {}, {}
if node_ids:
all_nodes = self.convert_id(self.shell.getNodes({}, []))
node_list = [
node for node in all_nodes if node['node_id'] in node_ids
]
nodes = list_to_dict(node_list, 'node_id')
if slice_ids:
all_slices = self.convert_id(self.shell.getSlices({}, []))
slice_list = [
slice for slice in all_slices if slice['slice_id'] in slice_ids
]
slices = list_to_dict(slice_list, 'slice_id')
if user_ids:
all_users = self.convert_id(self.shell.getUsers())
user_list = [
user for user in all_users if user['user_id'] in user_ids
]
users = list_to_dict(user_list, 'user_id')
# convert ids to hrns
for record in records:
# get all relevant data
type = record['type']
pointer = record['pointer']
auth_hrn = self.hrn
testbed_name = self.testbedInfo['name']
if pointer == -1:
continue
if 'user_ids' in record:
usernames = [users[user_id]['username'] for user_id in record['user_ids'] \
if user_id in users]
user_hrns = [
".".join([auth_hrn, testbed_name, username])
for username in usernames
]
record['users'] = user_hrns
if 'slice_ids' in record:
slicenames = [slices[slice_id]['slice_name'] for slice_id in record['slice_ids'] \
if slice_id in slices]
slice_hrns = [
slicename_to_hrn(auth_hrn, slicename)
for slicename in slicenames
]
record['slices'] = slice_hrns
if 'node_ids' in record:
hostnames = [nodes[node_id]['hostname'] for node_id in record['node_ids'] \
if node_id in nodes]
node_hrns = [
hostname_to_hrn(auth_hrn, login_base, hostname)
for hostname in hostnames
]
record['nodes'] = node_hrns
if 'expires' in record:
date = utcparse(record['expires'])
datestring = datetime_to_string(date)
record['expires'] = datestring
return records
