def fill_record_hrns(self, records): """ convert nitos ids to hrns """ # get ids slice_ids, user_ids, node_ids = [], [], [] for record in records: if 'user_ids' in record: user_ids.extend(record['user_ids']) if 'slice_ids' in record: slice_ids.extend(record['slice_ids']) if 'node_ids' in record: node_ids.extend(record['node_ids']) # get nitos records slices, users, nodes = {}, {}, {} if node_ids: all_nodes = self.convert_id(self.shell.getNodes({}, [])) node_list = [node for node in all_nodes if node['node_id'] in node_ids] nodes = list_to_dict(node_list, 'node_id') if slice_ids: all_slices = self.convert_id(self.shell.getSlices({}, [])) slice_list = [slice for slice in all_slices if slice['slice_id'] in slice_ids] slices = list_to_dict(slice_list, 'slice_id') if user_ids: all_users = self.convert_id(self.shell.getUsers()) user_list = [user for user in all_users if user['user_id'] in user_ids] users = list_to_dict(user_list, 'user_id') # convert ids to hrns for record in records: # get all relevant data type = record['type'] pointer = record['pointer'] auth_hrn = self.hrn testbed_name = self.testbedInfo['name'] if pointer == -1: continue if 'user_ids' in record: usernames = [users[user_id]['username'] for user_id in record['user_ids'] \ if user_id in users] user_hrns = [".".join([auth_hrn, testbed_name, username]) for username in usernames] record['users'] = user_hrns if 'slice_ids' in record: slicenames = [slices[slice_id]['slice_name'] for slice_id in record['slice_ids'] \ if slice_id in slices] slice_hrns = [slicename_to_hrn(auth_hrn, slicename) for slicename in slicenames] record['slices'] = slice_hrns if 'node_ids' in record: hostnames = [nodes[node_id]['hostname'] for node_id in record['node_ids'] \ if node_id in nodes] node_hrns = [hostname_to_hrn(auth_hrn, login_base, hostname) for hostname in hostnames] record['nodes'] = node_hrns if 'expires' in record: date = utcparse(record['expires']) datestring = datetime_to_string(date) record['expires'] = datestring return records
def run (self, options): config = Config () interface_hrn = config.SFA_INTERFACE_HRN root_auth = config.SFA_REGISTRY_ROOT_AUTH shell = NitosShell (config) ######## retrieve all existing SFA objects all_records = global_dbsession.query(RegRecord).all() # create hash by (type,hrn) # we essentially use this to know if a given record is already known to SFA self.records_by_type_hrn = \ dict ( [ ( (record.type, record.hrn) , record ) for record in all_records ] ) # create hash by (type,pointer) self.records_by_type_pointer = \ dict ( [ ( (record.type, record.pointer) , record ) for record in all_records if record.pointer != -1] ) # initialize record.stale to True by default, then mark stale=False on the ones that are in use for record in all_records: record.stale=True ######## retrieve NITOS data # Get site info # retrieve only required stuf site = shell.getTestbedInfo() sites = [site] # create a hash of sites by login_base # # sites_by_login_base = dict ( [ ( site['login_base'], site ) for site in sites ] ) # Get all NITOS users users = shell.getUsers() # create a hash of users by user_id users_by_id = dict ( [ ( user['user_id'], user) for user in users ] ) # Get all NITOS public keys # accumulate key ids for keys retrieval # key_ids = [] # for person in persons: # key_ids.extend(person['key_ids']) # keys = shell.GetKeys( {'peer_id': None, 'key_id': key_ids, # 'key_type': 'ssh'} ) # # create a hash of keys by key_id # keys_by_id = dict ( [ ( key['key_id'], key ) for key in keys ] ) # create a dict user_id -> [ (nitos)keys ] keys_by_user_id = dict ( [ ( user['user_id'], user['keys']) for user in users ] ) # Get all nitos nodes nodes = shell.getNodes({}, []) # create hash by node_id nodes_by_id = dict ( [ (node['node_id'], node) for node in nodes ] ) # Get all nitos slices slices = shell.getSlices({}, []) # create hash by slice_id slices_by_id = dict ( [ (slice['slice_id'], slice) for slice in slices ] ) # start importing for site in sites: #for i in [0]: site_hrn = _get_site_hrn(interface_hrn, site) # import if hrn is not in list of existing hrns or if the hrn exists # but its not a site record site_record=self.locate_by_type_hrn ('authority', site_hrn) if not site_record: try: urn = hrn_to_urn(site_hrn, 'authority') if not self.auth_hierarchy.auth_exists(urn): self.auth_hierarchy.create_auth(urn) auth_info = self.auth_hierarchy.get_auth_info(urn) site_record = RegAuthority(hrn=site_hrn, gid=auth_info.get_gid_object(), pointer=0, authority=get_authority(site_hrn)) site_record.just_created() global_dbsession.add(site_record) global_dbsession.commit() self.logger.info("NitosImporter: imported authority (site) : %s" % site_record) self.remember_record (site_record) except: # if the site import fails then there is no point in trying to import the # site's child records (node, slices, persons), so skip them. self.logger.log_exc("NitosImporter: failed to import site. Skipping child records") continue else: # xxx update the record ... pass site_record.stale=False # import node records for node in nodes: site_auth = get_authority(site_hrn) site_name = site['name'] node_hrn = hostname_to_hrn(site_auth, site_name, node['hostname']) # xxx this sounds suspicious if len(node_hrn) > 64: node_hrn = node_hrn[:64] node_record = self.locate_by_type_hrn ( 'node', node_hrn ) if not node_record: try: pkey = Keypair(create=True) urn = hrn_to_urn(node_hrn, 'node') node_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey) node_record = RegNode (hrn=node_hrn, gid=node_gid, pointer =node['node_id'], authority=get_authority(node_hrn)) node_record.just_created() global_dbsession.add(node_record) global_dbsession.commit() self.logger.info("NitosImporter: imported node: %s" % node_record) self.remember_record (node_record) except: self.logger.log_exc("NitosImporter: failed to import node") else: # xxx update the record ... pass node_record.stale=False # import users for user in users: user_hrn = username_to_hrn(interface_hrn, site['name'], user['username']) # xxx suspicious again if len(user_hrn) > 64: user_hrn = user_hrn[:64] user_urn = hrn_to_urn(user_hrn, 'user') user_record = self.locate_by_type_hrn ( 'user', user_hrn) # return a tuple pubkey (a nitos key object) and pkey (a Keypair object) def init_user_key (user): pubkey = None pkey = None if user['keys']: # randomly pick first key in set for key in user['keys']: pubkey = key try: pkey = convert_public_key(pubkey) break except: continue if not pkey: self.logger.warn('NitosImporter: unable to convert public key for %s' % user_hrn) pkey = Keypair(create=True) else: # the user has no keys. Creating a random keypair for the user's gid self.logger.warn("NitosImporter: user %s does not have a NITOS public key"%user_hrn) pkey = Keypair(create=True) return (pubkey, pkey) # new user try: if not user_record: (pubkey,pkey) = init_user_key (user) user_gid = self.auth_hierarchy.create_gid(user_urn, create_uuid(), pkey) user_gid.set_email(user['email']) user_record = RegUser (hrn=user_hrn, gid=user_gid, pointer=user['user_id'], authority=get_authority(user_hrn), email=user['email']) if pubkey: user_record.reg_keys=[RegKey (pubkey)] else: self.logger.warning("No key found for user %s"%user_record) user_record.just_created() global_dbsession.add (user_record) global_dbsession.commit() self.logger.info("NitosImporter: imported user: %s" % user_record) self.remember_record ( user_record ) else: # update the record ? # if user's primary key has changed then we need to update the # users gid by forcing an update here sfa_keys = user_record.reg_keys def sfa_key_in_list (sfa_key,nitos_user_keys): for nitos_key in nitos_user_keys: if nitos_key==sfa_key: return True return False # are all the SFA keys known to nitos ? new_keys=False if not sfa_keys and user['keys']: new_keys = True else: for sfa_key in sfa_keys: if not sfa_key_in_list (sfa_key.key,user['keys']): new_keys = True if new_keys: (pubkey,pkey) = init_user_key (user) user_gid = self.auth_hierarchy.create_gid(user_urn, create_uuid(), pkey) if not pubkey: user_record.reg_keys=[] else: user_record.reg_keys=[ RegKey (pubkey)] user_record.gid = user_gid user_record.just_updated() self.logger.info("NitosImporter: updated user: %s" % user_record) user_record.email = user['email'] global_dbsession.commit() user_record.stale=False except: self.logger.log_exc("NitosImporter: failed to import user %s %s"%(user['user_id'],user['email'])) # import slices for slice in slices: slice_hrn = slicename_to_hrn(interface_hrn, site['name'], slice['slice_name']) slice_record = self.locate_by_type_hrn ('slice', slice_hrn) if not slice_record: try: pkey = Keypair(create=True) urn = hrn_to_urn(slice_hrn, 'slice') slice_gid = self.auth_hierarchy.create_gid(urn, create_uuid(), pkey) slice_record = RegSlice (hrn=slice_hrn, gid=slice_gid, pointer=slice['slice_id'], authority=get_authority(slice_hrn)) slice_record.just_created() global_dbsession.add(slice_record) global_dbsession.commit() self.logger.info("NitosImporter: imported slice: %s" % slice_record) self.remember_record ( slice_record ) except: self.logger.log_exc("NitosImporter: failed to import slice") else: # xxx update the record ... self.logger.warning ("Slice update not yet implemented") pass # record current users affiliated with the slice slice_record.reg_researchers = \ [ self.locate_by_type_pointer ('user',int(user_id)) for user_id in slice['user_ids'] ] global_dbsession.commit() slice_record.stale=False ### remove stale records # special records must be preserved system_hrns = [interface_hrn, root_auth, interface_hrn + '.slicemanager'] for record in all_records: if record.hrn in system_hrns: record.stale=False if record.peer_authority: record.stale=False for record in all_records: try: stale=record.stale except: stale=True self.logger.warning("stale not found with %s"%record) if stale: self.logger.info("NitosImporter: deleting stale record: %s" % record) global_dbsession.delete(record) global_dbsession.commit()
def fill_record_hrns(self, records): """ convert nitos ids to hrns """ # get ids slice_ids, user_ids, node_ids = [], [], [] for record in records: if 'user_ids' in record: user_ids.extend(record['user_ids']) if 'slice_ids' in record: slice_ids.extend(record['slice_ids']) if 'node_ids' in record: node_ids.extend(record['node_ids']) # get nitos records slices, users, nodes = {}, {}, {} if node_ids: all_nodes = self.convert_id(self.shell.getNodes({}, [])) node_list = [ node for node in all_nodes if node['node_id'] in node_ids ] nodes = list_to_dict(node_list, 'node_id') if slice_ids: all_slices = self.convert_id(self.shell.getSlices({}, [])) slice_list = [ slice for slice in all_slices if slice['slice_id'] in slice_ids ] slices = list_to_dict(slice_list, 'slice_id') if user_ids: all_users = self.convert_id(self.shell.getUsers()) user_list = [ user for user in all_users if user['user_id'] in user_ids ] users = list_to_dict(user_list, 'user_id') # convert ids to hrns for record in records: # get all relevant data type = record['type'] pointer = record['pointer'] auth_hrn = self.hrn testbed_name = self.testbedInfo['name'] if pointer == -1: continue if 'user_ids' in record: usernames = [users[user_id]['username'] for user_id in record['user_ids'] \ if user_id in users] user_hrns = [ ".".join([auth_hrn, testbed_name, username]) for username in usernames ] record['users'] = user_hrns if 'slice_ids' in record: slicenames = [slices[slice_id]['slice_name'] for slice_id in record['slice_ids'] \ if slice_id in slices] slice_hrns = [ slicename_to_hrn(auth_hrn, slicename) for slicename in slicenames ] record['slices'] = slice_hrns if 'node_ids' in record: hostnames = [nodes[node_id]['hostname'] for node_id in record['node_ids'] \ if node_id in nodes] node_hrns = [ hostname_to_hrn(auth_hrn, login_base, hostname) for hostname in hostnames ] record['nodes'] = node_hrns if 'expires' in record: date = utcparse(record['expires']) datestring = datetime_to_string(date) record['expires'] = datestring return records