Exemple #1
0
 def delete_record(self, hrn, type):
     # delete the record
     table = SfaTable()
     record_list = table.find({'type': type, 'hrn': hrn})
     for record in record_list:
         self.logger.info("Import: removing record %s %s" % (type, hrn))
         table.remove(record)        
Exemple #2
0
def list(api, xrn, origin_hrn=None):
    hrn, type = urn_to_hrn(xrn)
    # load all know registry names into a prefix tree and attempt to find
    # the longest matching prefix
    records = []
    registries = api.registries
    registry_hrns = registries.keys()
    tree = prefixTree()
    tree.load(registry_hrns)
    registry_hrn = tree.best_match(hrn)
   
    #if there was no match then this record belongs to an unknow registry
    if not registry_hrn:
        raise MissingAuthority(xrn)
    
    # if the best match (longest matching hrn) is not the local registry,
    # forward the request
    records = []    
    if registry_hrn != api.hrn:
        credential = api.getCredential()
        record_list = registries[registry_hrn].List(xrn, credential)
        records = [SfaRecord(dict=record).as_dict() for record in record_list]
    
    # if we still have not found the record yet, try the local registry
    if not records:
        if not api.auth.hierarchy.auth_exists(hrn):
            raise MissingAuthority(hrn)

        table = SfaTable()
        records = table.find({'authority': hrn})

    return records
Exemple #3
0
    def create_top_level_auth_records(self, hrn):
        """
        Create top level records (includes root and sub authorities (local/remote)
        """
        urn = hrn_to_urn(hrn, 'authority')
        # make sure parent exists
        parent_hrn = get_authority(hrn)
        if not parent_hrn:
            parent_hrn = hrn
        if not parent_hrn == hrn:
            self.create_top_level_auth_records(parent_hrn)

        # create the authority if it doesnt already exist 
        if not self.AuthHierarchy.auth_exists(urn):
            self.logger.info("Import: creating top level authorities")
            self.AuthHierarchy.create_auth(urn)
        
        # create the db record if it doesnt already exist    
        auth_info = self.AuthHierarchy.get_auth_info(hrn)
        table = SfaTable()
        auth_record = table.find({'type': 'authority', 'hrn': hrn})

        if not auth_record:
            auth_record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=-1)
            auth_record['authority'] = get_authority(auth_record['hrn'])
            self.logger.info("Import: inserting authority record for %s"%hrn)
            table.insert(auth_record)
Exemple #4
0
    def call(self, cred, record_dict, origin_hrn=None):
        user_cred = Credential(string=cred)

        #log the call
        if not origin_hrn:
            origin_hrn = user_cred.get_gid_caller().get_hrn()    
        self.api.logger.info("interface: %s\tcaller-hrn: %s\ttarget-hrn: %s\tmethod-name: %s"%(self.api.interface, origin_hrn, None, self.name))

        # validate the cred
        self.api.auth.check(cred, "register")

        # make sure this is a peer record
        if 'peer_authority' not in record_dict or \
           not record_dict['peer_authority']: 
            raise SfaInvalidArgument, "peer_authority must be specified" 

        record = SfaRecord(dict = record_dict)
        type, hrn, peer_authority = record['type'], record['hrn'], record['peer_authority']
        record['authority'] = get_authority(record['hrn'])
        # verify permissions
        self.api.auth.verify_cred_is_me(cred)

        # check if record already exists
        table = SfaTable()
        existing_records = table.find({'type': type, 'hrn': hrn, 'peer_authority': peer_authority})
        if existing_records:
            for existing_record in existing_records:
                if existing_record['pointer'] != record['pointer']:
                    record['record_id'] = existing_record['record_id']
                    table.update(record)
        else:
            record_id = table.insert(record)
 
        return 1
Exemple #5
0
def export_gid(options):
    from sfa.util.table import SfaTable
    # lookup the record for the specified hrn 
    hrn = options.export
    type = options.type
    # check sfa table first
    filter = {'hrn': hrn}
    if type:
        filter['type'] = type                    
    table = SfaTable()
    records = table.find(filter)
    if not records:
        # check the authorities hierarchy 
        hierarchy = Hierarchy()
        try:
            auth_info = hierarchy.get_auth_info()
            gid = auth_info.gid_object 
        except:
            print "Record: %s not found" % hrn
            sys.exit(1)
    else:
        record = records[0]
        gid = GID(string=record['gid'])
        
    # get the outfile
    outfile = options.outfile
    if not outfile:
        outfile = os.path.abspath('./%s.gid' % gid.get_hrn())

    # save it
    if options.verbose:
        print "Writing %s gid to %s" % (gid.get_hrn(), outfile)
    gid.save_to_file(outfile, save_parents=True)
Exemple #6
0
    def import_slice(self, parent_hrn, slice):
        slicename = slice['name'].split("_",1)[-1]
        slicename = _cleanup_string(slicename)

        if not slicename:
            self.logger.error("Import: failed to parse slice name %s" %slice['name'])
            return

        hrn = parent_hrn + "." + slicename
        self.logger.info("Import: slice %s"%hrn)

        pkey = Keypair(create=True)
        urn = hrn_to_urn(hrn, 'slice')
        slice_gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
        slice_record = SfaRecord(hrn=hrn, gid=slice_gid, type="slice", pointer=slice['slice_id'])
        slice_record['authority'] = get_authority(slice_record['hrn'])
        table = SfaTable()
        existing_records = table.find({'hrn': hrn, 'type': 'slice', 'pointer': slice['slice_id']})
        if not existing_records:
            table.insert(slice_record)
        else:
            self.logger.info("Import: %s exists, updating " % hrn)
            existing_record = existing_records[0]
            slice_record['record_id'] = existing_record['record_id']
            table.update(slice_record)
Exemple #7
0
    def import_site(self, hrn, site):
        shell = self.shell
        plc_auth = self.plc_auth
        urn = hrn_to_urn(hrn, 'authority')
        self.logger.info("Import: site %s"%hrn)

        # create the authority
        if not self.AuthHierarchy.auth_exists(urn):
            self.AuthHierarchy.create_auth(urn)

        auth_info = self.AuthHierarchy.get_auth_info(urn)

        table = SfaTable()
        auth_record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="authority", pointer=site['site_id'])
        auth_record['authority'] = get_authority(auth_record['hrn'])
        existing_records = table.find({'hrn': hrn, 'type': 'authority', 'pointer': site['site_id']})
        if not existing_records:
            table.insert(auth_record)
        else:
            self.logger.info("Import: %s exists, updating " % hrn)
            existing_record = existing_records[0]
            auth_record['record_id'] = existing_record['record_id']
            table.update(auth_record)

        return hrn
def main():
    config = Config()
    if not config.SFA_REGISTRY_ENABLED:
        sys.exit(0)

    # Get the path to the sfa server key/cert files from 
    # the sfa hierarchy object
    sfa_hierarchy = Hierarchy()
    sfa_key_path = sfa_hierarchy.basedir
    key_file = os.path.join(sfa_key_path, "server.key")
    cert_file = os.path.join(sfa_key_path, "server.cert")
    key = Keypair(filename=key_file) 

    # get a connection to our local sfa registry
    # and a valid credential
    authority = config.SFA_INTERFACE_HRN
    url = 'http://%s:%s/' %(config.SFA_REGISTRY_HOST, config.SFA_REGISTRY_PORT)
    registry = xmlrpcprotocol.get_server(url, key_file, cert_file)
    sfa_api = SfaAPI(key_file = key_file, cert_file = cert_file, interface='registry')
    credential = sfa_api.getCredential()

    # get peer registries
    registries = Registries(sfa_api)
    tree = prefixTree()
    tree.load(registries.keys())
    
    # get local peer records
    table = SfaTable()
    peer_records = table.find({'~peer_authority': None})
    found_records = []
    hrn_dict = {}
    for record in peer_records:
        registry_hrn = tree.best_match(record['hrn'])
        if registry_hrn not in hrn_dict:
            hrn_dict[registry_hrn] = []
        hrn_dict[registry_hrn].append(record['hrn'])

    # attempt to resolve the record at the authoritative interface 
    for registry_hrn in hrn_dict:
        if registry_hrn in registries:
            records = []
            target_hrns = hrn_dict[registry_hrn]    
            try:
                records = registries[registry_hrn].Resolve(target_hrns, credential)
                found_records.extend([record['hrn'] for record in records])
            except ServerException:
                # an exception will be thrown if the record doenst exist
                # if so remove the record from the local registry
                continue
            except:
                # this deosnt necessarily mean the records dont exist
                # lets give them the benefit of the doubt here (for now)
                found_records.extend(target_hrns)
                traceback.print_exc()

    # remove what wasnt found 
    for peer_record in peer_records:
        if peer_record['hrn'] not in found_records:
            registries[sfa_api.hrn].Remove(peer_record['hrn'], credential, peer_record['type'])
Exemple #9
0
    def import_node(self, hrn, node):
        self.logger.info("Import: node %s" % hrn)
        # ASN.1 will have problems with hrn's longer than 64 characters
        if len(hrn) > 64:
            hrn = hrn[:64]

        table = SfaTable()
        node_record = table.find({'type': 'node', 'hrn': hrn})
        pkey = Keypair(create=True)
        urn = hrn_to_urn(hrn, 'node')
        node_gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
        node_record = SfaRecord(hrn=hrn, gid=node_gid, type="node", pointer=node['node_id'])
        node_record['authority'] = get_authority(node_record['hrn'])
        existing_records = table.find({'hrn': hrn, 'type': 'node', 'pointer': node['node_id']})
        if not existing_records:
            table.insert(node_record)
        else:
            self.logger.info("Import: %s exists, updating " % hrn)
            existing_record = existing_records[0]
            node_record['record_id'] = existing_record['record_id']
            table.update(node_record)
Exemple #10
0
def update_cert_records(gids):
    """
    Make sure there is a record in the registry for the specified gids. 
    Removes old records from the db.
    """
    # import SfaTable here so this module can be loaded by ComponentAPI
    from sfa.util.table import SfaTable
    from sfa.util.record import SfaRecord

    if not gids:
        return
    table = SfaTable()
    # get records that actually exist in the db
    gid_urns = [gid.get_urn() for gid in gids]
    hrns_expected = [gid.get_hrn() for gid in gids]
    records_found = table.find({"hrn": hrns_expected, "pointer": -1})

    # remove old records
    for record in records_found:
        if record["hrn"] not in hrns_expected and record["hrn"] != self.api.config.SFA_INTERFACE_HRN:
            table.remove(record)

    # TODO: store urn in the db so we do this in 1 query
    for gid in gids:
        hrn, type = gid.get_hrn(), gid.get_type()
        record = table.find({"hrn": hrn, "type": type, "pointer": -1})
        if not record:
            record = {
                "hrn": hrn,
                "type": type,
                "pointer": -1,
                "authority": get_authority(hrn),
                "gid": gid.save_to_string(save_parents=True),
            }
            record = SfaRecord(dict=record)
            table.insert(record)
Exemple #11
0
def remove(api, xrn, origin_hrn=None):

    table = SfaTable()
    filter = {'hrn': xrn.get_hrn()}
    hrn=xrn.get_hrn()
    type=xrn.get_type()
    if type and type not in ['all', '*']:
        filter['type'] = type

    records = table.find(filter)
    if not records: raise RecordNotFound(hrn)
    record = records[0]
    type = record['type']

    credential = api.getCredential()
    registries = api.registries

    # Try to remove the object from the PLCDB of federated agg.
    # This is attempted before removing the object from the local agg's PLCDB and sfa table
    if hrn.startswith(api.hrn) and type in ['user', 'slice', 'authority']:
        for registry in registries:
            if registry not in [api.hrn]:
                try:
                    result=registries[registry].remove_peer_object(credential, record, origin_hrn)
                except:
                    pass
    if type == "user":
        persons = api.plshell.GetPersons(api.plauth, record['pointer'])
        # only delete this person if he has site ids. if he doesnt, it probably means
        # he was just removed from a site, not actually deleted
        if persons and persons[0]['site_ids']:
            api.plshell.DeletePerson(api.plauth, record['pointer'])
    elif type == "slice":
        if api.plshell.GetSlices(api.plauth, record['pointer']):
            api.plshell.DeleteSlice(api.plauth, record['pointer'])
    elif type == "node":
        if api.plshell.GetNodes(api.plauth, record['pointer']):
            api.plshell.DeleteNode(api.plauth, record['pointer'])
    elif type == "authority":
        if api.plshell.GetSites(api.plauth, record['pointer']):
            api.plshell.DeleteSite(api.plauth, record['pointer'])
    else:
        raise UnknownSfaType(type)

    table.remove(record)

    return 1
Exemple #12
0
    def create_sm_client_record(self):
        """
        Create a user record for the Slicemanager service.
        """
        hrn = self.config.SFA_INTERFACE_HRN + '.slicemanager'
        urn = hrn_to_urn(hrn, 'user')
        if not self.AuthHierarchy.auth_exists(urn):
            self.logger.info("Import: creating Slice Manager user")
            self.AuthHierarchy.create_auth(urn)

        auth_info = self.AuthHierarchy.get_auth_info(hrn)
        table = SfaTable()
        sm_user_record = table.find({'type': 'user', 'hrn': hrn})
        if not sm_user_record:
            record = SfaRecord(hrn=hrn, gid=auth_info.get_gid_object(), type="user", pointer=-1)
            record['authority'] = get_authority(record['hrn'])
            table.insert(record)    
Exemple #13
0
    def import_person(self, parent_hrn, person):
        """
        Register a user record 
        """
        hrn = email_to_hrn(parent_hrn, person['email'])

        # ASN.1 will have problems with hrn's longer than 64 characters
        if len(hrn) > 64:
            hrn = hrn[:64]

        self.logger.info("Import: person %s"%hrn)
        key_ids = []
        if 'key_ids' in person and person['key_ids']:
            key_ids = person["key_ids"]
            # get the user's private key from the SSH keys they have uploaded
            # to planetlab
            keys = self.shell.GetKeys(self.plc_auth, key_ids)
            key = keys[0]['key']
            pkey = None
            try:
                pkey = convert_public_key(key)
            except:
                self.logger.warn('unable to convert public key for %s' % hrn) 
            if not pkey:
                pkey = Keypair(create=True)
        else:
            # the user has no keys
            self.logger.warn("Import: person %s does not have a PL public key"%hrn)
            # if a key is unavailable, then we still need to put something in the
            # user's GID. So make one up.
            pkey = Keypair(create=True)

        # create the gid
        urn = hrn_to_urn(hrn, 'user')
        person_gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
        table = SfaTable()
        person_record = SfaRecord(hrn=hrn, gid=person_gid, type="user", pointer=person['person_id'])
        person_record['authority'] = get_authority(person_record['hrn'])
        existing_records = table.find({'hrn': hrn, 'type': 'user', 'pointer': person['person_id']})
        if not existing_records:
            table.insert(person_record)
        else:
            self.logger.info("Import: %s exists, updating " % hrn)
            existing_record = existing_records[0]
            person_record['record_id'] = existing_record['record_id']
            table.update(person_record)
Exemple #14
0
def import_gid(options):
    """
    Import the specified gid into the registry (db and authorities 
    hierarchy) overwriting any previous gid.
    """
    from sfa.util.table import SfaTable
    from sfa.util.record import SfaRecord
    # load the gid
    gidfile = os.path.abspath(options.importgid)
    if not gidfile or not os.path.isfile(gidfile):
        print "No such gid: %s" % gidfile
        sys.exit(1)
    gid = GID(filename=gidfile)
    
    # check if it exists within the hierarchy
    hierarchy = Hierarchy()
    if not hierarchy.auth_exists(gid.get_hrn()):
        print "%s not found in hierarchy" % gid.get_hrn()
        sys.exit(1)

    # check if record exists in db
    table = SfaTable()
    records = table.find({'hrn': gid.get_hrn(), 'type': 'authority'})
    if not records:
        print "%s not found in record database" % get.get_hrn()  
        sys.exit(1)

    # update the database record
    record = records[0]
    record['gid'] = gid.save_to_string(save_parents=True)
    table.update(record)
    if options.verbose:
        print "Imported %s gid into db" % record['hrn']

    # update the hierarchy
    auth_info = hierarchy.get_auth_info(gid.get_hrn())  
    filename = auth_info.gid_filename
    gid.save_to_file(filename, save_parents=True)
    if options.verbose:
        print "Writing %s gid to %s" % (gid.get_hrn(), filename)

    # ending here
    return
Exemple #15
0
 def create_interface_records(self):
     """
     Create a record for each SFA interface
     """
     # just create certs for all sfa interfaces even if they
     # arent enabled
     interface_hrn = self.config.SFA_INTERFACE_HRN
     interfaces = ['authority+sa', 'authority+am', 'authority+sm']
     table = SfaTable()
     auth_info = self.AuthHierarchy.get_auth_info(interface_hrn)
     pkey = auth_info.get_pkey_object()
     for interface in interfaces:
         interface_record = table.find({'type': interface, 'hrn': interface_hrn})
         if not interface_record:
             self.logger.info("Import: interface %s %s " % (interface_hrn, interface))
             urn = hrn_to_urn(interface_hrn, interface)
             gid = self.AuthHierarchy.create_gid(urn, create_uuid(), pkey)
             record = SfaRecord(hrn=interface_hrn, gid=gid, type=interface, pointer=-1)  
             record['authority'] = get_authority(interface_hrn)
             table.insert(record) 
Exemple #16
0
    def call(self, cred, record, origin_hrn=None):
        user_cred = Credential(string=cred)

        #log the call
        if not origin_hrn:
            origin_hrn = user_cred.get_gid_caller().get_hrn()
        self.api.logger.info("interface: %s\tcaller-hrn: %s\ttarget-hrn: %s\tmethod-name: %s"%(self.api.interface, origin_hrn, record['hrn'], self.name))

        self.api.auth.check(cred, "remove")

        # Only allow the local interface or record owner to delete peer_records 
        try: self.api.auth.verify_object_permission(record['hrn'])
        except: self.api.auth.verify_cred_is_me(cred)
        
        table = SfaTable()
        hrn, type = record['hrn'], record['type']
        records = table.find({'hrn': hrn, 'type': type })
        for record in records:
            if record['peer_authority']:
                self.remove_plc_record(record)
                table.remove(record)
            
        return 1
Exemple #17
0
def register(api, record):

    hrn, type = record['hrn'], record['type']
    urn = hrn_to_urn(hrn,type)
    # validate the type
    if type not in ['authority', 'slice', 'node', 'user']:
        raise UnknownSfaType(type) 
    
    # check if record already exists
    table = SfaTable()
    existing_records = table.find({'type': type, 'hrn': hrn})
    if existing_records:
        raise ExistingRecord(hrn)
       
    record = SfaRecord(dict = record)
    record['authority'] = get_authority(record['hrn'])
    type = record['type']
    hrn = record['hrn']
    auth_info = api.auth.get_auth_info(record['authority'])
    pub_key = None
    # make sure record has a gid
    if 'gid' not in record:
        uuid = create_uuid()
        pkey = Keypair(create=True)
        if 'key' in record and record['key']:
            if isinstance(record['key'], types.ListType):
                pub_key = record['key'][0]
            else:
                pub_key = record['key']
            pkey = convert_public_key(pub_key)

        gid_object = api.auth.hierarchy.create_gid(urn, uuid, pkey)
        gid = gid_object.save_to_string(save_parents=True)
        record['gid'] = gid
        record.set_gid(gid)

    if type in ["authority"]:
        # update the tree
        if not api.auth.hierarchy.auth_exists(hrn):
            api.auth.hierarchy.create_auth(hrn_to_urn(hrn,'authority'))

        # get the GID from the newly created authority
        gid = auth_info.get_gid_object()
        record.set_gid(gid.save_to_string(save_parents=True))
        pl_record = api.sfa_fields_to_pl_fields(type, hrn, record)
        sites = api.plshell.GetSites(api.plauth, [pl_record['login_base']])
        if not sites:
            pointer = api.plshell.AddSite(api.plauth, pl_record)
        else:
            pointer = sites[0]['site_id']

        record.set_pointer(pointer)
        record['pointer'] = pointer

    elif (type == "slice"):
        acceptable_fields=['url', 'instantiation', 'name', 'description']
        pl_record = api.sfa_fields_to_pl_fields(type, hrn, record)
        for key in pl_record.keys():
            if key not in acceptable_fields:
                pl_record.pop(key)
        slices = api.plshell.GetSlices(api.plauth, [pl_record['name']])
        if not slices:
             pointer = api.plshell.AddSlice(api.plauth, pl_record)
        else:
             pointer = slices[0]['slice_id']
        record.set_pointer(pointer)
        record['pointer'] = pointer

    elif  (type == "user"):
        persons = api.plshell.GetPersons(api.plauth, [record['email']])
        if not persons:
            pointer = api.plshell.AddPerson(api.plauth, dict(record))
        else:
            pointer = persons[0]['person_id']

        if 'enabled' in record and record['enabled']:
            api.plshell.UpdatePerson(api.plauth, pointer, {'enabled': record['enabled']})
        # add this persons to the site only if he is being added for the first
        # time by sfa and doesont already exist in plc
        if not persons or not persons[0]['site_ids']:
            login_base = get_leaf(record['authority'])
            api.plshell.AddPersonToSite(api.plauth, pointer, login_base)

        # What roles should this user have?
        api.plshell.AddRoleToPerson(api.plauth, 'user', pointer)
        # Add the user's key
        if pub_key:
            api.plshell.AddPersonKey(api.plauth, pointer, {'key_type' : 'ssh', 'key' : pub_key})

    elif (type == "node"):
        pl_record = api.sfa_fields_to_pl_fields(type, hrn, record)
        login_base = hrn_to_pl_login_base(record['authority'])
        nodes = api.plshell.GetNodes(api.plauth, [pl_record['hostname']])
        if not nodes:
            pointer = api.plshell.AddNode(api.plauth, login_base, pl_record)
        else:
            pointer = nodes[0]['node_id']

    record['pointer'] = pointer
    record.set_pointer(pointer)
    record_id = table.insert(record)
    record['record_id'] = record_id

    # update membership for researchers, pis, owners, operators
    api.update_membership(None, record)

    return record.get_gid_object().save_to_string(save_parents=True)
Exemple #18
0
def main():

    process_options()
    config = Config()
    if not config.SFA_REGISTRY_ENABLED:
        sys.exit(0)
    root_auth = config.SFA_REGISTRY_ROOT_AUTH
    interface_hrn = config.SFA_INTERFACE_HRN
    keys_filename = config.config_path + os.sep + 'person_keys.py' 
    sfaImporter = sfaImport()
    if config.SFA_API_DEBUG: sfaImporter.logger.setLevelDebug()
    shell = sfaImporter.shell
    plc_auth = sfaImporter.plc_auth 
    
    # initialize registry db table
    table = SfaTable()
    if not table.exists():
       table.create()

    # create root authority 
    sfaImporter.create_top_level_auth_records(root_auth)
    if not root_auth == interface_hrn:
        sfaImporter.create_top_level_auth_records(interface_hrn)

    # create s user record for the slice manager
    sfaImporter.create_sm_client_record()

    # create interface records
    sfaImporter.logger.info("Import: creating interface records")
    sfaImporter.create_interface_records()

    # add local root authority's cert  to trusted list
    sfaImporter.logger.info("Import: adding " + interface_hrn + " to trusted list")
    authority = sfaImporter.AuthHierarchy.get_auth_info(interface_hrn)
    sfaImporter.TrustedRoots.add_gid(authority.get_gid_object())

    # special case for vini
    if ".vini" in interface_hrn and interface_hrn.endswith('vini'):
        # create a fake internet2 site first
        i2site = {'name': 'Internet2', 'abbreviated_name': 'I2',
                    'login_base': 'internet2', 'site_id': -1}
        sfaImporter.import_site(interface_hrn, i2site)
   
    # create dict of all existing sfa records
    existing_records = {}
    existing_hrns = []
    key_ids = []
    person_keys = {} 
    results = table.find()
    for result in results:
        existing_records[(result['hrn'], result['type'])] = result
        existing_hrns.append(result['hrn']) 
            
    # Get all plc sites
    sites = shell.GetSites(plc_auth, {'peer_id': None})
    sites_dict = {}
    for site in sites:
        sites_dict[site['login_base']] = site 
    
    # Get all plc users
    persons = shell.GetPersons(plc_auth, {'peer_id': None, 'enabled': True}, ['person_id', 'email', 'key_ids', 'site_ids'])
    persons_dict = {}
    for person in persons:
        persons_dict[person['person_id']] = person
        key_ids.extend(person['key_ids'])

    # Get all public keys
    keys = shell.GetKeys(plc_auth, {'peer_id': None, 'key_id': key_ids})
    keys_dict = {}
    for key in keys:
        keys_dict[key['key_id']] = key['key']

    # create a dict of person keys keyed on key_id 
    old_person_keys = load_keys(keys_filename)
    for person in persons:
        pubkeys = []
        for key_id in person['key_ids']:
            pubkeys.append(keys_dict[key_id])
        person_keys[person['person_id']] = pubkeys

    # Get all plc nodes  
    nodes = shell.GetNodes(plc_auth, {'peer_id': None}, ['node_id', 'hostname', 'site_id'])
    nodes_dict = {}
    for node in nodes:
        nodes_dict[node['node_id']] = node

    # Get all plc slices
    slices = shell.GetSlices(plc_auth, {'peer_id': None}, ['slice_id', 'name'])
    slices_dict = {}
    for slice in slices:
        slices_dict[slice['slice_id']] = slice
    # start importing 
    for site in sites:
        site_hrn = _get_site_hrn(interface_hrn, site)
        sfaImporter.logger.info("Importing site: %s" % site_hrn)

        # import if hrn is not in list of existing hrns or if the hrn exists
        # but its not a site record
        if site_hrn not in existing_hrns or \
           (site_hrn, 'authority') not in existing_records:
            sfaImporter.import_site(site_hrn, site)
             
        # import node records
        for node_id in site['node_ids']:
            if node_id not in nodes_dict:
                continue 
            node = nodes_dict[node_id]
            site_auth = get_authority(site_hrn)
            site_name = get_leaf(site_hrn)
            hrn =  hostname_to_hrn(site_auth, site_name, node['hostname'])
            if hrn not in existing_hrns or \
               (hrn, 'node') not in existing_records:
                sfaImporter.import_node(hrn, node)

        # import slices
        for slice_id in site['slice_ids']:
            if slice_id not in slices_dict:
                continue 
            slice = slices_dict[slice_id]
            hrn = slicename_to_hrn(interface_hrn, slice['name'])
            if hrn not in existing_hrns or \
               (hrn, 'slice') not in existing_records:
                sfaImporter.import_slice(site_hrn, slice)      

        # import persons
        for person_id in site['person_ids']:
            if person_id not in persons_dict:
                continue 
            person = persons_dict[person_id]
            hrn = email_to_hrn(site_hrn, person['email'])
            old_keys = []
            new_keys = []
            if person_id in old_person_keys:
                old_keys = old_person_keys[person_id]
            if person_id in person_keys:
                new_keys = person_keys[person_id]
            update_record = False
            for key in new_keys:
                if key not in old_keys:
                    update_record = True 

            if hrn not in existing_hrns or \
               (hrn, 'user') not in existing_records or update_record:
                sfaImporter.import_person(site_hrn, person)

    
    # remove stale records    
    system_records = [interface_hrn, root_auth, interface_hrn + '.slicemanager']
    for (record_hrn, type) in existing_records.keys():
        if record_hrn in system_records:
            continue
        
        record = existing_records[(record_hrn, type)]
        if record['peer_authority']:
            continue

        # dont delete vini's internet2 placeholdder record
        # normally this would be deleted becuase it does not have a plc record 
        if ".vini" in interface_hrn and interface_hrn.endswith('vini') and \
           record_hrn.endswith("internet2"):     
            continue

        found = False
        
        if type == 'authority':    
            for site in sites:
                site_hrn = interface_hrn + "." + site['login_base']
                if site_hrn == record_hrn and site['site_id'] == record['pointer']:
                    found = True
                    break

        elif type == 'user':
            login_base = get_leaf(get_authority(record_hrn))
            username = get_leaf(record_hrn)
            if login_base in sites_dict:
                site = sites_dict[login_base]
                for person in persons:
                    tmp_username = person['email'].split("@")[0]
                    alt_username = person['email'].split("@")[0].replace(".", "_").replace("+", "_")
                    if username in [tmp_username, alt_username] and \
                       site['site_id'] in person['site_ids'] and \
                       person['person_id'] == record['pointer']:
                        found = True
                        break
        
        elif type == 'slice':
            slicename = hrn_to_pl_slicename(record_hrn)
            for slice in slices:
                if slicename == slice['name'] and \
                   slice['slice_id'] == record['pointer']:
                    found = True
                    break    
 
        elif type == 'node':
            login_base = get_leaf(get_authority(record_hrn))
            nodename = Xrn.unescape(get_leaf(record_hrn))
            if login_base in sites_dict:
                site = sites_dict[login_base]
                for node in nodes:
                    tmp_nodename = node['hostname']
                    if tmp_nodename == nodename and \
                       node['site_id'] == site['site_id'] and \
                       node['node_id'] == record['pointer']:
                        found = True
                        break  
        else:
            continue 
        
        if not found:
            record_object = existing_records[(record_hrn, type)]
            sfaImporter.delete_record(record_hrn, type) 
                                   
    # save pub keys
    sfaImporter.logger.info('Import: saving current pub keys')
    save_keys(keys_filename, person_keys)