Example #1
0
def utcparse(input):
    """ Translate a string into a time using dateutil.parser.parse but make sure it's in UTC time and strip
the timezone, so that it's compatible with normal datetime.datetime objects.

For safety this can also handle inputs that are either timestamps, or datetimes
"""
    # prepare the input for the checks below by
    # casting strings ('1327098335') to ints
    if isinstance(input, StringTypes):
        try:
            input = int(input)
        except ValueError:
            pass

    if isinstance (input, datetime.datetime):
        logger.warn ("argument to utcparse already a datetime - doing nothing")
        return input
    elif isinstance (input, StringTypes):
        t = dateutil.parser.parse(input)
        if t.utcoffset() is not None:
            t = t.utcoffset() + t.replace(tzinfo=None)
        return t
    elif isinstance (input, (int,float,long)):
        return datetime.datetime.fromtimestamp(input)
    else:
        logger.error("Unexpected type in utcparse [%s]"%type(input))
Example #2
0
def utcparse(input):
    """ Translate a string into a time using dateutil.parser.parse but make sure it's in UTC time and strip
the timezone, so that it's compatible with normal datetime.datetime objects.

For safety this can also handle inputs that are either timestamps, or datetimes
"""
    # prepare the input for the checks below by
    # casting strings ('1327098335') to ints
    if isinstance(input, StringTypes):
        try:
            input = int(input)
        except ValueError:
            pass

    if isinstance(input, datetime.datetime):
        logger.warn("argument to utcparse already a datetime - doing nothing")
        return input
    elif isinstance(input, StringTypes):
        t = dateutil.parser.parse(input)
        if t.utcoffset() is not None:
            t = t.utcoffset() + t.replace(tzinfo=None)
        return t
    elif isinstance(input, (int, float, long)):
        return datetime.datetime.fromtimestamp(input)
    else:
        logger.error("Unexpected type in utcparse [%s]" % type(input))
Example #3
0
    def add_slicemgr_stat(self, rspec, callname, aggname, elapsed, status, exc_info=None):
        try:
            stats_tags = rspec.xml.xpath('//statistics[@call="%s"]' % callname)
            if stats_tags:
                stats_tag = stats_tags[0]
            else:
                stats_tag = rspec.xml.root.add_element("statistics", call=callname)

            stat_tag = stats_tag.add_element("aggregate", name=str(aggname), 
                                             elapsed=str(elapsed), status=str(status))

            if exc_info:
                exc_tag = stat_tag.add_element("exc_info", name=str(exc_info[1]))

                # formats the traceback as one big text blob
                #exc_tag.text = "\n".join(traceback.format_exception(exc_info[0], exc_info[1], exc_info[2]))

                # formats the traceback as a set of xml elements
                tb = traceback.extract_tb(exc_info[2])
                for item in tb:
                    exc_frame = exc_tag.add_element("tb_frame", filename=str(item[0]), 
                                                    line=str(item[1]), func=str(item[2]), code=str(item[3]))

        except Exception, e:
            logger.warn("add_slicemgr_stat failed on  %s: %s" %(aggname, str(e)))
Example #4
0
 def drop_slicemgr_stats(self, rspec):
     try:
         stats_elements = rspec.xml.xpath('//statistics')
         for node in stats_elements:
             node.getparent().remove(node)
     except Exception, e:
         logger.warn("drop_slicemgr_stats failed: %s " % (str(e)))
Example #5
0
    def verify_slice_attributes(self, slice, requested_slice_attributes, options=None, admin=False):
        if options is None: options={}
        append = options.get('append', True)
        # get list of attributes users ar able to manage
        filter = {'category': '*slice*'}
        if not admin:
            filter['|roles'] = ['user']
        slice_attributes = self.driver.shell.GetTagTypes(filter)
        valid_slice_attribute_names = [attribute['tagname'] for attribute in slice_attributes]

        # get sliver attributes
        added_slice_attributes = []
        removed_slice_attributes = []
        # we need to keep the slice hrn anyway
        ignored_slice_attribute_names = ['hrn']
        existing_slice_attributes = self.driver.shell.GetSliceTags({'slice_id': slice['slice_id']})

        # get attributes that should be removed
        for slice_tag in existing_slice_attributes:
            if slice_tag['tagname'] in ignored_slice_attribute_names:
                # If a slice already has a admin only role it was probably given to them by an
                # admin, so we should ignore it.
                ignored_slice_attribute_names.append(slice_tag['tagname'])
                attribute_found=True
            else:
                # If an existing slice attribute was not found in the request it should
                # be removed
                attribute_found=False
                for requested_attribute in requested_slice_attributes:
                    if requested_attribute['name'] == slice_tag['tagname'] and \
                       requested_attribute['value'] == slice_tag['value']:
                        attribute_found=True
                        break

            if not attribute_found and not append:
                removed_slice_attributes.append(slice_tag)

        # get attributes that should be added:
        for requested_attribute in requested_slice_attributes:
            # if the requested attribute wasn't found  we should add it
            if requested_attribute['name'] in valid_slice_attribute_names:
                attribute_found = False
                for existing_attribute in existing_slice_attributes:
                    if requested_attribute['name'] == existing_attribute['tagname'] and \
                       requested_attribute['value'] == existing_attribute['value']:
                        attribute_found=True
                        break
                if not attribute_found:
                    added_slice_attributes.append(requested_attribute)


        # remove stale attributes
        for attribute in removed_slice_attributes:
            try:
                self.driver.shell.DeleteSliceTag(attribute['slice_tag_id'])
            except Exception, e:
                logger.warn('Failed to remove sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
                                % (slice['name'], attribute['value'],  attribute.get('node_id'), str(e)))
Example #6
0
    def verify_chain(self, trusted_certs = None):
        # Verify a chain of certificates. Each certificate must be signed by
        # the public key contained in it's parent. The chain is recursed
        # until a certificate is found that is signed by a trusted root.

        # verify expiration time
        if self.cert.has_expired():
            logger.debug("verify_chain: NO, Certificate %s has expired" % self.get_printable_subject())
            raise CertExpired(self.get_printable_subject(), "client cert")

        # if this cert is signed by a trusted_cert, then we are set
        for trusted_cert in trusted_certs:
            if self.is_signed_by_cert(trusted_cert):
                # verify expiration of trusted_cert ?
                if not trusted_cert.cert.has_expired():
                    logger.debug("verify_chain: YES. Cert %s signed by trusted cert %s"%(
                            self.get_printable_subject(), trusted_cert.get_printable_subject()))
                    return trusted_cert
                else:
                    logger.debug("verify_chain: NO. Cert %s is signed by trusted_cert %s, but that signer is expired..."%(
                            self.get_printable_subject(),trusted_cert.get_printable_subject()))
                    raise CertExpired(self.get_printable_subject()," signer trusted_cert %s"%trusted_cert.get_printable_subject())

        # if there is no parent, then no way to verify the chain
        if not self.parent:
            logger.debug("verify_chain: NO. %s has no parent and issuer %s is not in %d trusted roots"%(self.get_printable_subject(), self.get_issuer(), len(trusted_certs)))
            raise CertMissingParent(self.get_printable_subject() + ": Issuer %s is not one of the %d trusted roots, and cert has no parent." % (self.get_issuer(), len(trusted_certs)))

        # if it wasn't signed by the parent...
        if not self.is_signed_by_cert(self.parent):
            logger.debug("verify_chain: NO. %s is not signed by parent %s, but by %s"%\
                             (self.get_printable_subject(), 
                              self.parent.get_printable_subject(), 
                              self.get_issuer()))
            raise CertNotSignedByParent("%s: Parent %s, issuer %s"\
                                            % (self.get_printable_subject(), 
                                               self.parent.get_printable_subject(),
                                               self.get_issuer()))

        # Confirm that the parent is a CA. Only CAs can be trusted as
        # signers.
        # Note that trusted roots are not parents, so don't need to be
        # CAs.
        # Ugly - cert objects aren't parsed so we need to read the
        # extension and hope there are no other basicConstraints
        if not self.parent.isCA and not (self.parent.get_extension('basicConstraints') == 'CA:TRUE'):
            logger.warn("verify_chain: cert %s's parent %s is not a CA" % \
                            (self.get_printable_subject(), self.parent.get_printable_subject()))
            raise CertNotSignedByParent("%s: Parent %s not a CA" % (self.get_printable_subject(),
                                                                    self.parent.get_printable_subject()))

        # if the parent isn't verified...
        logger.debug("verify_chain: .. %s, -> verifying parent %s"%\
                         (self.get_printable_subject(),self.parent.get_printable_subject()))
        self.parent.verify_chain(trusted_certs)

        return
Example #7
0
    def sign(self):
        if not self.issuer_privkey:
            logger.warn("Cannot sign credential (no private key)")
            return
        if not self.issuer_gid:
            logger.warn("Cannot sign credential (no issuer gid)")
            return
        doc = parseString(self.get_xml())
        sigs = doc.getElementsByTagName("signatures")[0]

        # Create the signature template to be signed
        signature = Signature()
        signature.set_refid(self.get_refid())
        sdoc = parseString(signature.get_xml())        
        sig_ele = doc.importNode(sdoc.getElementsByTagName("Signature")[0], True)
        sigs.appendChild(sig_ele)

        self.xml = doc.toxml()


        # Split the issuer GID into multiple certificates if it's a chain
        chain = GID(filename=self.issuer_gid)
        gid_files = []
        while chain:
            gid_files.append(chain.save_to_random_tmp_file(False))
            if chain.get_parent():
                chain = chain.get_parent()
            else:
                chain = None


        # Call out to xmlsec1 to sign it
        ref = 'Sig_%s' % self.get_refid()
        filename = self.save_to_random_tmp_file()
        command='%s --sign --node-id "%s" --privkey-pem %s,%s %s' \
            % (self.xmlsec_path, ref, self.issuer_privkey, ",".join(gid_files), filename)
#        print 'command',command
        signed = os.popen(command).read()
        os.remove(filename)

        for gid_file in gid_files:
            os.remove(gid_file)

        self.xml = signed

        # This is no longer a legacy credential
        if self.legacy:
            self.legacy = None

        # Update signatures
        self.decode()       
Example #8
0
    def remove (self, sfa_record):
        type = sfa_record['type']

        if type == 'user':
            user=None
            hrn = sfa_record.get('hrn')
            try:
                user = self.shell.auth_manager.users.find(name=hrn)
                keyname = OSXrn(xrn=hrn, type='user').get_slicename()
            except(KeystoneExceptions.NotFound):
                print "This user[%s] isn't exist in Openstack" % hrn
                logger.warn("The user[%s] isn't exist in Openstack" % hrn)
            if user:
                self.shell.auth_manager.users.delete(user.id)
                # Update connection for the current client
                auth_name = Xrn(hrn).get_authority_hrn()
                self.shell.compute_manager.connect(username=user.name, tenant=auth_name, password=user.name)
                self.shell.compute_manager.keypairs.delete(key=keyname)
                # Update initial connection info
                self.init_compute_manager_conn()
                logger.info("User[%s] removed from Openstack and SFA registry" % user.name) 
        
        elif type == 'slice':
            hrn = sfa_record.get('hrn')
            slice = self.shell.auth_manager.tenants.find(name=hrn)
            if slice:
                self.shell.auth_manager.tenants.delete(slice.id)
            logger.info("Slice[%s] removed from Openstack and SFA registry" % slice.name) 
        
        elif type == 'authority':   
            hrn = sfa_record.get('hrn')
            auth_tenant = self.shell.auth_manager.tenants.find(name=hrn)
            if auth_tenant:
                self.shell.auth_manager.tenants.delete(auth_tenant.id) 
            logger.info("Authority[%s] removed from Openstack and SFA registry" % auth_tenant.name)

        else:
            raise Exception(type)
        return True
Example #9
0
 def get_instances(self, xrn):
     # parse slice names and sliver ids
     slice_names=[]
     sliver_ids=[]
     instances=[]
     if xrn.type == 'slice':
         slice_names.append(xrn.get_hrn())
     else:
         print "[WARN] We don't know the xrn[%s]" % xrn.type
         logger.warn("[WARN] We don't know the xrn[%s], Check it!" % xrn.type)
         
     # look up instances
     try:
         for slice_name in slice_names:
             servers = self.driver.shell.compute_manager.servers.findall()
             instances.extend(servers)
         for sliver_id in sliver_ids:
             servers = self.driver.shell.compute_manager.servers.findall()
             instances.extend(servers)
     except(exceptions.Unauthorized):
         print "[WARN] The instance(s) in Openstack is/are not permitted."
         logger.warn("The instance(s) in Openstack is/are not permitted.")
     return list( set(instances) )
Example #10
0
    def __init__(self, create=False, subject=None, string=None, filename=None):
        self.gidCaller = None
        self.gidObject = None
        self.expiration = None
        self.privileges = None
        self.issuer_privkey = None
        self.issuer_gid = None
        self.issuer_pubkey = None
        self.parent = None
        self.signature = None
        self.xml = None
        self.refid = None
        self.legacy = None

        # Check if this is a legacy credential, translate it if so
        if string or filename:
            if string:                
                str = string
            elif filename:
                str = file(filename).read()
                
            if str.strip().startswith("-----"):
                self.legacy = CredentialLegacy(False,string=str)
                self.translate_legacy(str)
            else:
                self.xml = str
                self.decode()

        # Find an xmlsec1 path
        self.xmlsec_path = ''
        paths = ['/usr/bin','/usr/local/bin','/bin','/opt/bin','/opt/local/bin']
        for path in paths:
            if os.path.isfile(path + '/' + 'xmlsec1'):
                self.xmlsec_path = path + '/' + 'xmlsec1'
                break
        if not self.xmlsec_path:
            logger.warn("Could not locate binary for xmlsec1 - SFA will be unable to sign stuff !!")
Example #11
0
        # get attributes that should be added:
        for requested_attribute in requested_slice_attributes:
            # if the requested attribute wasn't found  we should add it
            if requested_attribute['name'] in valid_slice_attribute_names:
                attribute_found = False
                for existing_attribute in existing_slice_attributes:
                    if requested_attribute['name'] == existing_attribute['tagname'] and \
                       requested_attribute['value'] == existing_attribute['value']:
                        attribute_found=True
                        break
                if not attribute_found:
                    added_slice_attributes.append(requested_attribute)


        # remove stale attributes
        for attribute in removed_slice_attributes:
            try:
                self.driver.shell.DeleteSliceTag(attribute['slice_tag_id'])
            except Exception, e:
                logger.warn('Failed to remove sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
                                % (slice['name'], attribute['value'],  attribute.get('node_id'), str(e)))

        # add requested_attributes
        for attribute in added_slice_attributes:
            try:
                self.driver.shell.AddSliceTag(slice['name'], attribute['name'], attribute['value'], attribute.get('node_id', None))
            except Exception, e:
                logger.warn('Failed to add sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
                                % (slice['name'], attribute['value'],  attribute.get('node_id'), str(e)))

Example #12
0
    def encode(self):
        # Create the XML document
        doc = Document()
        signed_cred = doc.createElement("signed-credential")

        # Declare namespaces
        # Note that credential/policy.xsd are really the PG schemas
        # in a PL namespace.
        # Note that delegation of credentials between the 2 only really works
        # cause those schemas are identical.
        # Also note these PG schemas talk about PG tickets and CM policies.
        signed_cred.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance")
        signed_cred.setAttribute("xsi:noNamespaceSchemaLocation", "http://www.planet-lab.org/resources/sfa/credential.xsd")
        signed_cred.setAttribute("xsi:schemaLocation", "http://www.planet-lab.org/resources/sfa/ext/policy/1 http://www.planet-lab.org/resources/sfa/ext/policy/1/policy.xsd")

        # PG says for those last 2:
       #signed_cred.setAttribute("xsi:noNamespaceSchemaLocation", "http://www.protogeni.net/resources/credential/credential.xsd")
       # signed_cred.setAttribute("xsi:schemaLocation", "http://www.protogeni.net/resources/credential/ext/policy/1 http://www.protogeni.net/resources/credential/ext/policy/1/policy.xsd")

        doc.appendChild(signed_cred)  
        
        # Fill in the <credential> bit        
        cred = doc.createElement("credential")
        cred.setAttribute("xml:id", self.get_refid())
        signed_cred.appendChild(cred)
        append_sub(doc, cred, "type", "privilege")
        append_sub(doc, cred, "serial", "8")
        append_sub(doc, cred, "owner_gid", self.gidCaller.save_to_string())
        append_sub(doc, cred, "owner_urn", self.gidCaller.get_urn())
        append_sub(doc, cred, "target_gid", self.gidObject.save_to_string())
        append_sub(doc, cred, "target_urn", self.gidObject.get_urn())
        append_sub(doc, cred, "uuid", "")
        if not self.expiration:
            logger.debug("Creating credential valid for %s s"%DEFAULT_CREDENTIAL_LIFETIME)
            self.set_expiration(datetime.datetime.utcnow() + datetime.timedelta(seconds=DEFAULT_CREDENTIAL_LIFETIME))
        self.expiration = self.expiration.replace(microsecond=0)
        append_sub(doc, cred, "expires", self.expiration.strftime(SFATIME_FORMAT))
        privileges = doc.createElement("privileges")
        cred.appendChild(privileges)

        if self.privileges:
            rights = self.get_privileges()
            for right in rights.rights:
                priv = doc.createElement("privilege")
                append_sub(doc, priv, "name", right.kind)
                append_sub(doc, priv, "can_delegate", str(right.delegate).lower())
                privileges.appendChild(priv)

        # Add the parent credential if it exists
        if self.parent:
            sdoc = parseString(self.parent.get_xml())
            # If the root node is a signed-credential (it should be), then
            # get all its attributes and attach those to our signed_cred
            # node.
            # Specifically, PG and PLadd attributes for namespaces (which is reasonable),
            # and we need to include those again here or else their signature
            # no longer matches on the credential.
            # We expect three of these, but here we copy them all:
            #  signed_cred.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance")
            # and from PG (PL is equivalent, as shown above):
            #  signed_cred.setAttribute("xsi:noNamespaceSchemaLocation", "http://www.protogeni.net/resources/credential/credential.xsd")
            #  signed_cred.setAttribute("xsi:schemaLocation", "http://www.protogeni.net/resources/credential/ext/policy/1 http://www.protogeni.net/resources/credential/ext/policy/1/policy.xsd")

            # HOWEVER!
            # PL now also declares these, with different URLs, so
            # the code notices those attributes already existed with
            # different values, and complains.
            # This happens regularly on delegation now that PG and
            # PL both declare the namespace with different URLs.
            # If the content ever differs this is a problem,
            # but for now it works - different URLs (values in the attributes)
            # but the same actual schema, so using the PG schema
            # on delegated-to-PL credentials works fine.

            # Note: you could also not copy attributes
            # which already exist. It appears that both PG and PL
            # will actually validate a slicecred with a parent
            # signed using PG namespaces and a child signed with PL
            # namespaces over the whole thing. But I don't know
            # if that is a bug in xmlsec1, an accident since
            # the contents of the schemas are the same,
            # or something else, but it seems odd. And this works.
            parentRoot = sdoc.documentElement
            if parentRoot.tagName == "signed-credential" and parentRoot.hasAttributes():
                for attrIx in range(0, parentRoot.attributes.length):
                    attr = parentRoot.attributes.item(attrIx)
                    # returns the old attribute of same name that was
                    # on the credential
                    # Below throws InUse exception if we forgot to clone the attribute first
                    oldAttr = signed_cred.setAttributeNode(attr.cloneNode(True))
                    if oldAttr and oldAttr.value != attr.value:
                        msg = "Delegating cred from owner %s to %s over %s:\n - Replaced attribute %s value '%s' with '%s'" % \
                              (self.parent.gidCaller.get_urn(), self.gidCaller.get_urn(), self.gidObject.get_urn(), oldAttr.name, oldAttr.value, attr.value)
                        logger.warn(msg)
                        #raise CredentialNotVerifiable("Can't encode new valid delegated credential: %s" % msg)

            p_cred = doc.importNode(sdoc.getElementsByTagName("credential")[0], True)
            p = doc.createElement("parent")
            p.appendChild(p_cred)
            cred.appendChild(p)
        # done handling parent credential

        # Create the <signatures> tag
        signatures = doc.createElement("signatures")
        signed_cred.appendChild(signatures)

        # Add any parent signatures
        if self.parent:
            for cur_cred in self.get_credential_list()[1:]:
                sdoc = parseString(cur_cred.get_signature().get_xml())
                ele = doc.importNode(sdoc.getElementsByTagName("Signature")[0], True)
                signatures.appendChild(ele)
                
        # Get the finished product
        self.xml = doc.toxml()
Example #13
0
    def run_instances(self, tenant_name, user_name, rspec, key_name, pubkeys):
        # TODO : Kulcloud
        # It'll use Openstack admin info. as authoirty
        zones = self.get_availability_zones()
        # add the sfa admin user to this tenant and update our Openstack client connection
        # to use these credentials for the rest of this session. This emsures that the instances
        # we create will be assigned to the correct tenant.
        self.driver.shell.compute_manager.connect(username=user_name, tenant=tenant_name, password=user_name)
        self.driver.shell.network_manager.connect(username=user_name, tenant=tenant_name, password=user_name)
        logger.info( "Checking if the created tenant[%s] or not ..." % tenant_name )
        tenant = self.driver.shell.auth_manager.tenants.find(name=tenant_name)

        if len(pubkeys):
            files = None
        else:
            authorized_keys = "\n".join(pubkeys)
            files = {'/root/.ssh/authorized_keys': authorized_keys}
        net_dict = self.create_network(tenant_id=tenant.id) #TODO : Network creation
        router = self.create_router(tenant_id=tenant.id) #TODO : Router creation
        nics=[{'net-id': net_dict['id']}]

        # Iterate over clouds/zones/nodes
        slivers = []
        rspec = RSpec(rspec)     # TODO  XML Input
        for node in rspec.version.get_nodes_with_slivers():
            instances = node.get('slivers', [])
            for instance in instances:
                server_name = instance['sliver_name']
                # Check if instance exists or not
                servers = self.driver.shell.compute_manager.servers.findall(name=server_name)
                if len(servers) != 0:
                    for server in servers:
                        slivers.append(server)
                        logger.info("The server[%s] already existed ..." % server.name)
                    continue

                try: 
                    flavor = self.driver.shell.compute_manager.flavors.find(name=instance['flavor']['name'])
                    image = self.driver.shell.compute_manager.images.find(name=instance['boot_image']['name'])
                    zone_name = instance['availability_zone']['name']
                    for zone in zones:
                        if zone == zone_name:
                            break
                    else:
                        logger.warn("The requested zone_name[%s] is invalid ... So it's changed " % zone_name)
                        zone_name = zone

                    group_names=[]   
                    for sec_group in self.driver.secgroups_with_rules(instance['security_groups']):
                        group_names.append(sec_group.name)

                    metadata = {}
                    if node.get('component_id'):
                        metadata['component_id'] = node['component_id']
                    if node.get('component_manager_id'):
                        metadata['component_manager_id'] = node['component_manager_id']

                    # Create a server for the user
                    server = self.driver.shell.compute_manager.servers.create(
                                                               flavor=flavor.id,
                                                               image=image.id,
                                                               nics=nics,
                                                               availability_zone=zone_name,
                                                               key_name=key_name,
                                                               security_groups=group_names,
                                                               meta=metadata,
                                                               name=server_name)
#                                                               files=files,
                    server = self.check_server_status(server)
                    slivers.append(server)
                    logger.info("Created Openstack instance [%s]" % server_name)

                except Exception, err:
                    logger.log_exc(err)
Example #14
0
    def encode(self):
        # Create the XML document
        doc = Document()
        signed_cred = doc.createElement("signed-credential")

        # Declare namespaces
        # Note that credential/policy.xsd are really the PG schemas
        # in a PL namespace.
        # Note that delegation of credentials between the 2 only really works
        # cause those schemas are identical.
        # Also note these PG schemas talk about PG tickets and CM policies.
        signed_cred.setAttribute("xmlns:xsi",
                                 "http://www.w3.org/2001/XMLSchema-instance")
        signed_cred.setAttribute(
            "xsi:noNamespaceSchemaLocation",
            "http://www.planet-lab.org/resources/sfa/credential.xsd")
        signed_cred.setAttribute(
            "xsi:schemaLocation",
            "http://www.planet-lab.org/resources/sfa/ext/policy/1 http://www.planet-lab.org/resources/sfa/ext/policy/1/policy.xsd"
        )

        # PG says for those last 2:
        #        signed_cred.setAttribute("xsi:noNamespaceSchemaLocation", "http://www.protogeni.net/resources/credential/credential.xsd")
        #        signed_cred.setAttribute("xsi:schemaLocation", "http://www.protogeni.net/resources/credential/ext/policy/1 http://www.protogeni.net/resources/credential/ext/policy/1/policy.xsd")

        doc.appendChild(signed_cred)

        # Fill in the <credential> bit
        cred = doc.createElement("credential")
        cred.setAttribute("xml:id", self.get_refid())
        signed_cred.appendChild(cred)
        append_sub(doc, cred, "type", "privilege")
        append_sub(doc, cred, "serial", "8")
        append_sub(doc, cred, "owner_gid", self.gidCaller.save_to_string())
        append_sub(doc, cred, "owner_urn", self.gidCaller.get_urn())
        append_sub(doc, cred, "target_gid", self.gidObject.save_to_string())
        append_sub(doc, cred, "target_urn", self.gidObject.get_urn())
        append_sub(doc, cred, "uuid", "")
        if not self.expiration:
            self.set_expiration(datetime.datetime.utcnow() +
                                datetime.timedelta(
                                    seconds=DEFAULT_CREDENTIAL_LIFETIME))
        self.expiration = self.expiration.replace(microsecond=0)
        append_sub(doc, cred, "expires", self.expiration.isoformat())
        privileges = doc.createElement("privileges")
        cred.appendChild(privileges)

        if self.privileges:
            rights = self.get_privileges()
            for right in rights.rights:
                priv = doc.createElement("privilege")
                append_sub(doc, priv, "name", right.kind)
                append_sub(doc, priv, "can_delegate",
                           str(right.delegate).lower())
                privileges.appendChild(priv)

        # Add the parent credential if it exists
        if self.parent:
            sdoc = parseString(self.parent.get_xml())
            # If the root node is a signed-credential (it should be), then
            # get all its attributes and attach those to our signed_cred
            # node.
            # Specifically, PG and PLadd attributes for namespaces (which is reasonable),
            # and we need to include those again here or else their signature
            # no longer matches on the credential.
            # We expect three of these, but here we copy them all:
            #        signed_cred.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance")
            # and from PG (PL is equivalent, as shown above):
            #        signed_cred.setAttribute("xsi:noNamespaceSchemaLocation", "http://www.protogeni.net/resources/credential/credential.xsd")
            #        signed_cred.setAttribute("xsi:schemaLocation", "http://www.protogeni.net/resources/credential/ext/policy/1 http://www.protogeni.net/resources/credential/ext/policy/1/policy.xsd")

            # HOWEVER!
            # PL now also declares these, with different URLs, so
            # the code notices those attributes already existed with
            # different values, and complains.
            # This happens regularly on delegation now that PG and
            # PL both declare the namespace with different URLs.
            # If the content ever differs this is a problem,
            # but for now it works - different URLs (values in the attributes)
            # but the same actual schema, so using the PG schema
            # on delegated-to-PL credentials works fine.

            # Note: you could also not copy attributes
            # which already exist. It appears that both PG and PL
            # will actually validate a slicecred with a parent
            # signed using PG namespaces and a child signed with PL
            # namespaces over the whole thing. But I don't know
            # if that is a bug in xmlsec1, an accident since
            # the contents of the schemas are the same,
            # or something else, but it seems odd. And this works.
            parentRoot = sdoc.documentElement
            if parentRoot.tagName == "signed-credential" and parentRoot.hasAttributes(
            ):
                for attrIx in range(0, parentRoot.attributes.length):
                    attr = parentRoot.attributes.item(attrIx)
                    # returns the old attribute of same name that was
                    # on the credential
                    # Below throws InUse exception if we forgot to clone the attribute first
                    oldAttr = signed_cred.setAttributeNode(
                        attr.cloneNode(True))
                    if oldAttr and oldAttr.value != attr.value:
                        msg = "Delegating cred from owner %s to %s over %s:\n - Replaced attribute %s value '%s' with '%s'" % (
                            self.parent.gidCaller.get_urn(),
                            self.gidCaller.get_urn(), self.gidObject.get_urn(),
                            oldAttr.name, oldAttr.value, attr.value)
                        logger.warn(msg)
                        #raise CredentialNotVerifiable("Can't encode new valid delegated credential: %s" % msg)

            p_cred = doc.importNode(
                sdoc.getElementsByTagName("credential")[0], True)
            p = doc.createElement("parent")
            p.appendChild(p_cred)
            cred.appendChild(p)
        # done handling parent credential

        # Create the <signatures> tag
        signatures = doc.createElement("signatures")
        signed_cred.appendChild(signatures)

        # Add any parent signatures
        if self.parent:
            for cur_cred in self.get_credential_list()[1:]:
                sdoc = parseString(cur_cred.get_signature().get_xml())
                ele = doc.importNode(
                    sdoc.getElementsByTagName("Signature")[0], True)
                signatures.appendChild(ele)

        # Get the finished product
        self.xml = doc.toxml()
Example #15
0
    def verify_chain(self, trusted_certs=None):
        # Verify a chain of certificates. Each certificate must be signed by
        # the public key contained in it's parent. The chain is recursed
        # until a certificate is found that is signed by a trusted root.

        # verify expiration time
        if self.cert.has_expired():
            logger.debug("verify_chain: NO our certificate %s has expired" %
                         self.get_printable_subject())
            raise CertExpired(self.get_printable_subject(), "client cert")

        # if this cert is signed by a trusted_cert, then we are set
        for trusted_cert in trusted_certs:
            if self.is_signed_by_cert(trusted_cert):
                # verify expiration of trusted_cert ?
                if not trusted_cert.cert.has_expired():
                    logger.debug(
                        "verify_chain: YES cert %s signed by trusted cert %s" %
                        (self.get_printable_subject(),
                         trusted_cert.get_printable_subject()))
                    return trusted_cert
                else:
                    logger.debug(
                        "verify_chain: NO cert %s is signed by trusted_cert %s, but this is expired..."
                        % (self.get_printable_subject(),
                           trusted_cert.get_printable_subject()))
                    raise CertExpired(
                        self.get_printable_subject(),
                        " signer trusted_cert %s" %
                        trusted_cert.get_printable_subject())

        # if there is no parent, then no way to verify the chain
        if not self.parent:
            logger.debug(
                "verify_chain: NO %s has no parent and issuer %s is not in %d trusted roots"
                % (self.get_printable_subject(), self.get_issuer(),
                   len(trusted_certs)))
            raise CertMissingParent(
                self.get_printable_subject(),
                "Non trusted issuer: %s out of %d trusted roots" %
                (self.get_issuer(), len(trusted_certs)))

        # if it wasn't signed by the parent...
        if not self.is_signed_by_cert(self.parent):
            logger.debug(
                "verify_chain: NO %s is not signed by parent %s, but by %s" %
                self.get_printable_subject(),
                self.parent.get_printable_subject(), self.get_issuer())
            return CertNotSignedByParent(
                self.get_printable_subject(), "parent %s, issuer %s" %
                (selr.parent.get_printable_subject(), self.get_issuer()))

        # Confirm that the parent is a CA. Only CAs can be trusted as
        # signers.
        # Note that trusted roots are not parents, so don't need to be
        # CAs.
        # Ugly - cert objects aren't parsed so we need to read the
        # extension and hope there are no other basicConstraints
        if not self.parent.isCA and not (
                self.parent.get_extension('basicConstraints') == 'CA:TRUE'):
            logger.warn("verify_chain: cert %s's parent %s is not a CA" %
                        (self.get_printable_subject(),
                         self.parent.get_printable_subject()))
            return CertNotSignedByParent(
                self.get_printable_subject(),
                "Parent %s not a CA" % self.parent.get_printable_subject())

        # if the parent isn't verified...
        logger.debug("verify_chain: .. %s, -> verifying parent %s" %
                     (self.get_printable_subject(),
                      self.parent.get_printable_subject()))
        self.parent.verify_chain(trusted_certs)

        return
Example #16
0
    def verify_slice_attributes(self,
                                slice,
                                requested_slice_attributes,
                                options={},
                                admin=False):
        append = options.get('append', True)
        # get list of attributes users ar able to manage
        filter = {'category': '*slice*'}
        if not admin:
            filter['|roles'] = ['user']
        slice_attributes = self.driver.shell.GetTagTypes(filter)
        valid_slice_attribute_names = [
            attribute['tagname'] for attribute in slice_attributes
        ]

        # get sliver attributes
        added_slice_attributes = []
        removed_slice_attributes = []
        # we need to keep the slice hrn anyway
        ignored_slice_attribute_names = ['hrn']
        existing_slice_attributes = self.driver.shell.GetSliceTags(
            {'slice_id': slice['slice_id']})

        # get attributes that should be removed
        for slice_tag in existing_slice_attributes:
            if slice_tag['tagname'] in ignored_slice_attribute_names:
                # If a slice already has a admin only role it was probably given to them by an
                # admin, so we should ignore it.
                ignored_slice_attribute_names.append(slice_tag['tagname'])
                attribute_found = True
            else:
                # If an existing slice attribute was not found in the request it should
                # be removed
                attribute_found = False
                for requested_attribute in requested_slice_attributes:
                    if requested_attribute['name'] == slice_tag['tagname'] and \
                       requested_attribute['value'] == slice_tag['value']:
                        attribute_found = True
                        break

            if not attribute_found and not append:
                removed_slice_attributes.append(slice_tag)

        # get attributes that should be added:
        for requested_attribute in requested_slice_attributes:
            # if the requested attribute wasn't found  we should add it
            if requested_attribute['name'] in valid_slice_attribute_names:
                attribute_found = False
                for existing_attribute in existing_slice_attributes:
                    if requested_attribute['name'] == existing_attribute['tagname'] and \
                       requested_attribute['value'] == existing_attribute['value']:
                        attribute_found = True
                        break
                if not attribute_found:
                    added_slice_attributes.append(requested_attribute)

        # remove stale attributes
        for attribute in removed_slice_attributes:
            try:
                self.driver.shell.DeleteSliceTag(attribute['slice_tag_id'])
            except Exception, e:
                logger.warn('Failed to remove sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
                                % (slice['name'], attribute['value'],  attribute.get('node_id'), str(e)))
Example #17
0
class PlSlices:

    rspec_to_slice_tag = {'max_rate': 'net_max_rate'}

    def __init__(self, driver):
        self.driver = driver

    def get_slivers(self, xrn, node=None):
        hrn, type = urn_to_hrn(xrn)

        slice_name = hrn_to_pl_slicename(hrn)
        # XX Should we just call PLCAPI.GetSliceTicket(slice_name) instead
        # of doing all of this?
        #return self.driver.shell.GetSliceTicket(self.auth, slice_name)

        # from PLCAPI.GetSlivers.get_slivers()
        slice_fields = [
            'slice_id', 'name', 'instantiation', 'expires', 'person_ids',
            'slice_tag_ids'
        ]
        slices = self.driver.shell.GetSlices(slice_name, slice_fields)
        # Build up list of users and slice attributes
        person_ids = set()
        all_slice_tag_ids = set()
        for slice in slices:
            person_ids.update(slice['person_ids'])
            all_slice_tag_ids.update(slice['slice_tag_ids'])
        person_ids = list(person_ids)
        all_slice_tag_ids = list(all_slice_tag_ids)
        # Get user information
        all_persons_list = self.driver.shell.GetPersons(
            {
                'person_id': person_ids,
                'enabled': True
            }, ['person_id', 'enabled', 'key_ids'])
        all_persons = {}
        for person in all_persons_list:
            all_persons[person['person_id']] = person

        # Build up list of keys
        key_ids = set()
        for person in all_persons.values():
            key_ids.update(person['key_ids'])
        key_ids = list(key_ids)
        # Get user account keys
        all_keys_list = self.driver.shell.GetKeys(
            key_ids, ['key_id', 'key', 'key_type'])
        all_keys = {}
        for key in all_keys_list:
            all_keys[key['key_id']] = key
        # Get slice attributes
        all_slice_tags_list = self.driver.shell.GetSliceTags(all_slice_tag_ids)
        all_slice_tags = {}
        for slice_tag in all_slice_tags_list:
            all_slice_tags[slice_tag['slice_tag_id']] = slice_tag

        slivers = []
        for slice in slices:
            keys = []
            for person_id in slice['person_ids']:
                if person_id in all_persons:
                    person = all_persons[person_id]
                    if not person['enabled']:
                        continue
                    for key_id in person['key_ids']:
                        if key_id in all_keys:
                            key = all_keys[key_id]
                            keys += [{
                                'key_type': key['key_type'],
                                'key': key['key']
                            }]
            attributes = []
            # All (per-node and global) attributes for this slice
            slice_tags = []
            for slice_tag_id in slice['slice_tag_ids']:
                if slice_tag_id in all_slice_tags:
                    slice_tags.append(all_slice_tags[slice_tag_id])
            # Per-node sliver attributes take precedence over global
            # slice attributes, so set them first.
            # Then comes nodegroup slice attributes
            # Followed by global slice attributes
            sliver_attributes = []

            if node is not None:
                for sliver_attribute in filter(
                        lambda a: a['node_id'] == node['node_id'], slice_tags):
                    sliver_attributes.append(sliver_attribute['tagname'])
                    attributes.append({
                        'tagname': sliver_attribute['tagname'],
                        'value': sliver_attribute['value']
                    })

            # set nodegroup slice attributes
            for slice_tag in filter(
                    lambda a: a['nodegroup_id'] in node['nodegroup_ids'],
                    slice_tags):
                # Do not set any nodegroup slice attributes for
                # which there is at least one sliver attribute
                # already set.
                if slice_tag not in slice_tags:
                    attributes.append({
                        'tagname': slice_tag['tagname'],
                        'value': slice_tag['value']
                    })

            for slice_tag in filter(lambda a: a['node_id'] is None,
                                    slice_tags):
                # Do not set any global slice attributes for
                # which there is at least one sliver attribute
                # already set.
                if slice_tag['tagname'] not in sliver_attributes:
                    attributes.append({
                        'tagname': slice_tag['tagname'],
                        'value': slice_tag['value']
                    })

            # XXX Sanity check; though technically this should be a system invariant
            # checked with an assertion
            if slice['expires'] > MAXINT: slice['expires'] = MAXINT

            slivers.append({
                'hrn': hrn,
                'name': slice['name'],
                'slice_id': slice['slice_id'],
                'instantiation': slice['instantiation'],
                'expires': slice['expires'],
                'keys': keys,
                'attributes': attributes
            })

        return slivers

    def get_sfa_peer(self, xrn):
        hrn, type = urn_to_hrn(xrn)

        # return the authority for this hrn or None if we are the authority
        sfa_peer = None
        slice_authority = get_authority(hrn)
        site_authority = get_authority(slice_authority)

        if site_authority != self.driver.hrn:
            sfa_peer = site_authority

        return sfa_peer

    def verify_slice_leases(self, slice, rspec_requested_leases):

        leases = self.driver.shell.GetLeases(
            {
                'name': slice['name'],
                'clip': int(time.time())
            }, ['lease_id', 'name', 'hostname', 't_from', 't_until'])
        grain = self.driver.shell.GetLeaseGranularity()

        requested_leases = []
        for lease in rspec_requested_leases:
            requested_lease = {}
            slice_hrn, _ = urn_to_hrn(lease['slice_id'])

            top_auth_hrn = top_auth(slice_hrn)
            site_hrn = '.'.join(slice_hrn.split('.')[:-1])
            slice_part = slice_hrn.split('.')[-1]
            if top_auth_hrn == self.driver.hrn:
                login_base = slice_hrn.split('.')[-2][:12]
            else:
                login_base = hash_loginbase(site_hrn)

            slice_name = '_'.join([login_base, slice_part])

            if slice_name != slice['name']:
                continue
            elif Xrn(lease['component_id']).get_authority_urn().split(
                    ':')[0] != self.driver.hrn:
                continue

            hostname = xrn_to_hostname(lease['component_id'])
            # fill the requested node with nitos ids
            requested_lease['name'] = slice['name']
            requested_lease['hostname'] = hostname
            requested_lease['t_from'] = int(lease['start_time'])
            requested_lease['t_until'] = int(lease['duration']) * grain + int(
                lease['start_time'])
            requested_leases.append(requested_lease)

        # prepare actual slice leases by lease_id
        leases_by_id = {}
        for lease in leases:
            leases_by_id[lease['lease_id']] = {'name': lease['name'], 'hostname': lease['hostname'], \
                                               't_from': lease['t_from'], 't_until': lease['t_until']}

        added_leases = []
        kept_leases_id = []
        deleted_leases_id = []
        for lease_id in leases_by_id:
            if leases_by_id[lease_id] not in requested_leases:
                deleted_leases_id.append(lease_id)
            else:
                kept_leases_id.append(lease_id)
                requested_leases.remove(leases_by_id[lease_id])
        added_leases = requested_leases

        try:
            self.driver.shell.DeleteLeases(deleted_leases_id)
            for lease in added_leases:
                self.driver.shell.AddLeases(lease['hostname'], slice['name'],
                                            lease['t_from'], lease['t_until'])

        except:
            logger.log_exc('Failed to add/remove slice leases')

        return leases

    def verify_slice_nodes(self, slice_urn, slice, rspec_nodes):

        slivers = {}
        for node in rspec_nodes:
            hostname = node.get('component_name')
            client_id = node.get('client_id')
            component_id = node.get('component_id').strip()
            if hostname:
                hostname = hostname.strip()
            elif component_id:
                hostname = xrn_to_hostname(component_id)
            if hostname:
                slivers[hostname] = {
                    'client_id': client_id,
                    'component_id': component_id
                }

        nodes = self.driver.shell.GetNodes(
            slice['node_ids'], ['node_id', 'hostname', 'interface_ids'])
        current_slivers = [node['hostname'] for node in nodes]

        # remove nodes not in rspec
        deleted_nodes = list(set(current_slivers).difference(slivers.keys()))

        # add nodes from rspec
        added_nodes = list(set(slivers.keys()).difference(current_slivers))

        try:
            self.driver.shell.AddSliceToNodes(slice['name'], added_nodes)
            self.driver.shell.DeleteSliceFromNodes(slice['name'],
                                                   deleted_nodes)

        except:
            logger.log_exc('Failed to add/remove slice from nodes')

        slices = self.driver.shell.GetSlices(slice['name'], ['node_ids'])
        resulting_nodes = self.driver.shell.GetNodes(slices[0]['node_ids'])

        # update sliver allocations
        for node in resulting_nodes:
            client_id = slivers[node['hostname']]['client_id']
            component_id = slivers[node['hostname']]['component_id']
            sliver_hrn = '%s.%s-%s' % (self.driver.hrn, slice['slice_id'],
                                       node['node_id'])
            sliver_id = Xrn(sliver_hrn, type='sliver').urn
            record = SliverAllocation(sliver_id=sliver_id,
                                      client_id=client_id,
                                      component_id=component_id,
                                      slice_urn=slice_urn,
                                      allocation_state='geni_allocated')
            record.sync(self.driver.api.dbsession())
        return resulting_nodes

    def free_egre_key(self):
        used = set()
        for tag in self.driver.shell.GetSliceTags({'tagname': 'egre_key'}):
            used.add(int(tag['value']))

        for i in range(1, 256):
            if i not in used:
                key = i
                break
        else:
            raise KeyError("No more EGRE keys available")

        return str(key)

    def verify_slice_links(self, slice, requested_links, nodes):

        if not requested_links:
            return

        # exit if links are not supported here
        topology = Topology()
        if not topology:
            return

        # build dict of nodes
        nodes_dict = {}
        interface_ids = []
        for node in nodes:
            nodes_dict[node['node_id']] = node
            interface_ids.extend(node['interface_ids'])
        # build dict of interfaces
        interfaces = self.driver.shell.GetInterfaces(interface_ids)
        interfaces_dict = {}
        for interface in interfaces:
            interfaces_dict[interface['interface_id']] = interface

        slice_tags = []

        # set egre key
        slice_tags.append({'name': 'egre_key', 'value': self.free_egre_key()})

        # set netns
        slice_tags.append({'name': 'netns', 'value': '1'})

        # set cap_net_admin
        # need to update the attribute string?
        slice_tags.append({'name': 'capabilities', 'value': 'CAP_NET_ADMIN'})

        for link in requested_links:
            # get the ip address of the first node in the link
            ifname1 = Xrn(link['interface1']['component_id']).get_leaf()

            if ifname1:
                ifname_parts = ifname1.split(':')
                node_raw = ifname_parts[0]
                device = None
                if len(ifname_parts) > 1:
                    device = ifname_parts[1]
                node_id = int(node_raw.replace('node', ''))
                node = nodes_dict[node_id]
                if1 = interfaces_dict[node['interface_ids'][0]]
                ipaddr = if1['ip']
                topo_rspec = VLink.get_topo_rspec(link, ipaddr)
                # set topo_rspec tag
                slice_tags.append({
                    'name': 'topo_rspec',
                    'value': str([topo_rspec]),
                    'node_id': node_id
                })
                # set vini_topo tag
                slice_tags.append({
                    'name': 'vini_topo',
                    'value': 'manual',
                    'node_id': node_id
                })
                #self.driver.shell.AddSliceTag(slice['name'], 'topo_rspec', str([topo_rspec]), node_id)

        self.verify_slice_attributes(slice,
                                     slice_tags, {'append': True},
                                     admin=True)

    def verify_site(self,
                    slice_xrn,
                    slice_record={},
                    sfa_peer=None,
                    options={}):
        #(slice_hrn, type) = urn_to_hrn(slice_xrn)
        #top_auth_hrn = top_auth(slice_hrn)
        #site_hrn = '.'.join(slice_hrn.split('.')[:-1])
        #if top_auth_hrn == self.driver.hrn:
        #    login_base = slice_hrn.split('.')[-2][:12]
        #else:
        #    login_base = hash_loginbase(site_hrn)
        plxrn = PlXrn(xrn=slice_xrn)
        slice_hrn = plxrn.get_hrn()
        type = plxrn.get_type()
        site_hrn = plxrn.get_authority_hrn()
        authority_name = plxrn.pl_authname()
        slicename = plxrn.pl_slicename()
        login_base = plxrn.pl_login_base()

        sites = self.driver.shell.GetSites(
            {'peer_id': None},
            ['site_id', 'name', 'abbreviated_name', 'login_base', 'hrn'])

        # filter sites by hrn
        site_exists = [site for site in sites if site['hrn'] == site_hrn]

        if not site_exists:
            # create new site record
            site = {
                'name': 'sfa:%s' % site_hrn,
                'abbreviated_name': site_hrn,
                'login_base': login_base,
                'max_slices': 100,
                'max_slivers': 1000,
                'enabled': True,
                'peer_site_id': None
            }

            site['site_id'] = self.driver.shell.AddSite(site)
            # Set site HRN
            self.driver.shell.SetSiteHrn(int(site['site_id']), site_hrn)
            # Tag this as created through SFA
            self.driver.shell.SetSiteSfaCreated(int(site['site_id']), 'True')
            # exempt federated sites from monitor policies
            self.driver.shell.AddSiteTag(int(site['site_id']),
                                         'exempt_site_until', "20200101")

        else:
            site = site_exists[0]

        return site

    def verify_slice(self,
                     slice_hrn,
                     slice_record,
                     sfa_peer,
                     expiration,
                     options={}):
        #top_auth_hrn = top_auth(slice_hrn)
        #site_hrn = '.'.join(slice_hrn.split('.')[:-1])
        #slice_part = slice_hrn.split('.')[-1]
        #if top_auth_hrn == self.driver.hrn:
        #    login_base = slice_hrn.split('.')[-2][:12]
        #else:
        #    login_base = hash_loginbase(site_hrn)
        #slice_name = '_'.join([login_base, slice_part])
        plxrn = PlXrn(xrn=slice_hrn)
        slice_hrn = plxrn.get_hrn()
        type = plxrn.get_type()
        site_hrn = plxrn.get_authority_hrn()
        authority_name = plxrn.pl_authname()
        slicename = plxrn.pl_slicename()
        login_base = plxrn.pl_login_base()

        slices = self.driver.shell.GetSlices({'peer_id': None},
                                             ['slice_id', 'name', 'hrn'])
        # Filter slices by HRN
        slice_exists = [slice for slice in slices if slice['hrn'] == slice_hrn]
        expires = int(datetime_to_epoch(utcparse(expiration)))
        if not slice_exists:
            if slice_record:
                url = slice_record.get('url', slice_hrn)
                description = slice_record.get('description', slice_hrn)
            else:
                url = slice_hrn
                description = slice_hrn
            slice = {
                'name': slice_name,
                'url': url,
                'description': description
            }
            # add the slice
            slice['slice_id'] = self.driver.shell.AddSlice(slice)
            # set the slice HRN
            self.driver.shell.SetSliceHrn(int(slice['slice_id']), slice_hrn)
            # Tag this as created through SFA
            self.driver.shell.SetSliceSfaCreated(int(slice['slice_id']),
                                                 'True')
            # set the expiration
            self.driver.shell.UpdateSlice(int(slice['slice_id']),
                                          {'expires': expires})

        else:
            slice = slice_exists[0]
            #Update expiration if necessary
            if slice.get('expires', None) != expires:
                self.driver.shell.UpdateSlice(int(slice['slice_id']),
                                              {'expires': expires})

        return self.driver.shell.GetSlices(int(slice['slice_id']))[0]

    def verify_persons(self,
                       slice_hrn,
                       slice_record,
                       users,
                       sfa_peer,
                       options={}):
        top_auth_hrn = top_auth(slice_hrn)
        site_hrn = '.'.join(slice_hrn.split('.')[:-1])
        slice_part = slice_hrn.split('.')[-1]
        users_by_hrn = {}
        for user in users:
            user['hrn'], _ = urn_to_hrn(user['urn'])
            users_by_hrn[user['hrn']] = user

        if top_auth_hrn == self.driver.hrn:
            login_base = slice_hrn.split('.')[-2][:12]
        else:
            login_base = hash_loginbase(site_hrn)

        slice_name = '_'.join([login_base, slice_part])

        persons = self.driver.shell.GetPersons({'peer_id': None},
                                               ['person_id', 'email', 'hrn'])
        sites = self.driver.shell.GetSites({'peer_id': None}, [
            'node_ids', 'site_id', 'name', 'person_ids', 'slice_ids',
            'login_base', 'hrn'
        ])
        site = [site for site in sites if site['hrn'] == site_hrn][0]
        slices = self.driver.shell.GetSlices({'peer_id': None}, [
            'slice_id', 'node_ids', 'person_ids', 'expires', 'site_id', 'name',
            'hrn'
        ])
        slice = [slice for slice in slices if slice['hrn'] == slice_hrn][0]
        slice_persons = self.driver.shell.GetPersons(
            {
                'peer_id': None,
                'person_id': slice['person_ids']
            }, ['person_id', 'email', 'hrn'])

        persons_by_hrn = {}
        persons_by_email = {}
        for person in persons:
            persons_by_hrn[person['hrn']] = person
            persons_by_email[person['email']] = person
        slice_persons_by_hrn = {}
        for slice_person in slice_persons:
            slice_persons_by_hrn[slice_person['hrn']] = slice_person

        # sort persons by HRN
        persons_to_add = set(users_by_hrn.keys()).difference(
            slice_persons_by_hrn.keys())
        persons_to_delete = set(slice_persons_by_hrn.keys()).difference(
            users_by_hrn.keys())
        persons_to_keep = set(users_by_hrn.keys()).intersection(
            slice_persons_by_hrn.keys())

        persons_to_verify_keys = {}

        # Add persons or add persons to slice
        for person_hrn in persons_to_add:
            person_email = users_by_hrn[person_hrn].get(
                'email', "*****@*****.**" % person_hrn.split('.')[-1])
            if person_email and person_email in persons_by_email.keys():
                # check if the user already exist in PL
                person_id = persons_by_email[person_email]['person_id']
                self.driver.shell.AddPersonToSlice(person_id,
                                                   slice['slice_id'])
                persons_to_verify_keys[person_id] = users_by_hrn[person_hrn]

            else:
                person = {
                    'first_name':
                    person_hrn,
                    'last_name':
                    person_hrn,
                    'email':
                    users_by_hrn[person_hrn].get(
                        'email', "*****@*****.**" % person_hrn.split('.')[-1]),
                }

                person_id = self.driver.shell.AddPerson(person)
                self.driver.shell.AddRoleToPerson('user', int(person_id))
                # enable the account
                self.driver.shell.UpdatePerson(int(person_id),
                                               {'enabled': True})
                self.driver.shell.SetPersonSfaCreated(int(person_id), 'True')
                self.driver.shell.AddPersonToSite(int(person_id),
                                                  site['site_id'])
                self.driver.shell.AddPersonToSlice(int(person_id),
                                                   slice['slice_id'])
                self.driver.shell.SetPersonHrn(int(person_id), person_hrn)

                # Add keys
                for key in users_by_hrn[person_hrn].get('keys', []):
                    key = {'key': key, 'key_type': 'ssh'}
                    self.driver.shell.AddPersonKey(person_id, key)

        # Delete persons from slice
        for person_hrn in persons_to_delete:
            person_id = slice_persons_by_hrn[person_hrn].get('person_id')
            slice_id = slice['slice_id']
            self.driver.shell.DeletePersonFromSlice(person_id, slice_id)

        # Update kept persons
        for person_hrn in persons_to_keep:
            person_id = slice_persons_by_hrn[person_hrn].get('person_id')
            persons_to_verify_keys[person_id] = users_by_hrn[person_hrn]

        self.verify_keys(persons_to_verify_keys, options)

        return persons_to_add

    def verify_keys(self, persons_to_verify_keys, options={}):
        # we only add keys that comes from sfa to persons in PL
        for person_id in persons_to_verify_keys:
            person_sfa_keys = persons_to_verify_keys[person_id].get('keys', [])
            person_pl_keys = self.driver.shell.GetKeys(
                {'person_id': int(person_id)})
            person_pl_keys_list = [key['key'] for key in person_pl_keys]

            keys_to_add = set(person_sfa_keys).difference(person_pl_keys_list)

            for key_string in keys_to_add:
                key = {'key': key_string, 'key_type': 'ssh'}
                self.driver.shell.AddPersonKey(int(person_id), key)

    def verify_slice_attributes(self,
                                slice,
                                requested_slice_attributes,
                                options={},
                                admin=False):
        append = options.get('append', True)
        # get list of attributes users ar able to manage
        filter = {'category': '*slice*'}
        if not admin:
            filter['|roles'] = ['user']
        slice_attributes = self.driver.shell.GetTagTypes(filter)
        valid_slice_attribute_names = [
            attribute['tagname'] for attribute in slice_attributes
        ]

        # get sliver attributes
        added_slice_attributes = []
        removed_slice_attributes = []
        # we need to keep the slice hrn anyway
        ignored_slice_attribute_names = ['hrn']
        existing_slice_attributes = self.driver.shell.GetSliceTags(
            {'slice_id': slice['slice_id']})

        # get attributes that should be removed
        for slice_tag in existing_slice_attributes:
            if slice_tag['tagname'] in ignored_slice_attribute_names:
                # If a slice already has a admin only role it was probably given to them by an
                # admin, so we should ignore it.
                ignored_slice_attribute_names.append(slice_tag['tagname'])
                attribute_found = True
            else:
                # If an existing slice attribute was not found in the request it should
                # be removed
                attribute_found = False
                for requested_attribute in requested_slice_attributes:
                    if requested_attribute['name'] == slice_tag['tagname'] and \
                       requested_attribute['value'] == slice_tag['value']:
                        attribute_found = True
                        break

            if not attribute_found and not append:
                removed_slice_attributes.append(slice_tag)

        # get attributes that should be added:
        for requested_attribute in requested_slice_attributes:
            # if the requested attribute wasn't found  we should add it
            if requested_attribute['name'] in valid_slice_attribute_names:
                attribute_found = False
                for existing_attribute in existing_slice_attributes:
                    if requested_attribute['name'] == existing_attribute['tagname'] and \
                       requested_attribute['value'] == existing_attribute['value']:
                        attribute_found = True
                        break
                if not attribute_found:
                    added_slice_attributes.append(requested_attribute)

        # remove stale attributes
        for attribute in removed_slice_attributes:
            try:
                self.driver.shell.DeleteSliceTag(attribute['slice_tag_id'])
            except Exception, e:
                logger.warn('Failed to remove sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
                                % (slice['name'], attribute['value'],  attribute.get('node_id'), str(e)))

        # add requested_attributes
        for attribute in added_slice_attributes:
            try:
                self.driver.shell.AddSliceTag(slice['name'], attribute['name'],
                                              attribute['value'],
                                              attribute.get('node_id', None))
            except Exception, e:
                logger.warn('Failed to add sliver attribute. name: %s, value: %s, node_id: %s\nCause:%s'\
                                % (slice['name'], attribute['value'],  attribute.get('node_id'), str(e)))