def test_entity_attribute(self): entity_id = root(self.t).get('entityID') self.md.set_entity_attributes(root(self.t), {"http://ns.example.org": "foo"}) self.md.store.update(root(self.t), entity_id) e = self.md.lookup("{%s}%s" % ("http://ns.example.org", 'foo'))[0] assert (e is not None) assert (e.get('entityID') == entity_id)
def test_store_and_retrieve(self): with patch.multiple("sys", exit=self.sys_exit, stdout=StreamCapturing(sys.stdout)): tmpdir = tempfile.mkdtemp() os.rmdir(tmpdir) # lets make sure 'store' can recreate it try: self.exec_pipeline(""" - load: - file://%s/metadata/test01.xml - select - store: directory: %s """ % (self.datadir, tmpdir)) t1 = parse_xml(resource_filename("metadata/test01.xml", self.datadir)) assert t1 is not None entity_id = 'https://idp.example.com/saml2/idp/metadata.php' sha1id = hash_id(entity_id, prefix=False) fn = "%s/%s.xml" % (tmpdir, sha1id) assert os.path.exists(fn) t2 = parse_xml(fn) assert t2 is not None assert root(t1).get('entityID') == root(t2).get('entityID') assert root(t2).get('entityID') == entity_id except IOError: raise Skip finally: shutil.rmtree(tmpdir)
def test_first_select_as(self): with patch.multiple("sys", exit=self.sys_exit, stdout=StreamCapturing(sys.stdout)): tmpfile = tempfile.NamedTemporaryFile('w').name try: self.exec_pipeline(""" - load: - file://%s/metadata/test01.xml - select as FOO - first - publish: %s """ % (self.datadir, tmpfile)) t1 = parse_xml(resource_filename("metadata/test01.xml", self.datadir)) assert t1 is not None entity_id = 'https://idp.example.com/saml2/idp/metadata.php' t2 = parse_xml(tmpfile) assert t2 is not None assert root(t1).get('entityID') == root(t2).get('entityID') assert root(t2).get('entityID') == entity_id except PipeException: pass except IOError: raise Skip finally: try: os.unlink(tmpfile) except: pass
def import_metadata(self, t, url=None): """ :param t: An EntitiesDescriptor element :param url: An optional URL to used to identify the EntitiesDescriptor in the MDRepository Import an EntitiesDescriptor element using the @Name attribute (or the supplied url parameter). All EntityDescriptor elements are stripped of any @ID attribute and are then indexed before the collection is stored in the MDRepository object. """ if url is None: top = t.xpath("//md:EntitiesDescriptor", namespaces=NS) if top is not None and len(top) == 1: url = top[0].get("Name", None) if url is None: raise MetadataException("No collection name found") self[url] = t # we always clean incoming ID # add to the index ne = 0 if t is not None: if root(t).tag == "{%s}EntityDescriptor" % NS['md']: self._index_entity(root(t)) ne += 1 else: for e in t.findall(".//{%s}EntityDescriptor" % NS['md']): self._index_entity(e) ne += 1 self.fire(type=EVENT_IMPORTED_METADATA, size=ne, url=url) return ne
def import_metadata(self, t, url=None): """ :param t: An EntitiesDescriptor element :param url: An optional URL to used to identify the EntitiesDescriptor in the MDRepository Import an EntitiesDescriptor element using the @Name attribute (or the supplied url parameter). All EntityDescriptor elements are stripped of any @ID attribute and are then indexed before the collection is stored in the MDRepository object. """ if url is None: top = t.xpath("//md:EntitiesDescriptor", namespaces=NS) if top is not None and len(top) == 1: url = top[0].get("Name", None) if url is None: raise ValueError("No collection name found") self[url] = t # we always clean incoming ID # add to the index ne = 0 if t is not None: if root(t).tag == "{%s}EntityDescriptor" % NS['md']: if root(t).attrib.has_key('ID'): del root(t).attrib['ID'] self.index.add(root(t)) ne += 1 else: for e in t.findall(".//{%s}EntityDescriptor" % NS['md']): if e.attrib.has_key('ID'): del e.attrib['ID'] self.index.add(e) ne += 1 return ne
def test_utils(self): entity_id = root(self.t).get('entityID') self.md.store.update(root(self.t), entity_id) e = self.md.lookup(entity_id)[0] assert (is_idp(e)) assert (not is_sp(e)) icon = entity_icon_url(e) assert ('url' in icon) assert ('https://www.example.com/static/images/umu_logo.jpg' in icon['url']) assert ('width' in icon) assert ('358' == icon['width']) assert ('height' in icon) assert ('63' == icon['height']) assert ('62' != icon['height']) domains = entity_domains(e) assert ('example.com' in domains) assert ('example.net' in domains) assert ('idp.example.com' not in domains) assert ('foo.com' not in domains) edup = deepcopy(e) name, desc = entity_extended_display(e) assert(name == 'Example University') assert(desc == 'Identity Provider for Example University') disp = entity_display_name(e) assert (disp == 'Example University') for elt in e.findall(".//{%s}DisplayName" % NS['mdui']): elt.getparent().remove(elt) disp = entity_display_name(e) assert (disp == 'The Example University') for elt in e.findall(".//{%s}OrganizationDisplayName" % NS['md']): elt.getparent().remove(elt) disp = entity_display_name(e) assert (disp == 'ExampleU') for elt in e.findall(".//{%s}OrganizationName" % NS['md']): elt.getparent().remove(elt) disp = entity_display_name(e) assert (disp == entity_id) e = edup subs = entity_domains(e) assert ('example.com' in subs) assert ('example.net' in subs) assert ('idp.example.com' not in subs) summary = entity_simple_summary(e) assert (summary['title'] == 'Example University') assert (summary['descr'] == 'Identity Provider for Example University') assert (summary['entityID'] == entity_id) assert ('domains' in summary) assert ('id' in summary) empty = entity_simple_summary(None) assert (not empty)
def test_utils(self): entity_id = root(self.t).get("entityID") self.md.store.update(root(self.t), entity_id) e = self.md.lookup(entity_id)[0] assert self.md.is_idp(e) assert not self.md.is_sp(e) assert self.md.icon(e) in [ "https://www.example.com/static/images/logo.jpg", "https://www.example.com/static/images/logo_eng.jpg", ] domains = self.md.domains(e) assert "example.com" in domains assert "example.net" in domains assert "idp.example.com" in domains assert "foo.com" not in domains edup = deepcopy(e) name, desc = self.md.ext_display(e) assert name == "Example University" assert desc == "Identity Provider for Example University" disp = self.md.display(e) assert disp == "Example University" for elt in e.findall(".//{%s}DisplayName" % NS["mdui"]): elt.getparent().remove(elt) disp = self.md.display(e) assert disp == "The Example University" for elt in e.findall(".//{%s}OrganizationDisplayName" % NS["md"]): elt.getparent().remove(elt) disp = self.md.display(e) assert disp == "ExampleU" for elt in e.findall(".//{%s}OrganizationName" % NS["md"]): elt.getparent().remove(elt) disp = self.md.display(e) assert disp == entity_id e = edup subs = self.md.sub_domains(e) assert "example.com" in subs assert "example.net" in subs assert "idp.example.com" not in subs summary = self.md.simple_summary(e) assert summary["title"] == "Example University" assert summary["descr"] == "Identity Provider for Example University" assert summary["value"] == entity_id assert "icon" in summary assert "icon_url" in summary and summary["icon"] == summary["icon_url"] assert "domains" in summary assert "id" in summary empty = self.md.simple_summary(None) assert not empty
def test_utils(self): entity_id = root(self.t).get('entityID') self.md.store.update(root(self.t), entity_id) e = self.md.lookup(entity_id)[0] assert (self.md.is_idp(e)) assert (not self.md.is_sp(e)) assert (self.md.icon(e) in ['https://www.example.com/static/images/logo.jpg', 'https://www.example.com/static/images/logo_eng.jpg'] ) domains = self.md.domains(e) assert ('example.com' in domains) assert ('example.net' in domains) assert ('idp.example.com' not in domains) assert ('foo.com' not in domains) edup = deepcopy(e) name, desc = self.md.ext_display(e) assert(name == 'Example University') assert(desc == 'Identity Provider for Example University') disp = self.md.display(e) assert (disp == 'Example University') for elt in e.findall(".//{%s}DisplayName" % NS['mdui']): elt.getparent().remove(elt) disp = self.md.display(e) assert (disp == 'The Example University') for elt in e.findall(".//{%s}OrganizationDisplayName" % NS['md']): elt.getparent().remove(elt) disp = self.md.display(e) assert (disp == 'ExampleU') for elt in e.findall(".//{%s}OrganizationName" % NS['md']): elt.getparent().remove(elt) disp = self.md.display(e) assert (disp == entity_id) e = edup subs = self.md.sub_domains(e) assert ('example.com' in subs) assert ('example.net' in subs) assert ('idp.example.com' not in subs) summary = self.md.simple_summary(e) assert (summary['title'] == 'Example University') assert (summary['descr'] == 'Identity Provider for Example University') assert (summary['value'] == entity_id) assert ('icon' in summary) assert ('icon_url' in summary and summary['icon'] == summary['icon_url']) assert ('domains' in summary) assert ('id' in summary) empty = self.md.simple_summary(None) assert (not empty)
def entities(self, t=None): """ :param t: An EntitiesDescriptor element Returns the list of contained EntityDescriptor elements """ if t is None: return [] elif root(t).tag == "{%s}EntityDescriptor" % NS['md']: return [root(t)] else: return t.findall(".//{%s}EntityDescriptor" % NS['md'])
def update(self, t, tid=None, ts=None, merge_strategy=None): # log.debug("memory store update: %s: %s" % (repr(t), tid)) relt = root(t) assert (relt is not None) ne = 0 if relt.tag == "{%s}EntityDescriptor" % NS['md']: # log.debug("memory store setting entity descriptor") self._unindex(relt) self._index(relt) self.entities[relt.get('entityID')] = relt # TODO: merge? if tid is not None: self.md[tid] = [relt.get('entityID')] ne += 1 # log.debug("keys %s" % self.md.keys()) elif relt.tag == "{%s}EntitiesDescriptor" % NS['md']: if tid is None: tid = relt.get('Name') lst = [] for e in iter_entities(t): self.update(e) lst.append(e.get('entityID')) ne += 1 self.md[tid] = lst return ne
def test_lookup_intersect_empty_test01(self): store = MemoryStore() store.update(self.test01) entity_id = root(self.test01).get('entityID') e = store.lookup("%s=%s+%s=%s" % (ATTRS['domain'], 'example.com', ATTRS['role'], 'sp')) print e assert (len(e) == 0)
def test_replace_ndn(self): idp = find_entity(root(self.t2), 'https://idp.nordu.net/idp/shibboleth') assert (idp is not None) idp2 = copy.deepcopy(idp) assert idp2 is not None for o in idp2.findall(".//{%s}OrganizationName" % NS['md']): o.text = "FOO" idp2.set('ID', 'kaka4711') replace_existing(idp, idp2) idp3 = find_entity(root(self.t2), 'kaka4711', attr='ID') assert (idp3 is not None) for o in idp2.findall(".//{%s}OrganizationName" % NS['md']): assert (o.text == "FOO") remove(idp3, None) idp = find_entity(root(self.t2), 'kaka4711', attr='ID') assert (idp3 is not None)
def test_lookup_intersect_empty_test01(self): store = self._redis_store() store.update(self.test01) entity_id = root(self.test01).get('entityID') assert (entity_id is not None) e = store.lookup("{%s}%s+{%s}%s" % (ATTRS['domain'], 'example.com', ATTRS['role'], 'sp')) assert (len(e) == 0)
def test_display(self): swamid = root(self.swamid) self.md.store.update(swamid, swamid.get('Name')) funet_connect = self.md.lookup('https://connect.funet.fi/shibboleth')[0] name, desc = self.md.ext_display(funet_connect) assert(name == 'FUNET E-Meeting Service') dn = self.md.display(funet_connect)
def setUp(self): self.md = MDRepository(store=MemoryStore) self.datadir = resource_filename('metadata', 'test/data') self.xml_source = os.path.join(self.datadir, 'test01.xml') self.swamid_source = os.path.join(self.datadir, 'swamid-2.0-test.xml') self.swamid = root(parse_xml(self.swamid_source)) self.t = parse_xml(self.xml_source) self.non_metadata = parse_xml(resource_filename("not-metadata.xml", self.datadir))
def test_maintain_test01(self): store = self._redis_store() store.update(self.test01) entity_id = root(self.test01).get('entityID') assert (entity_id is not None) d = dict() store.periodic(d) assert('Last Periodic Maintenance' in d)
def test_alias_ndn(self): r = requests.get("http://127.0.0.1:%s/ndn.xml" % self.port) assert (r.status_code == 200) # assert (r.encoding == 'utf8') t = parse_xml(six.BytesIO(r.content)) assert (t is not None) assert (root(t).get('entityID') == 'https://idp.nordu.net/idp/shibboleth') validate_document(t)
def test_alias_ndn(self): r = requests.get("http://127.0.0.1:%s/ndn.xml" % self.port) assert r.status_code == 200 # assert (r.encoding == 'utf8') t = parse_xml(StringIO(r.content)) assert t is not None assert root(t).get("entityID") == "https://idp.nordu.net/idp/shibboleth" validate_document(t)
def update(self, t, tid=None, ts=None, merge_strategy=None): # TODO: merge ? log.debug("redis store update: %s: %s" % (t, tid)) relt = root(t) ne = 0 if ts is None: ts = int(_now() + 3600 * 24 * 4) # 4 days is the arbitrary default expiration if relt.tag == "{%s}EntityDescriptor" % NS['md']: if tid is None: tid = relt.get('entityID') with self.rc.pipeline() as p: self.update_entity(relt, t, tid, ts, p) entity_id = relt.get("entityID") if entity_id is not None: self.membership("entities", entity_id, ts, p) for ea, eav in entity_attribute_dict(relt).iteritems(): for v in eav: # log.debug("%s=%s" % (ea, v)) self.membership("{%s}%s" % (ea, v), tid, ts, p) p.zadd("%s#values" % ea, v, ts) p.sadd("#attributes", ea) for hn in ('sha1', 'sha256', 'md5'): tid_hash = hex_digest(tid, hn) p.set("{%s}%s#alias" % (hn, tid_hash), tid) if ts is not None: p.expireat(tid_hash, ts) p.execute() ne += 1 elif relt.tag == "{%s}EntitiesDescriptor" % NS['md']: if tid is None: tid = relt.get('Name') ts = self._expiration(relt) with self.rc.pipeline() as p: self.update_entity(relt, t, tid, ts, p) for e in iter_entities(t): ne += self.update(e, ts=ts) entity_id = e.get("entityID") if entity_id is not None: self.membership(tid, entity_id, ts, p) self.membership("entities", entity_id, ts, p) p.execute() else: raise ValueError("Bad metadata top-level element: '%s'" % root(t).tag) return ne
def test_non_metadata(self): e = root(self.non_metadata) assert self.md.expiration(e) is None try: self.md.annotate(e,"kaka","x","y") self.md.set_entity_attributes(e, dict(a=1)) assert False except MetadataException: pass
def test_md_query_single(self): q = quote_plus('https://idp.nordu.net/idp/shibboleth') r = requests.get("http://127.0.0.1:%s/entities/%s" % (self.port, q)) assert (r.status_code == 200) assert ('application/xml' in r.headers['Content-Type']) t = parse_xml(six.BytesIO(r.content)) assert (t is not None) e = root(t) assert (e.get('entityID') == 'https://idp.nordu.net/idp/shibboleth')
def test_md_query_single(self): q = urllib.quote_plus("https://idp.nordu.net/idp/shibboleth") r = requests.get("http://127.0.0.1:%s/entities/%s" % (self.port, q)) assert r.status_code == 200 assert "application/xml" in r.headers["Content-Type"] t = parse_xml(StringIO(r.content)) assert t is not None e = root(t) assert e.get("entityID") == "https://idp.nordu.net/idp/shibboleth"
def test_lookup_test01(self): store = MemoryStore() store.update(self.test01) entity_id = root(self.test01).get('entityID') e = store.lookup(entity_id) assert (len(e) == 1) assert (e[0] is not None) assert (e[0].get('entityID') is not None) assert (e[0].get('entityID') == entity_id)
def test_lookup_intersect_test01(self): store = MemoryStore() store.update(self.test01) entity_id = root(self.test01).get('entityID') e = store.lookup("%s=%s+%s=%s" % (ATTRS['domain'], 'example.com', ATTRS['role'], 'idp')) print e assert (len(e) == 1) assert (e[0] is not None) assert (e[0].get('entityID') is not None) assert (e[0].get('entityID') == entity_id)
def test_lookup_intersect_test01(self): store = MemoryStore() store.update(self.test01) entity_id = root(self.test01).get('entityID') e = store.lookup( "%s=%s+%s=%s" % (ATTRS['domain'], 'example.com', ATTRS['role'], 'idp')) assert (len(e) == 1) assert (e[0] is not None) assert (e[0].get('entityID') is not None) assert (e[0].get('entityID') == entity_id)
def test_lookup_test01(self): store = self._redis_store() store.update(self.test01) entity_id = root(self.test01).get('entityID') assert (entity_id is not None) e = store.lookup(entity_id) print("%s: %s" % (entity_id, e)) assert (len(e) == 1) assert (e[0] is not None) assert (e[0].get('entityID') is not None) assert (e[0].get('entityID') == entity_id)
def test_lookup_test01(self): store = self._redis_store() store.update(self.test01) entity_id = root(self.test01).get('entityID') assert (entity_id is not None) e = store.lookup(entity_id) print "%s: %s" % (entity_id, e) assert (len(e) == 1) assert (e[0] is not None) assert (e[0].get('entityID') is not None) assert (e[0].get('entityID') == entity_id)
def test_lookup_test01(self): store = RedisWhooshStore(directory=self.dir, clear=True, name="test", redis=fakeredis.FakeStrictRedis()) store.update(self.test01, etag='test01', lazy=False) entity_id = root(self.test01).get('entityID') e = store.lookup(entity_id) assert (len(e) == 1) assert (e[0] is not None) assert (e[0].get('entityID') is not None) assert (e[0].get('entityID') == entity_id)
def find_in_document(t, member): relt = root(t) if is_text(member): if '!' in member: (src, xp) = member.split("!") return relt.xpath(xp, namespaces=NS, smart_strings=False) else: lst = [] for e in iter_entities(relt): if e.get('entityID') == member: lst.append(e) return lst raise MetadataException("unknown format for filtr member: %s" % member)
def test_search_test01(self): store = RedisWhooshStore(directory=self.dir, clear=True, name="test", redis=fakeredis.FakeStrictRedis()) store.update(self.test01, etag='test01', lazy=False) entity_id = root(self.test01).get('entityID') for q in ('example', 'Example', 'university'): e = list(store.search(q)) assert (len(e) == 1) assert (e[0] is not None) assert (e[0].get('entityID') is not None) assert (e[0].get('entityID') == entity_id)
def test_lookup_intersect_test01(self): store = RedisWhooshStore(directory=self.dir, clear=True, name="test", redis=fakeredis.FakeStrictRedis()) store.update(self.test01, etag='test01', lazy=False) entity_id = root(self.test01).get('entityID') e = store.lookup( "%s=%s+%s=%s" % (ATTRS['domain'], 'example.com', ATTRS['role'], 'idp')) assert (len(e) == 1) assert (e[0] is not None) assert (e[0].get('entityID') is not None) assert (e[0].get('entityID') == entity_id)
def expiration(t): relt = root(t) if relt.tag in ('{%s}EntityDescriptor' % NS['md'], '{%s}EntitiesDescriptor' % NS['md']): cache_duration = config.default_cache_duration valid_until = relt.get('validUntil', None) if valid_until is not None: now = utc_now().replace(microsecond=0) vu = iso2datetime(valid_until) return vu - now elif config.respect_cache_duration: cache_duration = relt.get('cacheDuration', config.default_cache_duration) return duration2timedelta(cache_duration) return None
def update(self, t, tid=None, etag=None, lazy=True): relt = root(t) assert relt is not None if relt.tag == "{%s}EntityDescriptor" % NS['md']: self._unindex(relt) self._index(relt) self.entities[relt.get('entityID')] = relt # TODO: merge? if tid is not None: self.md[tid] = [relt.get('entityID')] elif relt.tag == "{%s}EntitiesDescriptor" % NS['md']: if tid is None: tid = relt.get('Name') lst = [] for e in iter_entities(t): self.update(e) lst.append(e.get('entityID')) self.md[tid] = lst
def select(self, member, xp=None): """ Select a set of metadata elements and return an EntityDescriptor with the result of the select. :param member: A selector (cf below) :type member: basestring :param xp: An optional xpath filter :type xp: basestring :return: An interable of EntityDescriptor elements :rtype: etree.Element **Selector Syntax** - selector "+" selector - [sourceID] "!" xpath - attribute=value or {attribute}value - entityID - source (typically @Name from an EntitiesDescriptor set but could also be an alias) The first form results in the intersection of the results of doing a lookup on the selectors. The second form results in the EntityDescriptor elements from the source (defaults to all EntityDescriptors) that match the xpath expression. The attribute-value forms resuls in the EntityDescriptors that contain the specified entity attribute pair. If non of these forms apply, the lookup is done using either source ID (normally @Name from the EntitiesDescriptor) or the entityID of single EntityDescriptors. If member is a URI but isn't part of the metadata repository then it is fetched an treated as a list of (one per line) of selectors. If all else fails an empty list is returned. """ l = self._select(member) if hasattr(l, 'tag'): l = [l] elif hasattr(l, '__iter__'): l = list(l) if xp is None: return l else: log.debug("filtering %d entities using xpath %s" % (len(l), xp)) t = entitiesdescriptor(l, 'dummy', lookup_fn=self.lookup) if t is None: return [] l = root(t).xpath(xp, namespaces=NS, smart_strings=False) log.debug("got %d entities after filtering" % len(l)) return l
def update(self, t, tid=None, etag=None, lazy=True): relt = root(t) assert relt is not None if relt.tag == "{%s}EntityDescriptor" % NS['md']: ref = object_id(relt) parts = None if ref in self.parts: parts = self.parts[ref] if etag is not None and (parts is None or parts.get('etag', None) != etag): self.parts[ref] = { 'id': relt.get('entityID'), 'etag': etag, 'count': 1, 'items': [ref] } self.objects[ref] = relt self._last_modified = datetime.now() elif relt.tag == "{%s}EntitiesDescriptor" % NS['md']: if tid is None: tid = relt.get('Name') if etag is None: etag = hex_digest(dumptree(t, pretty_print=False), 'sha256') parts = None if tid in self.parts: parts = self.parts[tid] if parts is None or parts.get('etag', None) != etag: items = set() for e in iter_entities(t): ref = object_id(e) items.add(ref) self.objects[ref] = e self.parts[tid] = { 'id': tid, 'count': len(items), 'etag': etag, 'items': list(items) } self._last_modified = datetime.now() if not lazy: self._reindex()
def parse(self, resource: Resource, content: str) -> ParserInfo: info = ParserInfo(description='XRD links', expiration_time='never expires') t = parse_xml(unicode_stream(content)) relt = root(t) for xrd in t.iter("{%s}XRD" % NS['xrd']): for link in xrd.findall(".//{%s}Link[@rel='%s']" % (NS['xrd'], NS['md'])): link_href = link.get("href") certs = CertDict(link) fingerprints = list(certs.keys()) fp = None if len(fingerprints) > 0: fp = fingerprints[0] log.debug("XRD: {} verified by {}".format(link_href, fp)) child_opts = resource.opts.copy(update={'alias': None}) resource.add_child(link_href, child_opts) resource.last_seen = utc_now().replace(microsecond=0) resource.expire_time = None resource.never_expires = True return info
def update(self, t, tid=None, ts=None, merge_strategy=None): # log.debug("memory store update: %s: %s" % (repr(t), tid)) relt = root(t) assert (relt is not None) ne = 0 if relt.tag == "{%s}EntityDescriptor" % NS['md']: # log.debug("memory store setting entity descriptor") self._unindex(relt) self._index(relt) self.entities[relt.get('entityID')] = relt # TODO: merge? if tid is not None: self.md[tid] = relt ne += 1 # log.debug("keys %s" % self.md.keys()) elif relt.tag == "{%s}EntitiesDescriptor" % NS['md']: if tid is None: tid = relt.get('Name') for e in iter_entities(t): self.update(e) ne += 1 self.md[tid] = relt return ne
def summary(self, uri): """ :param uri: An EntitiesDescriptor URI present in the MDRepository :return: an information dict Returns a dict object with basic information about the EntitiesDescriptor """ seen = dict() info = dict() t = root(self[uri]) info['Name'] = t.get('Name', uri) info['cacheDuration'] = t.get('cacheDuration', None) info['validUntil'] = t.get('validUntil', None) info['Duplicates'] = [] info['Size'] = 0 for e in self.entities(self[uri]): entityID = e.get('entityID') if seen.get(entityID, False): info['Duplicates'].append(entityID) else: seen[entityID] = True info['Size'] += 1 return info
def _pubinfo(req, *opts): """ Sets publication info extension on EntityDescription element :param req: The request :param opts: Options (not used) :return: A modified working document Transforms the working document by setting the specified attribute on all of the EntityDescriptor elements of the active document. **Examples** .. code-block:: yaml - pubinfo: publisher: <publisher URL> """ if req.t is None: raise PipeException("Your pipeline is missing a select statement.") req.md.set_pubinfo(root(req.t), **req.args) return req.t
def test_entity_dict_test01(self): d = entity_attribute_dict(root(self.test01)) assert ('example.com' in d[ATTRS['domain']]) assert ('example.net' in d[ATTRS['domain']]) assert ('foo.com' not in d[ATTRS['domain']])
def test_lookup_intersect_empty_test01(self): store = MemoryStore() store.update(self.test01) entity_id = root(self.test01).get('entityID') e = store.lookup("%s=%s+%s=%s" % (ATTRS['domain'], 'example.com', ATTRS['role'], 'sp')) assert (len(e) == 0)
def finalize(req, *opts): """ Prepares the working document for publication/rendering. :param req: The request :param opts: Options (not used) :return: returns the working document with @Name, @cacheDuration and @validUntil set Set Name, ID, cacheDuration and validUntil on the toplevel EntitiesDescriptor element of the working document. Unless explicit provided the @Name is set from the request URI if the pipeline is executed in the pyFF server. The @ID is set to a string representing the current date/time and will be prefixed with the string provided, which defaults to '_'. The @cacheDuration element must be a valid xsd duration (eg PT5H for 5 hrs) and @validUntil can be either an absolute ISO 8601 time string or (more comonly) a relative time on the form .. code-block:: none \+?([0-9]+d)?\s*([0-9]+h)?\s*([0-9]+m)?\s*([0-9]+s)? For instance +45d 2m results in a time delta of 45 days and 2 minutes. The '+' sign is optional. If operating on a single EntityDescriptor then @Name is ignored (cf :py:mod:`pyff.pipes.builtins.first`). **Examples** .. code-block:: yaml - finalize: cacheDuration: PT8H validUntil: +10d ID: pyff """ if req.t is None: raise PipeException("Your plumbing is missing a select statement.") e = root(req.t) if e.tag == "{%s}EntitiesDescriptor" % NS['md']: name = req.args.get('name', None) if name is None or not len(name): name = req.args.get('Name', None) if name is None or not len(name): name = req.state.get('url', None) if name is None or not len(name): name = e.get('Name', None) if name is not None and len(name): e.set('Name', name) now = datetime.utcnow() mdid = req.args.get('ID', 'prefix _') if re.match('(\s)*prefix(\s)*', mdid): prefix = re.sub('^(\s)*prefix(\s)*', '', mdid) ID = now.strftime(prefix + "%Y%m%dT%H%M%SZ") else: ID = mdid if not e.get('ID'): e.set('ID', ID) valid_until = str(req.args.get('validUntil', e.get('validUntil', None))) if valid_until is not None and len(valid_until) > 0: offset = duration2timedelta(valid_until) if offset is not None: dt = now + offset e.set('validUntil', dt.strftime("%Y-%m-%dT%H:%M:%SZ")) elif valid_until is not None: try: dt = iso8601.parse_date(valid_until) dt = dt.replace(tzinfo=None) # make dt "naive" (tz-unaware) offset = dt - now e.set('validUntil', dt.strftime("%Y-%m-%dT%H:%M:%SZ")) except ValueError, ex: log.error("Unable to parse validUntil: %s (%s)" % (valid_until, ex)) # set a reasonable default: 50% of the validity # we replace this below if we have cacheDuration set req.state['cache'] = int(total_seconds(offset) / 50)
def sign(req, *opts): """ Sign the working document. :param req: The request :param opts: Options (unused) :return: returns the signed working document Sign expects a single dict with at least a 'key' key and optionally a 'cert' key. The 'key' argument references either a PKCS#11 uri or the filename containing a PEM-encoded non-password protected private RSA key. The 'cert' argument may be empty in which case the cert is looked up using the PKCS#11 token, or may point to a file containing a PEM-encoded X.509 certificate. **PKCS11 URIs** A pkcs11 URI has the form .. code-block:: xml pkcs11://<absolute path to SO/DLL>[:slot]/<object label>[?pin=<pin>] The pin parameter can be used to point to an environment variable containing the pin: "env:<ENV variable>". By default pin is "env:PYKCS11PIN" which tells sign to use the pin found in the PYKCS11PIN environment variable. This is also the default for PyKCS11 which is used to communicate with the PKCS#11 module. **Examples** .. code-block:: yaml - sign: key: pkcs11:///usr/lib/libsofthsm.so/signer This would sign the document using the key with label 'signer' in slot 0 of the /usr/lib/libsofthsm.so module. Note that you may need to run pyff with env PYKCS11PIN=<pin> .... for this to work. Consult the documentation of your PKCS#11 module to find out about any other configuration you may need. .. code-block:: yaml - sign: key: signer.key cert: signer.crt This example signs the document using the plain key and cert found in the signer.key and signer.crt files. """ if req.t is None: raise PipeException("Your pipeline is missing a select statement.") if not type(req.args) is dict: raise PipeException("Missing key and cert arguments to sign pipe") key_file = req.args.get('key', None) cert_file = req.args.get('cert', None) if key_file is None: raise PipeException("Missing key argument for sign pipe") if cert_file is None: log.info("Attempting to extract certificate from token...") opts = dict() relt = root(req.t) idattr = relt.get('ID') if idattr: opts['reference_uri'] = "#%s" % idattr xmlsec.sign(req.t, key_file, cert_file, **opts) return req.t
def test_entities_list(self): assert len(list(entities_list(root(self.t2)))) == 1032 assert len(list(entities_list(None))) == 0
def _unpickle(self, pickled_data): return root(parse_xml(BytesIO(pickled_data)))
def _get_metadata(self, key): return root(parse_xml(StringIO(self.rc.get("%s#metadata" % key))))
def test_sha1_hash(self): entity_id = root(self.t).get('entityID') self.md.store.update(root(self.t), entity_id) e = self.md.lookup(entity_id) assert (self.md.sha1_id(e[0]) == "{sha1}568515f6fae8c8b4d42d543853c96d08f051ef13") assert (hash_id(e[0], 'sha1', prefix=False) == "568515f6fae8c8b4d42d543853c96d08f051ef13")
def parse(self, resource: Resource, content: str) -> EidasMDParserInfo: info = EidasMDParserInfo(description='eIDAS MetadataServiceList', expiration_time='None') t = parse_xml(unicode_stream(content)) if config.xinclude: t.xinclude() relt = root(t) info.version = relt.get('Version', '0') info.issue_date = relt.get('IssueDate') info.next_update = relt.get('NextUpdate') if isinstance(info.next_update, str): resource.expire_time = iso2datetime(info.next_update) elif config.respect_cache_duration: duration = duration2timedelta(config.default_cache_duration) if not duration: # TODO: what is the right action here? raise ValueError( f'Invalid default cache duration: {config.default_cache_duration}' ) info.next_update = utc_now().replace(microsecond=0) + duration resource.expire_time = info.next_update info.expiration_time = 'None' if not resource.expire_time else resource.expire_time.isoformat( ) info.issuer_name = first_text(relt, "{%s}IssuerName" % NS['ser']) info.scheme_identifier = first_text(relt, "{%s}SchemeIdentifier" % NS['ser']) info.scheme_territory = first_text(relt, "{%s}SchemeTerritory" % NS['ser']) for mdl in relt.iter("{%s}MetadataList" % NS['ser']): for ml in mdl.iter("{%s}MetadataLocation" % NS['ser']): location = ml.get('Location') if location: certs = CertDict(ml) fingerprints = list(certs.keys()) fp = None if len(fingerprints) > 0: fp = fingerprints[0] ep = ml.find("{%s}Endpoint" % NS['ser']) if ep is not None and fp is not None: args = dict( country_code=mdl.get('Territory'), hide_from_discovery=strtobool( ep.get('HideFromDiscovery', 'false')), ) log.debug("MDSL[{}]: {} verified by {} for country {}". format(info.scheme_territory, location, fp, args.get('country_code'))) child_opts = resource.opts.copy(update={'alias': None}) child_opts.verify = fp r = resource.add_child(location, child_opts) # this is specific post-processing for MDSL files def _update_entities(_t, **kwargs): _country_code = kwargs.get('country_code') _hide_from_discovery = kwargs.get( 'hide_from_discovery') for e in iter_entities(_t): if _country_code: set_nodecountry(e, _country_code) if bool(_hide_from_discovery) and is_idp(e): set_entity_attributes( e, { ATTRS['entity-category']: 'http://refeds.org/category/hide-from-discovery' }) return _t r.add_via(Lambda(_update_entities, **args)) log.debug("Done parsing eIDAS MetadataServiceList") resource.last_seen = utc_now().replace(microsecond=0) resource.expire_time = None return info
def consumer(q, njobs, stats, next_jobs=None, resolved=None): if next_jobs is None: next_jobs = [] if resolved is None: resolved = set() nfinished = 0 while nfinished < njobs: info = None try: log.debug("waiting for next thread to finish...") thread = q.get(True) thread.join(timeout) if thread.isAlive(): raise MetadataException("thread timeout fetching '%s'" % thread.url) info = { 'Time Spent': thread.time() } if thread.ex is not None: raise thread.ex else: if thread.result is not None: info['Bytes'] = len(thread.result) else: raise MetadataException("empty response fetching '%s'" % thread.url) info['Cached'] = thread.cached info['Date'] = str(thread.date) info['Last-Modified'] = str(thread.last_modified) info['Tries'] = thread.tries xml = thread.result.strip() if thread.status is not None: info['Status'] = thread.status t = self.parse_metadata(StringIO(xml), key=thread.verify, base_url=thread.url) if t is None: self.fire(type=EVENT_IMPORT_FAIL, url=thread.url) raise MetadataException("no valid metadata found at '%s'" % thread.url) relt = root(t) if relt.tag in ('{%s}XRD' % NS['xrd'], '{%s}XRDS' % NS['xrd']): log.debug("%s looks like an xrd document" % thread.url) for xrd in t.xpath("//xrd:XRD", namespaces=NS): log.debug("xrd: %s" % xrd) for link in xrd.findall(".//{%s}Link[@rel='%s']" % (NS['xrd'], NS['md'])): url = link.get("href") certs = xmlsec.CertDict(link) fingerprints = certs.keys() fp = None if len(fingerprints) > 0: fp = fingerprints[0] log.debug("fingerprint: %s" % fp) next_jobs.append((url, fp, url, 0)) elif relt.tag in ('{%s}EntityDescriptor' % NS['md'], '{%s}EntitiesDescriptor' % NS['md']): cacheDuration = self.default_cache_duration if self.respect_cache_duration: cacheDuration = root(t).get('cacheDuration', self.default_cache_duration) offset = duration2timedelta(cacheDuration) if thread.cached: if thread.last_modified + offset < datetime.now() - duration2timedelta(self.min_cache_ttl): raise MetadataException("cached metadata expired") else: log.debug("found cached metadata for '%s' (last-modified: %s)" % (thread.url, thread.last_modified)) ne = self.import_metadata(t, url=thread.id) info['Number of Entities'] = ne else: log.debug("got fresh metadata for '%s' (date: %s)" % (thread.url, thread.date)) ne = self.import_metadata(t, url=thread.id) info['Number of Entities'] = ne info['Cache Expiration Time'] = str(thread.last_modified + offset) certs = xmlsec.CertDict(relt) cert = None if certs.values(): cert = certs.values()[0].strip() resolved.add((thread.url, cert)) else: raise MetadataException("unknown metadata type for '%s' (%s)" % (thread.url, relt.tag)) except Exception, ex: #traceback.print_exc(ex) log.warn("problem fetching '%s' (will retry): %s" % (thread.url, ex)) if info is not None: info['Exception'] = ex if thread.tries < self.retry_limit: next_jobs.append((thread.url, thread.verify, thread.id, thread.tries + 1)) else: #traceback.print_exc(ex) log.error("retry limit exceeded for %s (last error was: %s)" % (thread.url, ex)) finally:
def consumer(q, njobs, stats, next_jobs=None, resolved=None): if next_jobs is None: next_jobs = [] if resolved is None: resolved = set() nfinished = 0 while nfinished < njobs: info = None try: log.debug("waiting for next thread to finish...") thread = q.get(True) thread.join(timeout) if thread.isAlive(): raise MetadataException( "thread timeout fetching '%s'" % thread.url) info = { 'Time Spent': thread.time() } if thread.ex is not None: raise thread.ex else: if thread.result is not None: info['Bytes'] = len(thread.result) else: raise MetadataException( "empty response fetching '%s'" % thread.url) info['Cached'] = thread.cached info['Date'] = str(thread.date) info['Last-Modified'] = str(thread.last_modified) info['Tries'] = thread.tries xml = thread.result.strip() if thread.status is not None: info['Status'] = thread.resp.status_code t = self.parse_metadata( StringIO(xml), key=thread.verify, base_url=thread.url) if t is None: self.fire(type=EVENT_IMPORT_FAIL, url=thread.url) raise MetadataException( "no valid metadata found at '%s'" % thread.url) relt = root(t) if relt.tag in ('{%s}XRD' % NS['xrd'], '{%s}XRDS' % NS['xrd']): log.debug("%s looks like an xrd document" % thread.url) for xrd in t.xpath("//xrd:XRD", namespaces=NS): log.debug("xrd: %s" % xrd) for link in xrd.findall(".//{%s}Link[@rel='%s']" % (NS['xrd'], NS['md'])): url = link.get("href") certs = xmlsec.CertDict(link) fingerprints = certs.keys() fp = None if len(fingerprints) > 0: fp = fingerprints[0] log.debug("fingerprint: %s" % fp) next_jobs.append((url, fp, url, 0)) elif relt.tag in ('{%s}EntityDescriptor' % NS['md'], '{%s}EntitiesDescriptor' % NS['md']): cacheDuration = self.default_cache_duration if self.respect_cache_duration: cacheDuration = root(t).get( 'cacheDuration', self.default_cache_duration) offset = duration2timedelta(cacheDuration) if thread.cached: if thread.last_modified + offset < datetime.now() - duration2timedelta(self.min_cache_ttl): raise MetadataException( "cached metadata expired") else: log.debug("found cached metadata for '%s' (last-modified: %s)" % (thread.url, thread.last_modified)) ne = self.import_metadata(t, url=thread.id) info['Number of Entities'] = ne else: log.debug("got fresh metadata for '%s' (date: %s)" % ( thread.url, thread.date)) ne = self.import_metadata(t, url=thread.id) info['Number of Entities'] = ne info['Cache Expiration Time'] = str( thread.last_modified + offset) certs = xmlsec.CertDict(relt) cert = None if certs.values(): cert = certs.values()[0].strip() resolved.add((thread.url, cert)) else: raise MetadataException( "unknown metadata type for '%s' (%s)" % (thread.url, relt.tag)) except Exception, ex: # traceback.print_exc(ex) log.warn("problem fetching '%s' (will retry): %s" % (thread.url, ex)) if info is not None: info['Exception'] = ex if thread.tries < self.retry_limit: next_jobs.append( (thread.url, thread.verify, thread.id, thread.tries + 1)) else: # traceback.print_exc(ex) log.error( "retry limit exceeded for %s (last error was: %s)" % (thread.url, ex)) finally:
def test_maintain_test01(self): store = self._redis_store() store.update(self.test01) entity_id = root(self.test01).get('entityID') assert (entity_id is not None) d = dict()
def parse_saml_metadata( source: BytesIO, opts: ResourceOpts, base_url=None, validation_errors: Optional[Dict[str, Any]] = None, ): """Parse a piece of XML and return an EntitiesDescriptor element after validation. :param source: a file-like object containing SAML metadata :param opts: ResourceOpts instance :param base_url: use this base url to resolve relative URLs for XInclude processing :param validation_errors: A dict that will be used to return validation errors to the caller :return: Tuple with t (ElementTree), expire_time_offset, exception """ if validation_errors is None: validation_errors = dict() try: t = parse_xml(source, base_url=base_url) if config.xinclude: t.xinclude() expire_time_offset = metadata_expiration(t) t = check_signature(t, opts.verify) if opts.cleanup is not None: for cb in opts.cleanup: t = cb(t) else: # at least get rid of ID attribute for e in iter_entities(t): if e.get('ID') is not None: del e.attrib['ID'] t = root(t) filter_invalid = opts.filter_invalid if opts.fail_on_error: filter_invalid = False if opts.validate_schema: t = filter_or_validate(t, filter_invalid=filter_invalid, base_url=base_url, source=source, validation_errors=validation_errors) if t is not None: if t.tag == "{%s}EntityDescriptor" % NS['md']: t = entitiesdescriptor([t], base_url, copy=False, validate=True, filter_invalid=filter_invalid, nsmap=t.nsmap) except Exception as ex: log.debug(traceback.format_exc()) log.error("Error parsing {}: {}".format(base_url, ex)) if opts.fail_on_error: raise ex return None, None, ex log.debug("returning %d valid entities" % len(list(iter_entities(t)))) return t, expire_time_offset, None
def test_clone(self): entity_id = root(self.t).get('entityID') self.md.store.update(root(self.t), entity_id) nstore = deepcopy(self.md.store) assert (nstore.size() == self.md.store.size()) assert (nstore.lookup(entity_id) is not None)
def test_utils(self): entity_id = root(self.t).get('entityID') self.md.store.update(root(self.t), entity_id) e = self.md.lookup(entity_id)[0] assert (is_idp(e)) assert (not is_sp(e)) icon = entity_icon_url(e) assert ('url' in icon) assert ('https://www.example.com/static/images/umu_logo.jpg' in icon['url']) assert ('width' in icon) assert ('358' == icon['width']) assert ('height' in icon) assert ('63' == icon['height']) assert ('62' != icon['height']) domains = entity_domains(e) assert ('example.com' in domains) assert ('example.net' in domains) assert ('idp.example.com' not in domains) assert ('foo.com' not in domains) edup = deepcopy(e) name, desc = entity_extended_display(e) assert (name == 'Example University') assert (desc == 'Identity Provider for Example University') disp = entity_display_name(e) assert (disp == 'Example University') for elt in e.findall(".//{%s}DisplayName" % NS['mdui']): elt.getparent().remove(elt) disp = entity_display_name(e) assert (disp == 'The Example University') for elt in e.findall(".//{%s}OrganizationDisplayName" % NS['md']): elt.getparent().remove(elt) disp = entity_display_name(e) assert (disp == 'ExampleU') for elt in e.findall(".//{%s}OrganizationName" % NS['md']): elt.getparent().remove(elt) disp = entity_display_name(e) assert (disp == entity_id) e = edup subs = entity_domains(e) assert ('example.com' in subs) assert ('example.net' in subs) assert ('idp.example.com' not in subs) summary = entity_simple_summary(e) assert (summary['title'] == 'Example University') assert (summary['descr'] == 'Identity Provider for Example University') assert (summary['entityID'] == entity_id) assert ('domains' in summary) assert ('id' in summary) empty = entity_simple_summary(None) assert (not empty)
def fork(req, *opts): """ Make a copy of the working tree and process the arguments as a pipleline. This essentially resets the working tree and allows a new plumbing to run. Useful for producing multiple outputs from a single source. :param req: The request :param opts: Options (unused) :return: None **Examples** .. code-block:: yaml - select # select all entities - fork: - certreport - publish: output: "/tmp/annotated.xml" - fork: - xslt: stylesheet: tidy.xml - publish: output: "/tmp/clean.xml" The second fork in this example is strictly speaking not necessary since the main plumbing is still active but it may help to structure your plumbings this way. **Merging** Normally the result of the "inner" plumbing is disgarded - unless published or emit:ed to a calling client in the case of the MDX server - but by adding 'merge' to the options with an optional 'merge strategy' the behaviour can be changed to merge the result of the inner pipeline back to the parent working document. The default merge strategy is 'replace_existing' which replaces each EntityDescriptor found in the resulting document in the parent document (using the entityID as a pointer). Any python module path ('a.mod.u.le.callable') ending in a callable is accepted. If the path doesn't contain a '.' then it is assumed to reference one of the standard merge strategies in pyff.merge_strategies. For instance the following block can be used to set an attribute on a single entity: .. code-block:: yaml - fork merge: - select: http://sp.example.com/shibboleth-sp - setattr: attribute: value Note that unless you have a select statement before your fork merge you'll be merging into an empty active document which with the default merge strategy of replace_existing will result in an empty active document. To avoid this do a select before your fork, thus: .. code-block:: yaml - select - fork merge: - select: http://sp.example.com/shibboleth-sp - setattr: attribute: value """ nt = None if req.t is not None: nt = deepcopy(req.t) ip = Plumbing(pipeline=req.args, pid="{}.fork".format(req.plumbing.pid)) # ip.process(req.md,t=nt) ireq = Plumbing.Request(ip, req.md, nt) ip._process(ireq) if req.t is not None and ireq.t is not None and len(root(ireq.t)) > 0: if 'merge' in opts: sn = "pyff.merge_strategies.replace_existing" if opts[-1] != 'merge': sn = opts[-1] req.md.merge(req.t, ireq.t, strategy_name=sn)
def test_missing(self): swamid = root(self.swamid) self.md.store.update(swamid, swamid.get('Name')) missing = self.md.lookup('https://connect.funet.fi/shibboleth+missing') assert (len(missing) == 0)