def parse(self, resource: Resource, content: str) -> SAMLParserInfo: info = SAMLParserInfo(description='SAML Metadata', expiration_time='') t, expire_time_offset, exception = parse_saml_metadata( unicode_stream(content), base_url=resource.url, opts=resource.opts, validation_errors=info.validation_errors, ) if expire_time_offset is not None: now = utc_now() now = now.replace(microsecond=0) expire_time = now + expire_time_offset resource.expire_time = expire_time info.expiration_time = str(expire_time) if t is not None: resource.t = t resource.type = "application/samlmetadata+xml" for e in iter_entities(t): info.entities.append(e.get('entityID')) if exception is not None: resource.info.exception = exception return info
def expiration(t): relt = root(t) if relt.tag in ('{%s}EntityDescriptor' % NS['md'], '{%s}EntitiesDescriptor' % NS['md']): cache_duration = config.default_cache_duration valid_until = relt.get('validUntil', None) if valid_until is not None: now = utc_now().replace(microsecond=0) vu = iso2datetime(valid_until) return vu - now elif config.respect_cache_duration: cache_duration = relt.get('cacheDuration', config.default_cache_duration) return duration2timedelta(cache_duration) return None
def parse(self, resource: Resource, content: str) -> ParserInfo: resource.children = deque() info = ParserInfo(description='Directory', expiration_time='never expires') n = 0 for fn in find_matching_files(content, self.extensions): child_opts = resource.opts.copy(update={'alias': None}) resource.add_child("file://" + fn, child_opts) n += 1 if n == 0: raise IOError("no entities found in {}".format(content)) resource.never_expires = True resource.expire_time = None resource.last_seen = utc_now().replace(microsecond=0) return info
def parse(self, resource: Resource, content: str) -> ParserInfo: info = ParserInfo(description='XRD links', expiration_time='never expires') t = parse_xml(unicode_stream(content)) relt = root(t) for xrd in t.iter("{%s}XRD" % NS['xrd']): for link in xrd.findall(".//{%s}Link[@rel='%s']" % (NS['xrd'], NS['md'])): link_href = link.get("href") certs = CertDict(link) fingerprints = list(certs.keys()) fp = None if len(fingerprints) > 0: fp = fingerprints[0] log.debug("XRD: {} verified by {}".format(link_href, fp)) child_opts = resource.opts.copy(update={'alias': None}) resource.add_child(link_href, child_opts) resource.last_seen = utc_now().replace(microsecond=0) resource.expire_time = None resource.never_expires = True return info
def set_pubinfo(e, publisher=None, creation_instant=None): if e.tag != "{%s}EntitiesDescriptor" % NS['md']: raise MetadataException( "I can only set RegistrationAuthority to EntitiesDescriptor elements" ) if publisher is None: raise MetadataException("At least publisher must be provided") if creation_instant is None: creation_instant = datetime2iso(utc_now()) ext = entity_extensions(e) pi = ext.find(".//{%s}PublicationInfo" % NS['mdrpi']) if pi is not None: raise MetadataException("A PublicationInfo element is already present") pi = etree.Element("{%s}PublicationInfo" % NS['mdrpi']) pi.set('publisher', publisher) if creation_instant: pi.set('creationInstant', creation_instant) ext.append(pi)
def parse(self, getter: Callable[[str], Response]) -> Deque[Resource]: data, status, info = self.load_resource(getter) if not data: raise ResourceException( f'Nothing to parse when loading resource {self}') info.state = ResourceLoadState.Parsing # local import to avoid circular import from pyff.parse import parse_resource info.parser_info = parse_resource(self, data) if status != 218: # write backup unless we just loaded from backup self.last_seen = utc_now().replace(microsecond=0) self.save_backup(data) info.state = ResourceLoadState.Parsed if self.t is not None: if self.post: for cb in self.post: if self.t is not None: self.t = cb(self.t, self.opts.dict()) if self.is_expired(): info.expired = True raise ResourceException("Resource at {} expired on {}".format( self.url, self.expire_time)) else: info.expired = False if info.parser_info: for (eid, error) in list( info.parser_info.validation_errors.items()): log.error(error) info.state = ResourceLoadState.Ready return self.children
def mkapp(*args: Any, **kwargs: Any) -> Any: md = kwargs.pop('md', None) if md is None: md = MDRepository() if config.devel_memory_profile: launch_memory_usage_server() with Configurator(debug_logger=log) as ctx: ctx.add_subscriber(add_cors_headers_response_callback, NewRequest) if config.aliases is None: config.aliases = dict() if config.modules is None: config.modules = [] ctx.registry.config = config config.modules.append('pyff.builtins') for mn in config.modules: importlib.import_module(mn) pipeline = None if args: pipeline = list(args) if pipeline is None and config.pipeline: pipeline = [config.pipeline] ctx.registry.scheduler = md.scheduler if pipeline is not None: ctx.registry.pipeline = pipeline ctx.registry.plumbings = [plumbing(v) for v in pipeline] ctx.registry.aliases = config.aliases ctx.registry.md = md if config.caching_enabled: ctx.registry.cache = TTLCache(config.cache_size, config.cache_ttl) else: ctx.registry.cache = NoCache() ctx.add_route('robots', '/robots.txt') ctx.add_view(robots_handler, route_name='robots') ctx.add_route('webfinger', '/.well-known/webfinger', request_method='GET') ctx.add_view(webfinger_handler, route_name='webfinger') ctx.add_route('search', '/api/search', request_method='GET') ctx.add_view(search_handler, route_name='search') ctx.add_route('status', '/api/status', request_method='GET') ctx.add_view(status_handler, route_name='status') ctx.add_route('resources', '/api/resources', request_method='GET') ctx.add_view(resources_handler, route_name='resources') ctx.add_route('pipeline', '/api/pipeline', request_method='GET') ctx.add_view(pipeline_handler, route_name='pipeline') ctx.add_route('call', '/api/call/{entry}', request_method=['POST', 'PUT']) ctx.add_view(process_handler, route_name='call') ctx.add_route('request', '/*path', request_method='GET') ctx.add_view(request_handler, route_name='request') start = utc_now() + timedelta(seconds=1) if config.update_frequency > 0: ctx.registry.scheduler.add_job( call, 'interval', id="call/update", args=['update'], start_date=start, misfire_grace_time=10, seconds=config.update_frequency, replace_existing=True, max_instances=1, timezone=pytz.utc, ) return ctx.make_wsgi_app()
def is_expired(self) -> bool: if self.never_expires or self.expire_time is None: return False return self.expire_time < utc_now()
def parse(self, resource: Resource, content: str) -> EidasMDParserInfo: info = EidasMDParserInfo(description='eIDAS MetadataServiceList', expiration_time='None') t = parse_xml(unicode_stream(content)) if config.xinclude: t.xinclude() relt = root(t) info.version = relt.get('Version', '0') info.issue_date = relt.get('IssueDate') info.next_update = relt.get('NextUpdate') if isinstance(info.next_update, str): resource.expire_time = iso2datetime(info.next_update) elif config.respect_cache_duration: duration = duration2timedelta(config.default_cache_duration) if not duration: # TODO: what is the right action here? raise ValueError( f'Invalid default cache duration: {config.default_cache_duration}' ) info.next_update = utc_now().replace(microsecond=0) + duration resource.expire_time = info.next_update info.expiration_time = 'None' if not resource.expire_time else resource.expire_time.isoformat( ) info.issuer_name = first_text(relt, "{%s}IssuerName" % NS['ser']) info.scheme_identifier = first_text(relt, "{%s}SchemeIdentifier" % NS['ser']) info.scheme_territory = first_text(relt, "{%s}SchemeTerritory" % NS['ser']) for mdl in relt.iter("{%s}MetadataList" % NS['ser']): for ml in mdl.iter("{%s}MetadataLocation" % NS['ser']): location = ml.get('Location') if location: certs = CertDict(ml) fingerprints = list(certs.keys()) fp = None if len(fingerprints) > 0: fp = fingerprints[0] ep = ml.find("{%s}Endpoint" % NS['ser']) if ep is not None and fp is not None: args = dict( country_code=mdl.get('Territory'), hide_from_discovery=strtobool( ep.get('HideFromDiscovery', 'false')), ) log.debug("MDSL[{}]: {} verified by {} for country {}". format(info.scheme_territory, location, fp, args.get('country_code'))) child_opts = resource.opts.copy(update={'alias': None}) child_opts.verify = fp r = resource.add_child(location, child_opts) # this is specific post-processing for MDSL files def _update_entities(_t, **kwargs): _country_code = kwargs.get('country_code') _hide_from_discovery = kwargs.get( 'hide_from_discovery') for e in iter_entities(_t): if _country_code: set_nodecountry(e, _country_code) if bool(_hide_from_discovery) and is_idp(e): set_entity_attributes( e, { ATTRS['entity-category']: 'http://refeds.org/category/hide-from-discovery' }) return _t r.add_via(Lambda(_update_entities, **args)) log.debug("Done parsing eIDAS MetadataServiceList") resource.last_seen = utc_now().replace(microsecond=0) resource.expire_time = None return info