def get_catalog_refs(self, base_url, rootElement=None): """Gets the catalogRef elements from the catalog XML document or optionally those within a specified element. @param rootElement - element within which to find dataset elements or None to find them within the root catalog element. @return list of catalogRef entities """ root = self.dom.documentElement if rootElement is None else rootElement cr_elements = xml_util.getChildrenByNameNS(root, self.ns, 'catalogRef') catalog_refs = [] for el in cr_elements: ref_url = xml_util.getAttributeByLocalName(el, 'href') url = url_util.make_url(base_url, ref_url) catalog_ref = CatalogRef( xml_util.getAttributeByLocalName(el, 'href'), xml_util.getAttributeByLocalName(el, 'title'), url) catalog_refs.append(catalog_ref) # Recurse into contained datasets. ds_elements = xml_util.getChildrenByNameNS(root, self.ns, 'dataset') for ds_element in ds_elements: catalog_refs.extend(self.get_catalog_refs(base_url, ds_element)) return catalog_refs
def get_catalog_refs(self, base_url, rootElement=None): """Gets the catalogRef elements from the catalog XML document or optionally those within a specified element. @param rootElement - element within which to find dataset elements or None to find them within the root catalog element. @return list of catalogRef entities """ root = self.dom.documentElement if rootElement is None else rootElement cr_elements = xml_util.getChildrenByNameNS(root, self.ns, 'catalogRef') catalog_refs = [] for el in cr_elements: ref_url = xml_util.getAttributeByLocalName(el, 'href') url = url_util.make_url(base_url, ref_url) catalog_ref = CatalogRef(xml_util.getAttributeByLocalName(el, 'href'), xml_util.getAttributeByLocalName(el, 'title'), url) catalog_refs.append(catalog_ref) # Recurse into contained datasets. ds_elements = xml_util.getChildrenByNameNS(root, self.ns, 'dataset') for ds_element in ds_elements: catalog_refs.extend(self.get_catalog_refs(base_url, ds_element)) return catalog_refs
def check_single_access(base_url, service, access, config, results): """Checks access to a THREDDS dataset with specified access details for a single service, i.e., for a a service type that is not Compound. @param base_url - base URL of the catalog @param service_map - map of service names to service configurations read from the catalog @param access - dataset access details @param config - configuration @param results - accumulated results of check """ if service.service_type not in config.services_to_test: results.add_untested() results.add_untested_service_type(service.service_type) if not config.quiet: print("### Service of type %s not tested." % service.service_type) return access_allowed_count = 0 access_denied_count = 0 inconsistent = False extensions = [''] if service.extensions is None else service.extensions url = url_util.make_url( base_url, service.base_url) + access.url_path + service.suffix for extension in extensions: extended_url = url + extension (rc, msg, success) = httpget.check_url(extended_url, config) if not config.quiet: print(" %d [%s] %s" % (rc, msg, extended_url)) if service.is_public(extension): if not success: inconsistent = True if not config.quiet: print("### Expected access allowed.") elif service.is_forbidden(extension): if success: inconsistent = True if not config.quiet: print("### Expected access denied.") else: if success: access_allowed_count += 1 else: access_denied_count += 1 if access_allowed_count + access_denied_count > 0: if access_denied_count == 0: results.add_access_allowed() elif access_allowed_count == 0: results.add_access_denied() else: inconsistent = True if not config.quiet: print( "### Expected accesses either all allowed or all denied (excluding public and forbidden)." ) if inconsistent: results.add_inconsistent()
def check_single_access(base_url, service, access, config, results): """Checks access to a THREDDS dataset with specified access details for a single service, i.e., for a a service type that is not Compound. @param base_url - base URL of the catalog @param service_map - map of service names to service configurations read from the catalog @param access - dataset access details @param config - configuration @param results - accumulated results of check """ if service.service_type not in config.services_to_test: results.add_untested() results.add_untested_service_type(service.service_type) if not config.quiet: print("### Service of type %s not tested." % service.service_type) return access_allowed_count = 0 access_denied_count = 0 inconsistent = False extensions = [''] if service.extensions is None else service.extensions url = url_util.make_url(base_url, service.base_url) + access.url_path + service.suffix for extension in extensions: extended_url = url + extension (rc, msg, success) = httpget.check_url(extended_url, config) if not config.quiet: print(" %d [%s] %s" % (rc, msg, extended_url)) if service.is_public(extension): if not success: inconsistent = True if not config.quiet: print("### Expected access allowed.") elif service.is_forbidden(extension): if success: inconsistent = True if not config.quiet: print("### Expected access denied.") else: if success: access_allowed_count += 1 else: access_denied_count += 1 if access_allowed_count + access_denied_count > 0: if access_denied_count == 0: results.add_access_allowed() elif access_allowed_count == 0: results.add_access_denied() else: inconsistent = True if not config.quiet: print("### Expected accesses either all allowed or all denied (excluding public and forbidden).") if inconsistent: results.add_inconsistent()
service_map[s.name] = s for c in s.children: service_map[c.name] = c required_properties = config.required_container_properties + config.required_file_properties for d in cat.get_datasets(required_properties): if not config.quiet: print("Dataset: %s" % d.name) if not config.listonly: check_dataset(service_map, d, config, results) if config.recurse: for d in cat.get_datasets(required_properties): service = service_map.get(d.service_name, None) if service and service.service_type in SERVICE_TYPES_RETURNING_CATALOGS: ref_url = url_util.make_url(base_url, d.url_path) if not config.quiet: print("Dataset: %s Catalog ref: %s" % (d.name, ref_url)) _check_catalog(ref_url, config, results) for c in cat.get_catalog_refs(base_url): if not config.quiet: print("Catalog ref: %s" % c.url) _check_catalog(c.url, config, results) def check_dataset(service_map, dataset, config, results): """Checks for rqeuired properties in a dataset in a THREDDS catalog. @param service_map - map of service names to service configurations read from the catalog @param dataset - dataset @param config - configuration @param results - accumulated results of check
service_map[s.name] = s for c in s.children: service_map[c.name] = c required_properties = config.required_container_properties + config.required_file_properties for d in cat.get_datasets(required_properties): if not config.quiet: print("Dataset: %s" % d.name) if not config.listonly: check_dataset(service_map, d, config, results) if config.recurse: for d in cat.get_datasets(required_properties): service = service_map.get(d.service_name, None) if service and service.service_type in SERVICE_TYPES_RETURNING_CATALOGS: ref_url = url_util.make_url(base_url, d.url_path) if not config.quiet: print("Dataset: %s Catalog ref: %s" % (d.name, ref_url)) _check_catalog(ref_url, config, results) for c in cat.get_catalog_refs(base_url): if not config.quiet: print("Catalog ref: %s" % c.url) _check_catalog(c.url, config, results) def check_dataset(service_map, dataset, config, results): """Checks for rqeuired properties in a dataset in a THREDDS catalog. @param service_map - map of service names to service configurations read from the catalog @param dataset - dataset @param config - configuration