def get_catalog_refs(self, base_url, rootElement=None):
"""Gets the catalogRef elements from the catalog XML document or optionally those within a
specified element.
@param rootElement - element within which to find dataset elements or None to find them
within the root catalog element.
@return list of catalogRef entities
"""
root = self.dom.documentElement if rootElement is None else rootElement
cr_elements = xml_util.getChildrenByNameNS(root, self.ns, 'catalogRef')
catalog_refs = []
for el in cr_elements:
ref_url = xml_util.getAttributeByLocalName(el, 'href')
url = url_util.make_url(base_url, ref_url)
catalog_ref = CatalogRef(
xml_util.getAttributeByLocalName(el, 'href'),
xml_util.getAttributeByLocalName(el, 'title'), url)
catalog_refs.append(catalog_ref)
# Recurse into contained datasets.
ds_elements = xml_util.getChildrenByNameNS(root, self.ns, 'dataset')
for ds_element in ds_elements:
catalog_refs.extend(self.get_catalog_refs(base_url, ds_element))
return catalog_refs
def get_catalog_refs(self, base_url, rootElement=None):
"""Gets the catalogRef elements from the catalog XML document or optionally those within a
specified element.
@param rootElement - element within which to find dataset elements or None to find them
within the root catalog element.
@return list of catalogRef entities
"""
root = self.dom.documentElement if rootElement is None else rootElement
cr_elements = xml_util.getChildrenByNameNS(root, self.ns, 'catalogRef')
catalog_refs = []
for el in cr_elements:
ref_url = xml_util.getAttributeByLocalName(el, 'href')
url = url_util.make_url(base_url, ref_url)
catalog_ref = CatalogRef(xml_util.getAttributeByLocalName(el, 'href'),
xml_util.getAttributeByLocalName(el, 'title'),
url)
catalog_refs.append(catalog_ref)
# Recurse into contained datasets.
ds_elements = xml_util.getChildrenByNameNS(root, self.ns, 'dataset')
for ds_element in ds_elements:
catalog_refs.extend(self.get_catalog_refs(base_url, ds_element))
return catalog_refs
def check_single_access(base_url, service, access, config, results):
"""Checks access to a THREDDS dataset with specified access details for a single service, i.e.,
for a a service type that is not Compound.
@param base_url - base URL of the catalog
@param service_map - map of service names to service configurations read from the catalog
@param access - dataset access details
@param config - configuration
@param results - accumulated results of check
"""
if service.service_type not in config.services_to_test:
results.add_untested()
results.add_untested_service_type(service.service_type)
if not config.quiet:
print("### Service of type %s not tested." % service.service_type)
return
access_allowed_count = 0
access_denied_count = 0
inconsistent = False
extensions = [''] if service.extensions is None else service.extensions
url = url_util.make_url(
base_url, service.base_url) + access.url_path + service.suffix
for extension in extensions:
extended_url = url + extension
(rc, msg, success) = httpget.check_url(extended_url, config)
if not config.quiet:
print(" %d [%s] %s" % (rc, msg, extended_url))
if service.is_public(extension):
if not success:
inconsistent = True
if not config.quiet:
print("### Expected access allowed.")
elif service.is_forbidden(extension):
if success:
inconsistent = True
if not config.quiet:
print("### Expected access denied.")
else:
if success:
access_allowed_count += 1
else:
access_denied_count += 1
if access_allowed_count + access_denied_count > 0:
if access_denied_count == 0:
results.add_access_allowed()
elif access_allowed_count == 0:
results.add_access_denied()
else:
inconsistent = True
if not config.quiet:
print(
"### Expected accesses either all allowed or all denied (excluding public and forbidden)."
)
if inconsistent:
results.add_inconsistent()
def check_single_access(base_url, service, access, config, results):
"""Checks access to a THREDDS dataset with specified access details for a single service, i.e.,
for a a service type that is not Compound.
@param base_url - base URL of the catalog
@param service_map - map of service names to service configurations read from the catalog
@param access - dataset access details
@param config - configuration
@param results - accumulated results of check
"""
if service.service_type not in config.services_to_test:
results.add_untested()
results.add_untested_service_type(service.service_type)
if not config.quiet:
print("### Service of type %s not tested." % service.service_type)
return
access_allowed_count = 0
access_denied_count = 0
inconsistent = False
extensions = [''] if service.extensions is None else service.extensions
url = url_util.make_url(base_url, service.base_url) + access.url_path + service.suffix
for extension in extensions:
extended_url = url + extension
(rc, msg, success) = httpget.check_url(extended_url, config)
if not config.quiet:
print(" %d [%s] %s" % (rc, msg, extended_url))
if service.is_public(extension):
if not success:
inconsistent = True
if not config.quiet:
print("### Expected access allowed.")
elif service.is_forbidden(extension):
if success:
inconsistent = True
if not config.quiet:
print("### Expected access denied.")
else:
if success:
access_allowed_count += 1
else:
access_denied_count += 1
if access_allowed_count + access_denied_count > 0:
if access_denied_count == 0:
results.add_access_allowed()
elif access_allowed_count == 0:
results.add_access_denied()
else:
inconsistent = True
if not config.quiet:
print("### Expected accesses either all allowed or all denied (excluding public and forbidden).")
if inconsistent:
results.add_inconsistent()
service_map[s.name] = s
for c in s.children:
service_map[c.name] = c
required_properties = config.required_container_properties + config.required_file_properties
for d in cat.get_datasets(required_properties):
if not config.quiet:
print("Dataset: %s" % d.name)
if not config.listonly:
check_dataset(service_map, d, config, results)
if config.recurse:
for d in cat.get_datasets(required_properties):
service = service_map.get(d.service_name, None)
if service and service.service_type in SERVICE_TYPES_RETURNING_CATALOGS:
ref_url = url_util.make_url(base_url, d.url_path)
if not config.quiet:
print("Dataset: %s Catalog ref: %s" % (d.name, ref_url))
_check_catalog(ref_url, config, results)
for c in cat.get_catalog_refs(base_url):
if not config.quiet:
print("Catalog ref: %s" % c.url)
_check_catalog(c.url, config, results)
def check_dataset(service_map, dataset, config, results):
"""Checks for rqeuired properties in a dataset in a THREDDS catalog.
@param service_map - map of service names to service configurations read from the catalog
@param dataset - dataset
@param config - configuration
@param results - accumulated results of check
service_map[s.name] = s
for c in s.children:
service_map[c.name] = c
required_properties = config.required_container_properties + config.required_file_properties
for d in cat.get_datasets(required_properties):
if not config.quiet:
print("Dataset: %s" % d.name)
if not config.listonly:
check_dataset(service_map, d, config, results)
if config.recurse:
for d in cat.get_datasets(required_properties):
service = service_map.get(d.service_name, None)
if service and service.service_type in SERVICE_TYPES_RETURNING_CATALOGS:
ref_url = url_util.make_url(base_url, d.url_path)
if not config.quiet:
print("Dataset: %s Catalog ref: %s" % (d.name, ref_url))
_check_catalog(ref_url, config, results)
for c in cat.get_catalog_refs(base_url):
if not config.quiet:
print("Catalog ref: %s" % c.url)
_check_catalog(c.url, config, results)
def check_dataset(service_map, dataset, config, results):
"""Checks for rqeuired properties in a dataset in a THREDDS catalog.
@param service_map - map of service names to service configurations read from the catalog
@param dataset - dataset
@param config - configuration