def test_NXDL_item_catalog_creation(self):
cm = cache_manager.CacheManager()
fname = os.path.join(cm.default_file_set.path, 'nxdl.xsd')
self.assertTrue(os.path.exists(fname))
catalog = nxdl_schema.NXDL_item_catalog(fname)
self.assertTrue(isinstance(catalog, nxdl_schema.NXDL_item_catalog))
def __init__(self, path=None):
from punx import cache_manager
if path is None:
cm = cache_manager.CacheManager()
if cm is None or cm.default_file_set is None:
raise ValueError("Could not get NXDL file set from the cache")
path = cm.default_file_set.path
schema_file = os.path.join(path, "nxdl.xsd")
if not os.path.exists(schema_file):
raise FileNotFound(schema_file)
self.schema_file = schema_file
self.name = os.path.basename(path) # name of this cache
if not os.path.exists(self.schema_file):
raise FileNotFound("XML Schema file: " + self.schema_file)
self.lxml_tree = lxml.etree.parse(self.schema_file)
self.lxml_schema = lxml.etree.XMLSchema(self.lxml_tree)
self.lxml_root = self.lxml_tree.getroot()
nodes = self.lxml_root.xpath("xs:element", namespaces=self.ns)
if len(nodes) != 1:
raise InvalidNxdlFile(self.schema_file)
self.nxdl = Schema_Root(nodes[0],
ns_dict=self.ns,
schema_root=self.lxml_root,
schema_manager=self)
# cleanup these internal structures
del self.lxml_root
# del self.lxml_schema # needed for XML file validation
del self.lxml_tree
def func_configuration(args):
"""show internal configuration of punx"""
from . import cache_manager
cm = cache_manager.CacheManager()
print("Locally-available versions of NeXus definitions (NXDL files)")
print(cm.table_of_caches())
print("default NXDL file set: ", cm.default_file_set.ref)
def get_default_schema_manager():
"""
internal: convenience function
"""
from punx import cache_manager
cm = cache_manager.CacheManager()
assert (cm is not None)
assert (cm.default_file_set is not None)
return cm.default_file_set.schema_manager
def func_validate(args):
from punx import validate
if args.infile.endswith('.nxdl.xml'):
result = validate.validate_xml(args.infile)
if result is None:
print(args.infile, ' validates')
else:
try:
validator = validate.Data_File_Validator(args.infile)
except punx.FileNotFound:
exit_message('File not found: ' + args.infile)
except punx.HDF5_Open_Error:
exit_message('Could not open as HDF5: ' + args.infile)
except punx.SchemaNotFound as _exc:
exit_message(str(_exc))
# determine which findings are to be reported
report_choices, trouble = [], []
for c in args.report.upper().split(','):
if c in finding.VALID_STATUS_DICT:
report_choices.append(finding.VALID_STATUS_DICT[c])
else:
trouble.append(c)
if len(trouble) > 0:
msg = 'invalid choice(s) for *--report* option: '
msg += ','.join(trouble)
msg += '\n'
msg += '\t' + 'available choices: '
msg += ','.join(sorted(finding.VALID_STATUS_DICT.keys()))
exit_message(msg)
# run the validation
validator.validate() # lgtm [py/call/wrong-arguments]
# report the findings from the validation
from punx import cache_manager
cm = cache_manager.CacheManager()
print(':file: ' + os.path.basename(validator.fname))
msg = cm.default_file_set.ref_type
msg += ' ' + cm.default_file_set.ref
msg += ', ' + cache_manager.get_short_sha(cm.default_file_set.sha)
msg += ', ' + cm.default_file_set.cache + ' cache'
print(':NXDL cache info: ' + msg)
print(':NXDL GIT sha: ' + cm.default_file_set.sha)
print(':NXDL GIT date/time: ' + cm.default_file_set.last_modified)
print(':validation results shown: ' +
' '.join(sorted(map(str, report_choices))))
print('')
print('Validation findings')
print(validator.report_findings(report_choices))
print('summary statistics')
print(validator.report_findings_summary())
def func_validate(args):
"""
validate the content of a NeXus HDF5 data file of NXDL XML file
"""
from . import validate
cm = cache_manager.CacheManager()
if args.infile.endswith(".nxdl.xml"):
result = validate.validate_xml(args.infile)
if result is None:
print(args.infile, " validates")
return
file_sets = list(cm.all_file_sets.keys())
if args.file_set_name not in file_sets:
exit_message(
f"File set '{args.file_set_name}' is not available locally."
f" Either install it or use one of these: {', '.join(file_sets)}")
validator = validate.Data_File_Validator(args.file_set_name)
# determine which findings are to be reported
report_choices, trouble = [], []
for c in args.report.upper().split(","):
if c in finding.VALID_STATUS_DICT:
report_choices.append(c)
else:
trouble.append(c)
if len(trouble) > 0:
choices = ",".join(sorted(finding.VALID_STATUS_DICT.keys()))
exit_message(
f"invalid choice(s) for *--report* option: {','.join(trouble)}\n"
f"\t available choices: {choices}")
try:
# run the validation
validator.validate(args.infile)
except FileNotFound:
exit_message("File not found: " + args.infile)
except HDF5_Open_Error:
exit_message("Could not open as HDF5: " + args.infile)
except SchemaNotFound as _exc:
exit_message(str(_exc))
# report the findings from the validation
validator.print_report(statuses=report_choices)
print(f"NeXus definitions version: {args.file_set_name}")
def test_get_reference_keys(self):
cm = cache_manager.CacheManager()
fname = os.path.join(cm.default_file_set.path, 'nxdl.xsd')
self.assertTrue(os.path.exists(fname))
tree = lxml.etree.parse(fname)
ns = nxdl_schema.get_xml_namespace_dictionary()
query = '//xs:element/xs:complexType'
nodes = tree.getroot().xpath(query, namespaces=ns)
self.assertEqual(len(nodes), 3)
section, line = nxdl_schema.get_reference_keys(nodes[0])
self.assertTrue(isinstance(section, str))
self.assertTrue(isinstance(line, str))
self.assertEqual(section, query.split(':')[-1])
self.assertEqual(line.split()[0], 'Line')
def func_install(args):
"""
Install or update the named versions of the NeXus definitions.
Install into the user cache. (Developer manages the source cache.)
"""
from . import cache_manager
cm = cache_manager.CacheManager()
cache_dir = pathlib.Path(cm.user.path)
for file_set_name in args.file_set_name:
logger.info("cache_manager.download_file_set('%s', '%s', force=%s)",
file_set_name, cache_dir, args.update)
cache_manager.download_file_set(file_set_name,
cache_dir,
replace=args.update)
print(cm.table_of_caches())
print(f"default file set: {cm.default_file_set.ref}")
def test_get_named_parent_node(self):
cm = cache_manager.CacheManager()
fname = os.path.join(cm.default_file_set.path, 'nxdl.xsd')
self.assertTrue(os.path.exists(fname))
tree = lxml.etree.parse(fname)
ns = nxdl_schema.get_xml_namespace_dictionary()
query = '//xs:complexType//xs:element'
nodes = tree.getroot().xpath(query, namespaces=ns)
self.assertGreater(len(nodes), 0)
xml_node = nodes[0]
self.assertTrue(isinstance(xml_node, lxml.etree._Element))
parent_node = nxdl_schema.get_named_parent_node(xml_node)
self.assertTrue(isinstance(parent_node, lxml.etree._Element))
self.assertTrue('name' in parent_node.attrib)
query = '/xs:schema/xs:element'
nodes = tree.getroot().xpath(query, namespaces=ns)
self.assertEqual(len(nodes), 1)
parent_node = nxdl_schema.get_named_parent_node(nodes[0])
self.assertTrue(parent_node.tag.endswith('}schema'))
def parse_nxdlTypes(self):
"""
get the allowed data types and unit types from nxdlTypes.xsd
"""
if os.path.exists(self.schema_file):
path = os.path.dirname(self.schema_file)
else:
from punx import cache_manager
cm = cache_manager.CacheManager()
if cm is None or cm.default_file_set is None:
raise ValueError("Could not get NXDL file set from the cache")
path = cm.default_file_set.path
self.types_file = os.path.join(path, "nxdlTypes.xsd")
if not os.path.exists(self.types_file):
raise FileNotFound(self.types_file)
lxml_types_tree = lxml.etree.parse(self.types_file)
db = {}
root = lxml_types_tree.getroot()
for node in root:
if isinstance(node, lxml.etree._Comment):
pass
elif node.tag.endswith("}annotation"):
pass
else:
obj = Schema_nxdlType(node, ns_dict=self.ns, schema_root=root)
if obj.name is not None:
db[obj.name] = obj
# re-arrange
units = list(db["anyUnitsAttr"].values or [])
del db["anyUnitsAttr"]
del db["primitiveType"]
return db, units
def parse_command_line_arguments():
"""process command line"""
from . import cache_manager
cm = cache_manager.CacheManager()
doc = __doc__.strip().splitlines()[0]
doc += "\n version: " + __version__
doc += "\n URL: " + __url__
epilog = "Note: It is only necessary to use the first two (or"
epilog += " more) characters of any subcommand, enough that the"
epilog += " abbreviation is unique. "
epilog += " Such as: ``demonstrate`` can be abbreviated to"
epilog += " ``demo`` or even ``de``."
p = MyArgumentParser(prog=__package_name__, description=doc, epilog=epilog)
p.add_argument("-v", "--version", action="version", version=__version__)
# TODO: issue #9, stretch goal: GUI for any of this
# p.add_argument(
# '-g',
# '--gui',
# help='graphical user interface (TBA)')
subcommand = p.add_subparsers(
title="subcommand",
description="valid subcommands",
)
# --- subcommand: configuration
help_text = "show configuration details of punx"
p_sub = subcommand.add_parser("configuration", help=help_text)
p_sub.set_defaults(func=func_configuration)
# --- subcommand: demo
p_sub = subcommand.add_parser("demonstrate",
help="demonstrate HDF5 file validation")
# TODO: add_logging_argument(p_sub)
p_sub.set_defaults(func=func_demo)
# # --- subcommand hierarchy
# # TODO: issue #1 & #10
# help_text = 'show NeXus base class hierarchy from a given base class'
# p_sub = subcommand.add_parser('hierarchy', help=help_text)
# p_sub.set_defaults(func=func_hierarchy)
# #p_sub.add_argument('something', type=bool, help='something help_text')
# --- subcommand: install
help_text = "install NeXus definitions into the local cache"
p_sub = subcommand.add_parser("install", help=help_text)
p_sub.set_defaults(func=func_install)
help_text = "name(s) of reference NeXus NXDL file set"
help_text += f" -- default={cache_manager.GITHUB_NXDL_BRANCH}"
p_sub.add_argument("file_set_name",
default=[cache_manager.GITHUB_NXDL_BRANCH],
nargs="*",
help=help_text)
p_sub.add_argument(
"-u",
"--update",
action="store_true",
default=False,
help=
"force existing file set to update from NeXus repository on GitHub",
)
# TODO: add_logging_argument(p_sub)
# --- subcommand: tree
help_text = "show tree structure of HDF5 or NXDL file"
p_sub = subcommand.add_parser("tree", help=help_text)
p_sub.set_defaults(func=func_tree)
p_sub.add_argument("infile", help="HDF5 or NXDL file name")
p_sub.add_argument(
"-a",
action="store_false",
default=True,
dest="show_attributes",
help="Do not print attributes of HDF5 file structure",
)
help_text = "maximum number of array items to be shown"
p_sub.add_argument(
"-m",
"--max_array_items",
default=5,
type=int,
# choices=range(1,51),
help=help_text,
)
# TODO: add_logging_argument(p_sub)
# --- subcommand: validate
p_sub = subcommand.add_parser("validate", help="validate a NeXus file")
p_sub.add_argument("infile", help="HDF5 or NXDL file name")
p_sub.set_defaults(func=func_validate)
help_text = "NeXus NXDL file set (definitions) name for validation"
help_text += f" -- default={cm.default_file_set.ref}"
p_sub.add_argument(
"-f",
"--file_set_name",
default=cm.default_file_set.ref,
# nargs="*",
help=help_text)
reporting_choices = ",".join(sorted(finding.VALID_STATUS_DICT.keys()))
help_text = (
"select which validation findings to report, "
f"choices: {reporting_choices}"
" (separate with comma if more than one, do not use white space)")
p_sub.add_argument("--report", default=reporting_choices, help=help_text)
# TODO: add_logging_argument(p_sub)
return p.parse_args()