def test_float(self):
val_in = [1, 2.2, 3.3, "4"]
val_odml = [1.0, 2.2, 3.3, 4.0]
parent_sec = self.doc.sections[0]
_ = odml.Property(name="float test", dtype="float",
value=val_in, parent=parent_sec)
# Test correct json save and load.
odml.save(self.doc, self.json_file, "JSON")
jdoc = odml.load(self.json_file, "JSON")
self.assertEqual(jdoc.sections[0].properties[0].dtype, odml.dtypes.DType.float)
self.assertEqual(jdoc.sections[0].properties[0].values, val_odml)
self.assertEqual(self.doc, jdoc)
# Test correct xml save and load.
odml.save(self.doc, self.xml_file)
xdoc = odml.load(self.xml_file)
self.assertEqual(xdoc.sections[0].properties[0].dtype, odml.dtypes.DType.float)
self.assertEqual(xdoc.sections[0].properties[0].values, val_odml)
self.assertEqual(self.doc, xdoc)
# Test correct yaml save and load.
odml.save(self.doc, self.yaml_file, "YAML")
ydoc = odml.load(self.yaml_file, "YAML")
self.assertEqual(ydoc.sections[0].properties[0].dtype, odml.dtypes.DType.float)
self.assertEqual(ydoc.sections[0].properties[0].values, val_odml)
self.assertEqual(self.doc, ydoc)
def run_conversion(file_list, output_dir, report, source_format="XML"):
"""
Convert a list of odML files to the latest odML version.
:param file_list: list of files to be converted.
:param output_dir: Directory where odML files converted to
the latest odML version will be saved.
:param report: Reporting StringIO.
:param source_format: Original file format of the odML source files.
XML, JSON and YAML are supported, default is XML.
"""
# Exceptions are kept as broad as possible to ignore any non-odML or
# invalid odML files and ensuring everything that can be will be converted.
for curr_file in file_list:
file_path = str(curr_file.absolute())
report.write("[Info] Handling file '%s'\n" % file_path)
# When loading the current file succeeds, it is
# a recent odML format file and can be ignored.
try:
odml.load(file_path, source_format)
report.write("[Info] Skip recent version file '%s'" % file_path)
except Exception as exc:
out_name = os.path.splitext(os.path.basename(file_path))[0]
outfile = os.path.join(output_dir, "%s_conv.xml" % out_name)
try:
VerConf(file_path).write_to_file(outfile, source_format)
except Exception as exc:
# Ignore files we cannot parse or convert
report.write("[Error] version converting file '%s': '%s'\n" %
(file_path, exc))
def _convert_file(cls, input_path, output_path, res_format):
"""
Converts a file from given input_path to res_format. Will raise a ValueError
if the provided output format (res_format) is not supported.
:param input_path: full path including file name of the file to be converted.
:param output_path: full path including file name of the output file.
If required the file extension will be adjusted to the
output format.
:param res_format: Format the input file will be converted to. Only formats
listed in constant CONVERSION_FORMATS are supported.
"""
if res_format == "v1_1":
VersionConverter(input_path).write_to_file(output_path)
elif res_format == "odml":
if not output_path.endswith(".odml"):
file_path, _ = os.path.splitext(output_path)
output_path = file_path + ".odml"
odml.save(odml.load(input_path), output_path)
elif res_format in CONVERSION_FORMATS:
if not output_path.endswith(CONVERSION_FORMATS[res_format]):
file_path, _ = os.path.splitext(output_path)
output_path = file_path + CONVERSION_FORMATS[res_format]
RDFWriter(odml.load(input_path)).write_file(
output_path, res_format)
else:
raise ValueError("Format for output files is incorrect. "
"Please choose from the list: {}".format(
list(CONVERSION_FORMATS)))
def test_tuple(self):
val_type = "3-tuple"
val_in = "(1; 1; 1)"
val_odml = ["1", "1", "1"]
parent_sec = self.doc.sections[0]
_ = odml.Property(name="tuple test single", dtype=val_type,
value=val_in, parent=parent_sec)
# Test correct json save and load.
odml.save(self.doc, self.json_file, "JSON")
jdoc = odml.load(self.json_file, "JSON")
self.assertEqual(jdoc.sections[0].properties[0].dtype, val_type)
self.assertEqual(jdoc.sections[0].properties[0].values, [val_odml])
self.assertEqual(self.doc, jdoc)
# Test correct xml save and load.
odml.save(self.doc, self.xml_file)
xdoc = odml.load(self.xml_file)
self.assertEqual(xdoc.sections[0].properties[0].dtype, val_type)
self.assertEqual(xdoc.sections[0].properties[0].values, [val_odml])
self.assertEqual(self.doc, xdoc)
# Test correct yaml save and load.
odml.save(self.doc, self.yaml_file, "YAML")
ydoc = odml.load(self.yaml_file, "YAML")
self.assertEqual(ydoc.sections[0].properties[0].dtype, val_type)
self.assertEqual(ydoc.sections[0].properties[0].values, [val_odml])
self.assertEqual(self.doc, ydoc)
def test_invalid_parser(self):
with self.assertRaises(NotImplementedError):
odml.load(self.file, 'html')
doc = odml.load(self.file)
with self.assertRaises(NotImplementedError):
odml.save(doc, self.file + '_copy_html', 'html')
with self.assertRaises(NotImplementedError):
odml.display(doc, 'html')
def test_invalid_parser(self):
with self.assertRaises(ValueError):
odml.load(self.file, 'html')
doc = odml.load(self.file)
with self.assertRaises(ValueError):
odml.save(doc, self.file + '_copy_html', 'html')
with self.assertRaises(ValueError):
odml.display(doc, 'html')
def test_invalid_parser(self):
with self.assertRaises(NotImplementedError):
odml.load(self.file, "html")
doc = odml.load(self.file)
with self.assertRaises(NotImplementedError):
file_name = "%s_copy_html" % self.file
odml.save(doc, file_name, "html")
with self.assertRaises(NotImplementedError):
odml.display(doc, "html")
def test_nix_to_odml_datetime(self):
file_name_1 = 'tmp' + str(uuid.uuid4())
nix_path_1 = os.path.join(self.test_dir, file_name_1 + '.nix')
nix_file_1 = nix.File.open(nix_path_1, nix.FileMode.Overwrite)
odml_path_1 = os.path.join(self.test_dir, file_name_1 + '.xml')
sec_1 = nix_file_1.create_section(name="section")
prop_1 = sec_1.create_property(name="datetime property",
values_or_dtype="datetime")
prop_1.values = [
'2011-11-01 11:11:11', '2012-12-02 02:02:02', '2012-12-03T03:03:03'
]
convert.odmlwrite(nix_file_1, odml_path_1)
odml_doc_1 = odml.load(odml_path_1)
odml_prop_1 = odml_doc_1.sections[0].props[0]
vals_1 = odml_prop_1.values
self.assertEqual(getattr(odml_prop_1, "dtype"), odml.DType.datetime)
self.assertEqual(len(vals_1), 3)
self.assertEqual(vals_1, [
datetime.datetime(2011, 11, 1, 11, 11, 11),
datetime.datetime(2012, 12, 2, 2, 2, 2),
datetime.datetime(2012, 12, 3, 3, 3, 3)
])
nix_file_1.close()
file_name_2 = 'tmp' + str(uuid.uuid4())
nix_path_2 = os.path.join(self.test_dir, file_name_2 + '.nix')
nix_file_2 = nix.File.open(nix_path_2, nix.FileMode.Overwrite)
odml_path_2 = os.path.join(self.test_dir, file_name_2 + '.xml')
sec_2 = nix_file_2.create_section(name="section")
prop_2 = sec_2.create_property(name="datetime property 2",
values_or_dtype=np.str_)
prop_2.values = [
'2012-12-02 12:12:12', '2013-01-01 01:01:01', '2013-01-02T02:02:02'
]
setattr(prop_2, "odml_type", nix.OdmlType("datetime"))
convert.odmlwrite(nix_file_2, odml_path_2)
odml_doc_2 = odml.load(odml_path_2)
odml_prop_2 = odml_doc_2.sections[0].props[0]
vals = odml_prop_2.values
self.assertEqual(getattr(odml_prop_2, "dtype"), odml.DType.datetime)
self.assertEqual(len(vals), 3)
self.assertEqual(vals, [
datetime.datetime(2012, 12, 2, 12, 12, 12),
datetime.datetime(2013, 1, 1, 1, 1, 1),
datetime.datetime(2013, 1, 2, 2, 2, 2)
])
nix_file_2.close()
def test_time(self):
time_string = '12:34:56'
time = dt.time(12, 34, 56)
val_in = time_string
vals_in = [None, "", time_string, time]
parent_sec = self.doc.sections[0]
_ = odml.Property(name="time test single", dtype="time",
value=val_in, parent=parent_sec)
_ = odml.Property(name="time test", dtype="time",
value=vals_in, parent=parent_sec)
# Test correct json save and load.
odml.save(self.doc, self.json_file, "JSON")
jdoc = odml.load(self.json_file, "JSON")
self.assertEqual(jdoc.sections[0].properties[0].dtype, odml.dtypes.DType.time)
self.assertIsInstance(jdoc.sections[0].properties[0].values[0], dt.time)
self.assertEqual(jdoc.sections[0].properties[1].dtype, odml.dtypes.DType.time)
for val in jdoc.sections[0].properties[1].value:
self.assertIsInstance(val, dt.time)
self.assertEqual(jdoc.sections[0].properties[1].values[2], time)
self.assertEqual(jdoc.sections[0].properties[1].values[3], time)
self.assertEqual(self.doc, jdoc)
# Test correct xml save and load.
odml.save(self.doc, self.xml_file)
xdoc = odml.load(self.xml_file)
self.assertEqual(xdoc.sections[0].properties[0].dtype, odml.dtypes.DType.time)
self.assertIsInstance(xdoc.sections[0].properties[0].values[0], dt.time)
self.assertEqual(xdoc.sections[0].properties[1].dtype, odml.dtypes.DType.time)
for val in xdoc.sections[0].properties[1].value:
self.assertIsInstance(val, dt.time)
self.assertEqual(xdoc.sections[0].properties[1].values[2], time)
self.assertEqual(xdoc.sections[0].properties[1].values[2], time)
self.assertEqual(self.doc, xdoc)
# Test correct yaml save and load.
odml.save(self.doc, self.yaml_file, "YAML")
ydoc = odml.load(self.yaml_file, "YAML")
self.assertEqual(ydoc.sections[0].properties[0].dtype, odml.dtypes.DType.time)
self.assertIsInstance(ydoc.sections[0].properties[0].values[0], dt.time)
self.assertEqual(ydoc.sections[0].properties[1].dtype, odml.dtypes.DType.time)
for val in ydoc.sections[0].properties[1].value:
self.assertIsInstance(val, dt.time)
self.assertEqual(ydoc.sections[0].properties[1].values[2], time)
self.assertEqual(ydoc.sections[0].properties[1].values[2], time)
self.assertEqual(self.doc, ydoc)
def test_date(self):
date_string = '2018-08-31'
date = dt.date(2018, 8, 31)
val_in = date_string
vals_in = [None, "", date_string, date]
parent_sec = self.doc.sections[0]
_ = odml.Property(name="date test single", dtype="date",
value=val_in, parent=parent_sec)
_ = odml.Property(name="date test", dtype="date",
value=vals_in, parent=parent_sec)
# Test correct json save and load.
odml.save(self.doc, self.json_file, "JSON")
jdoc = odml.load(self.json_file, "JSON")
self.assertEqual(jdoc.sections[0].properties[0].dtype, odml.dtypes.DType.date)
self.assertIsInstance(jdoc.sections[0].properties[0].values[0], dt.date)
self.assertEqual(jdoc.sections[0].properties[1].dtype, odml.dtypes.DType.date)
for val in jdoc.sections[0].properties[1].value:
self.assertIsInstance(val, dt.date)
self.assertEqual(jdoc.sections[0].properties[1].values[2], date)
self.assertEqual(jdoc.sections[0].properties[1].values[3], date)
self.assertEqual(self.doc, jdoc)
# Test correct xml save and load.
odml.save(self.doc, self.xml_file)
xdoc = odml.load(self.xml_file)
self.assertEqual(xdoc.sections[0].properties[0].dtype, odml.dtypes.DType.date)
self.assertIsInstance(xdoc.sections[0].properties[0].values[0], dt.date)
self.assertEqual(xdoc.sections[0].properties[1].dtype, odml.dtypes.DType.date)
for val in xdoc.sections[0].properties[1].value:
self.assertIsInstance(val, dt.date)
self.assertEqual(xdoc.sections[0].properties[1].values[2], date)
self.assertEqual(xdoc.sections[0].properties[1].values[2], date)
self.assertEqual(self.doc, xdoc)
# Test correct yaml save and load.
odml.save(self.doc, self.yaml_file, "YAML")
ydoc = odml.load(self.yaml_file, "YAML")
self.assertEqual(ydoc.sections[0].properties[0].dtype, odml.dtypes.DType.date)
self.assertIsInstance(ydoc.sections[0].properties[0].values[0], dt.date)
self.assertEqual(ydoc.sections[0].properties[1].dtype, odml.dtypes.DType.date)
for val in ydoc.sections[0].properties[1].value:
self.assertIsInstance(val, dt.date)
self.assertEqual(ydoc.sections[0].properties[1].values[2], date)
self.assertEqual(ydoc.sections[0].properties[1].values[2], date)
self.assertEqual(self.doc, ydoc)
def load(self, uri):
self.file_uri = uri
file_path = uri_to_path(uri)
parser = get_parser_for_uri(file_path)
try:
self.document = odml.load(file_path, parser)
except InvalidVersionException as inver:
_, curr_file = os.path.split(file_path)
err_header = "Cannot open file '%s'." % curr_file
err_msg = (
"You are trying to open an odML file of an outdated format. "
"\n\nUse 'File .. import' to convert and open files of "
"a previous odML format.")
ErrorDialog(self.window, err_header, err_msg)
self.window.set_welcome()
return False
except Exception as exc:
ErrorDialog(self.window, "Error parsing '%s'" % file_path,
str(exc))
self.window.set_welcome()
return False
self.document.finalize()
# Make sure all Properties within all sections are properly
# initialized with the "pseudo_values" attribute.
for sec in self.document.sections:
handle_section_import(sec)
self.window.registry.add(self.document)
self.window._info_bar.show_info("Loading of %s done!" %
(os.path.basename(file_path)))
return True
def check_result(self):
doc = load(self.outfile)
# Test document attribute export
self.assertEqual(doc.author, "author")
self.assertEqual(doc.version, "v1.13")
repo = "http://portal.g-node.org/odml/terminologies/v1.1/terminologies.xml"
self.assertEqual(doc.repository, repo)
self.assertEqual(len(doc.sections), 3)
sec = doc.sections["sec_one"]
self.assertEqual(sec.definition, "def s1")
self.assertEqual(sec.reference, "ref s1")
self.assertEqual(sec.type, "mainsec")
self.assertEqual(len(sec.sections), 1)
self.assertEqual(len(sec.properties), 3)
prop = sec.properties["prop_one"]
self.assertEqual(prop.definition, "def prop1")
self.assertEqual(prop.dependency, "dep p1")
self.assertEqual(prop.dependency_value, "dep val p1")
self.assertEqual(prop.reference, "ref val 1")
self.assertEqual(prop.uncertainty, "11")
self.assertEqual(prop.unit, "arbitrary")
self.assertEqual(prop.value_origin, "filename val 1")
self.assertEqual(prop.dtype, "string")
self.assertEqual(len(prop.values), 3)
prop = sec.properties["prop_two"]
self.assertEqual(len(prop.values), 8)
prop = sec.properties["prop_three"]
self.assertEqual(len(prop.values), 0)
def main(filename):
# Determine input and output format
file_base, file_ext = os.path.splitext(filename)
if file_ext in ['.xml', '.odml']:
output_format = '.nix'
elif file_ext in ['.nix']:
output_format = '.xml'
else:
raise ValueError('Unknown file format {}'.format(file_ext))
# Check output file
outfilename = file_base + output_format
if os.path.exists(outfilename):
yesno = input("File {} already exists. "
"Overwrite? ".format(outfilename))
if yesno.lower() not in ("y", "yes"):
print("Aborted")
return
# Load, convert and save to new format
print("Saving to {} file...".format(output_format), end=" ", flush=True)
if output_format in ['.nix']:
odml_doc = odml.load(filename)
nixwrite(odml_doc, outfilename)
elif output_format in ['.xml', '.odml']:
nix_file = nix.File.open(filename, nix.FileMode.ReadOnly)
odmlwrite(nix_file, outfilename)
else:
raise ValueError('Unknown file format {}'.format(output_format))
print("Done")
def _convert_file(cls, input_path, output_path, res_format):
"""
Convert a file from given input_path to res_format.
"""
if res_format == "v1_1":
VersionConverter(input_path).write_to_file(output_path)
elif res_format == "odml":
if not output_path.endswith(".odml"):
p, _ = os.path.splitext(output_path)
output_path = p + ".odml"
odml.save(odml.load(input_path), output_path)
elif res_format in cls._conversion_formats:
if not output_path.endswith(cls._conversion_formats[res_format]):
p, _ = os.path.splitext(output_path)
output_path = p + cls._conversion_formats[res_format]
RDFWriter(odml.load(input_path)).write_file(output_path, res_format)
def test_nix_to_odml_none(self):
file_name = 'tmp'
nix_path = os.path.join(self.test_dir, file_name + '.nix')
odml_path = os.path.join(self.test_dir, file_name + '.xml')
odml.Section(name=None,
oid=None,
definition=None,
parent=self.odml_doc,
type="MustBeSet",
reference=None,
repository=None,
link=None,
include=None)
convert.nixwrite(self.odml_doc, nix_path, nix.FileMode.Overwrite)
nix_file_r = nix.File.open(nix_path, nix.FileMode.ReadOnly)
convert.odmlwrite(nix_file_r, odml_path)
odml_doc = odml.load(odml_path)
self.assertEqual(len(odml_doc.sections), 1)
odml_sec = odml_doc.sections[0]
self.assertGreater(len(getattr(odml_sec, "id")), 0)
self.assertEqual(getattr(odml_sec, "name"), getattr(odml_sec, "id"))
self.assertEqual(getattr(odml_sec, "type"), "MustBeSet")
self.assertEqual(getattr(odml_sec, "definition"), None)
self.assertEqual(getattr(odml_sec, "reference"), None)
self.assertEqual(getattr(odml_sec, "repository"), None)
self.assertEqual(getattr(odml_sec, "link"), None)
self.assertEqual(getattr(odml_sec, "include"), None)
nix_file_r.close()
def test_save_kwargs(self):
doc = odml.load(self.file)
file_name = "%s_copy" % self.file
# Test unsupported kwarg does not raise an exception
odml.save(doc, file_name, unsupported_kwarg="I do not matter")
os.remove(file_name)
def test_nix_to_odml_empty(self):
file_name = 'tmp'
nix_path = os.path.join(self.test_dir, file_name + '.nix')
odml_path = os.path.join(self.test_dir, file_name + '.xml')
odml.Property(parent=self.odml_doc.sections[0])
convert.nixwrite(self.odml_doc, nix_path, nix.FileMode.Overwrite)
nix_file_r = nix.File.open(nix_path, nix.FileMode.ReadOnly)
convert.odmlwrite(nix_file_r, odml_path)
odml_doc = odml.load(odml_path)
self.assertEqual(len(odml_doc.sections[0].props), 1)
odml_prop = odml_doc.sections[0].props[0]
self.assertGreater(len(getattr(odml_prop, "id")), 0)
self.assertEqual(getattr(odml_prop, "name"), getattr(odml_prop, "id"))
self.assertEqual(getattr(odml_prop, "values"), [])
self.assertEqual(getattr(odml_prop, "dtype"), None)
self.assertEqual(getattr(odml_prop, "definition"), None)
self.assertEqual(getattr(odml_prop, "unit"), None)
self.assertEqual(getattr(odml_prop, "uncertainty"), None)
self.assertEqual(getattr(odml_prop, "reference"), None)
self.assertEqual(getattr(odml_prop, "dependency"), None)
self.assertEqual(getattr(odml_prop, "dependency_value"), None)
self.assertEqual(getattr(odml_prop, "value_origin"), None)
nix_file_r.close()
def check_result(self):
doc = load(self.outfile)
# Test document attribute export
self.assertEqual(doc.author, "author")
self.assertEqual(doc.version, "v1.13")
repo = "https://terminologies.g-node.org/v1.1/terminologies.xml"
self.assertEqual(doc.repository, repo)
self.assertEqual(len(doc.sections), 3)
sec = doc.sections["sec_one"]
self.assertEqual(sec.definition, "def s1")
self.assertEqual(sec.reference, "ref s1")
self.assertEqual(sec.type, "mainsec")
self.assertEqual(len(sec.sections), 1)
self.assertEqual(len(sec.properties), 3)
prop = sec.properties["prop_one"]
self.assertEqual(prop.definition, "def prop1")
self.assertEqual(prop.dependency, "dep p1")
self.assertEqual(prop.dependency_value, "dep val p1")
self.assertEqual(prop.reference, "ref val 1")
self.assertEqual(prop.uncertainty, "11")
self.assertEqual(prop.unit, "arbitrary")
self.assertEqual(prop.value_origin, "filename val 1")
self.assertEqual(prop.dtype, "string")
self.assertEqual(len(prop.values), 3)
prop = sec.properties["prop_two"]
self.assertEqual(len(prop.values), 8)
prop = sec.properties["prop_three"]
self.assertEqual(len(prop.values), 0)
def test_load_dtypes_yaml(self):
"""
Test if loading yaml document raises validation errors
for Properties with undefined dtypes.
"""
path = os.path.join(RES_DIR, "validation_dtypes.yaml")
doc = odml.load(path, "YAML")
self.load_dtypes_validation(doc)
def save_load(self):
"""
Helper method to save and load the current state of the document
with all supported parsers.
:return: jdoc ... document loaded from JSON file
xdoc ... document loaded from XML file
ydoc ... document loaded from YAML file
"""
odml.save(self.doc, self.json_file, "JSON")
jdoc = odml.load(self.json_file, "JSON")
odml.save(self.doc, self.xml_file)
xdoc = odml.load(self.xml_file)
odml.save(self.doc, self.yaml_file, "YAML")
ydoc = odml.load(self.yaml_file, "YAML")
return jdoc, xdoc, ydoc
def test_load_section_yaml(self):
"""
Test if loading yaml document raises validation errors for Sections with undefined type.
"""
path = os.path.join(RES_DIR, "validation_section.yaml")
doc = odml.load(path, "YAML")
self.load_section_validation(doc)
def save_load(self):
"""
Helper method to save and load the current state of the document
with all supported parsers.
:return: jdoc ... document loaded from JSON file
xdoc ... document loaded from XML file
ydoc ... document loaded from YAML file
"""
odml.save(self.doc, self.json_file, "JSON")
jdoc = odml.load(self.json_file, "JSON")
odml.save(self.doc, self.xml_file)
xdoc = odml.load(self.xml_file)
odml.save(self.doc, self.yaml_file, "YAML")
ydoc = odml.load(self.yaml_file, "YAML")
return jdoc, xdoc, ydoc
def test_load_dtypes_json(self):
"""
Test if loading json document raises validation errors
for Properties with undefined dtypes.
"""
path = os.path.join(RES_DIR, "validation_dtypes.json")
doc = odml.load(path, "JSON")
self.load_dtypes_validation(doc)
def test_load_section_json(self):
"""
Test if loading json document raises validation errors for Sections with undefined type.
"""
path = os.path.join(RES_DIR, "validation_section.json")
doc = odml.load(path, "JSON")
self.load_section_validation(doc)
def run_conversion(file_list,
output_dir,
rdf_dir,
report,
source_format="XML"):
"""
Convert a list of odML files to the latest odML version if required
and export all files to XML RDF files in a specified output directory.
:param file_list: list of files to be exported to RDF.
:param output_dir: Directory where odML files converted to
the latest odML version will be saved.
:param rdf_dir: Directory where exported RDF files will be saved.
:param report: Reporting StringIO.
:param source_format: Original file format of the odML source files.
XML, JSON and YAML are supported, default is XML.
"""
# Exceptions are kept as broad as possible to ignore any non-odML or
# invalid odML files and ensuring everything that can be will be converted.
for curr_file in file_list:
file_path = unicode(curr_file.absolute())
report.write("[Info] Handling file '%s'\n" % file_path)
# When loading the current file succeeds, it is
# a recent odML format file and can be exported
# to RDF right away. Otherwise it needs to be
# converted to the latest odML version first.
try:
odml.load(file_path, source_format)
report.write("[Info] RDF conversion of '%s'\n" % file_path)
run_rdf_export(file_path, rdf_dir)
except Exception as exc:
out_name = os.path.splitext(os.path.basename(file_path))[0]
outfile = os.path.join(output_dir, "%s_conv.xml" % out_name)
try:
VerConf(file_path).write_to_file(outfile, source_format)
try:
report.write("[Info] RDF conversion of '%s'\n" % outfile)
run_rdf_export(outfile, rdf_dir)
except Exception as exc:
report.write("[Error] converting '%s' to RDF: '%s'\n" %
(file_path, exc))
except Exception as exc:
# Ignore files we cannot parse or convert
report.write("[Error] version converting file '%s': '%s'\n" %
(file_path, exc))
def convert(filename, outfilename=None, mode='append'):
# Determine input and output format
file_base, file_ext = os.path.splitext(filename)
if file_ext in ['.xml', '.odml']:
output_format = '.nix'
elif file_ext in ['.nix']:
output_format = '.xml'
else:
raise ValueError('Unknown file format {}'.format(file_ext))
# odML files can not be appended but only be overwritten
if mode != 'overwrite' and output_format in ['.xml', '.odml']:
mode = 'overwrite'
# Check output file
if not outfilename:
outfilename = file_base
if not output_format in outfilename:
outfilename += output_format
if os.path.exists(outfilename):
yesno = input("File {} already exists. "
"{} (y/n)? ".format(outfilename, mode.title()))
if yesno.lower() not in ("y", "yes"):
print("Aborted")
return
# Load, convert and save to new format
print("Saving to {} file... ".format(output_format))
if output_format in ['.nix']:
try:
odml_doc = odml.load(filename)
except InvalidVersionException:
yesno = input("odML file format version is outdated. "
"Automatically convert {} to the latest version "
"(y/n)? ".format(outfilename))
if yesno.lower() not in ("y", "yes"):
print(" Use the odml.tools.VersionConverter to convert "
"to the latest odML file version.")
print(" Aborted")
return
xml_string = VersionConverter(filename).convert()
odml_doc = ODMLReader().from_string(xml_string)
nixwrite(odml_doc, outfilename, mode=mode)
elif output_format in ['.xml', '.odml']:
with nix.File.open(filename, nix.FileMode.ReadOnly) as nix_file:
odmlwrite(nix_file, outfilename)
else:
raise ValueError('Unknown file format {}'.format(output_format))
print("\nDone")
def test_nix_to_odml_string(self):
file_name_1 = 'tmp' + str(uuid.uuid4())
nix_path_1 = os.path.join(self.test_dir, file_name_1 + '.nix')
nix_file_1 = nix.File.open(nix_path_1, nix.FileMode.Overwrite)
odml_path_1 = os.path.join(self.test_dir, file_name_1 + '.xml')
sec_1 = nix_file_1.create_section(name="section")
prop_1 = sec_1.create_property(name="string property",
values_or_dtype=np.str_)
prop_1.values = ['a', 'b', 'c']
convert.odmlwrite(nix_file_1, odml_path_1)
odml_doc_1 = odml.load(odml_path_1)
odml_prop_1 = odml_doc_1.sections[0].props[0]
vals_1 = odml_prop_1.values
self.assertEqual(getattr(odml_prop_1, "dtype"), odml.DType.string)
self.assertEqual(len(vals_1), 3)
self.assertEqual(vals_1, ['a', 'b', 'c'])
nix_file_1.close()
file_name_2 = 'tmp' + str(uuid.uuid4())
nix_path_2 = os.path.join(self.test_dir, file_name_2 + '.nix')
nix_file_2 = nix.File.open(nix_path_2, nix.FileMode.Overwrite)
odml_path_2 = os.path.join(self.test_dir, file_name_2 + '.xml')
sec_2 = nix_file_2.create_section(name="section")
prop_2 = sec_2.create_property(name="string property 2",
values_or_dtype=np.str_)
prop_2.values = ['d', 'e', 'f']
setattr(prop_2, "odml_type", nix.OdmlType("string"))
convert.odmlwrite(nix_file_2, odml_path_2)
odml_doc_2 = odml.load(odml_path_2)
odml_prop_2 = odml_doc_2.sections[0].props[0]
vals = odml_prop_2.values
self.assertEqual(getattr(odml_prop_2, "dtype"), odml.DType.string)
self.assertEqual(len(vals), 3)
self.assertEqual(vals, ["d", "e", "f"])
nix_file_2.close()
def test_nix_to_odml_float(self):
file_name_1 = 'tmp' + str(uuid.uuid4())
nix_path_1 = os.path.join(self.test_dir, file_name_1 + '.nix')
nix_file_1 = nix.File.open(nix_path_1, nix.FileMode.Overwrite)
odml_path_1 = os.path.join(self.test_dir, file_name_1 + '.xml')
sec_1 = nix_file_1.create_section(name="section")
prop_1 = sec_1.create_property(name="float property",
values_or_dtype=np.float_)
prop_1.values = [1.1, 2.2, 3.3]
convert.odmlwrite(nix_file_1, odml_path_1)
odml_doc_1 = odml.load(odml_path_1)
odml_prop_1 = odml_doc_1.sections[0].props[0]
vals_1 = odml_prop_1.values
self.assertEqual(getattr(odml_prop_1, "dtype"), odml.DType.float)
self.assertEqual(len(vals_1), 3)
self.assertEqual(vals_1, [1.1, 2.2, 3.3])
nix_file_1.close()
file_name_2 = 'tmp' + str(uuid.uuid4())
nix_path_2 = os.path.join(self.test_dir, file_name_2 + '.nix')
nix_file_2 = nix.File.open(nix_path_2, nix.FileMode.Overwrite)
odml_path_2 = os.path.join(self.test_dir, file_name_2 + '.xml')
sec_2 = nix_file_2.create_section(name="section")
prop_2 = sec_2.create_property(name="string float property",
values_or_dtype=np.str_)
prop_2.values = ["4.4", "5.5", "6.6"]
convert.odmlwrite(nix_file_2, odml_path_2)
odml_doc_2 = odml.load(odml_path_2)
odml_prop_2 = odml_doc_2.sections[0].props[0]
vals = odml_prop_2.values
self.assertEqual(getattr(odml_prop_2, "dtype"), odml.DType.float)
self.assertEqual(len(vals), 3)
self.assertEqual(vals, [4.4, 5.5, 6.6])
nix_file_2.close()
'''
def test_sec_cardinality(self):
"""
Test saving and loading of Section sections cardinality variants to
and from all supported file formats.
"""
doc = odml.Document()
sec_empty = "card_empty"
sec_max = "card_max"
sec_min = "card_min"
sec_full = "card_full"
card_dict = {
sec_empty: None,
sec_max: (None, 10),
sec_min: (2, None),
sec_full: (1, 5)
}
_ = odml.Section(name=sec_empty, type="test", parent=doc)
_ = odml.Section(name=sec_max, sec_cardinality=card_dict[sec_max], type="test", parent=doc)
_ = odml.Section(name=sec_min, sec_cardinality=card_dict[sec_min], type="test", parent=doc)
_ = odml.Section(name=sec_full, sec_cardinality=card_dict[sec_full],
type="test", parent=doc)
# Test saving to and loading from an XML file
odml.save(doc, self.xml_file)
xml_doc = odml.load(self.xml_file)
self._test_cardinality_load("sec_cardinality", xml_doc, card_dict,
sec_empty, sec_max, sec_min, sec_full)
# Test saving to and loading from a JSON file
odml.save(doc, self.json_file, "JSON")
json_doc = odml.load(self.json_file, "JSON")
self._test_cardinality_load("sec_cardinality", json_doc, card_dict,
sec_empty, sec_max, sec_min, sec_full)
# Test saving to and loading from a YAML file
odml.save(doc, self.yaml_file, "YAML")
yaml_doc = odml.load(self.yaml_file, "YAML")
self._test_cardinality_load("sec_cardinality", yaml_doc, card_dict,
sec_empty, sec_max, sec_min, sec_full)
def test_nix_to_odml_tuple(self):
file_name_1 = 'tmp' + str(uuid.uuid4())
nix_path_1 = os.path.join(self.test_dir, file_name_1 + '.nix')
nix_file_1 = nix.File.open(nix_path_1, nix.FileMode.Overwrite)
odml_path_1 = os.path.join(self.test_dir, file_name_1 + '.xml')
sec_1 = nix_file_1.create_section(name="section")
prop_1 = sec_1.create_property(name="2-tuple property",
values_or_dtype=np.str_)
prop_1.values = ["(1; 2)", "(3; 4)"]
convert.odmlwrite(nix_file_1, odml_path_1)
odml_doc_1 = odml.load(odml_path_1)
odml_prop_1 = odml_doc_1.sections[0].props[0]
vals_1 = odml_prop_1.values
self.assertEqual(getattr(odml_prop_1, "dtype"), "2-tuple")
self.assertEqual(len(vals_1), 2)
self.assertEqual(vals_1, [["1", "2"], ["3", "4"]])
nix_file_1.close()
file_name_2 = 'tmp' + str(uuid.uuid4())
nix_path_2 = os.path.join(self.test_dir, file_name_2 + '.nix')
nix_file_2 = nix.File.open(nix_path_2, nix.FileMode.Overwrite)
odml_path_2 = os.path.join(self.test_dir, file_name_2 + '.xml')
sec_2 = nix_file_2.create_section(name="section")
prop_2 = sec_2.create_property(name="3-tuple property",
values_or_dtype=np.str_)
prop_2.values = ["(1; 2; 3)", "(4; 5; 6)"]
convert.odmlwrite(nix_file_2, odml_path_2)
odml_doc_2 = odml.load(odml_path_2)
odml_prop_2 = odml_doc_2.sections[0].props[0]
vals = odml_prop_2.values
self.assertEqual(getattr(odml_prop_2, "dtype"), "3-tuple")
self.assertEqual(len(vals), 2)
self.assertEqual(vals, [["1", "2", "3"], ["4", "5", "6"]])
nix_file_2.close()
def _convert_file(cls, input_path, output_path, res_format):
"""
Convert a file from given input_path to res_format.
"""
if res_format == "v1_1":
VersionConverter(input_path).write_to_file(output_path)
elif res_format == "odml":
if not output_path.endswith(".odml"):
p, _ = os.path.splitext(output_path)
output_path = p + ".odml"
odml.save(odml.load(input_path), output_path)
elif res_format in CONVERSION_FORMATS:
if not output_path.endswith(CONVERSION_FORMATS[res_format]):
p, _ = os.path.splitext(output_path)
output_path = p + CONVERSION_FORMATS[res_format]
RDFWriter(odml.load(input_path)).write_file(
output_path, res_format)
else:
raise ValueError("Format for output files is incorrect. "
"Please choose from the list: {}".format(
list(CONVERSION_FORMATS)))
def load_from_file(self, load_from):
"""
load the data for the table from an odml-file
:param load_from: Name of the odml-file to load from
:type load_from: string
"""
self._odmldoc = odml.load(load_from,
show_warnings=self.show_odml_warnings)
# resolve links and includes
self._odmldoc.finalize()
def run_conversion(file_list, output_dir, rdf_dir, report, source_format="XML"):
"""
Convert a list of odML files to the latest odML version if required
and export all files to XML RDF files in a specified output directory.
:param file_list: list of files to be exported to RDF.
:param output_dir: Directory where odML files converted to
the latest odML version will be saved.
:param rdf_dir: Directory where exported RDF files will be saved.
:param report: Reporting StringIO.
:param source_format: Original file format of the odML source files.
XML, JSON and YAML are supported, default is XML.
"""
# Exceptions are kept as broad as possible to ignore any non-odML or
# invalid odML files and ensuring everything that can be will be converted.
for curr_file in file_list:
file_path = unicode(curr_file.absolute())
report.write("[Info] Handling file '%s'\n" % file_path)
# When loading the current file succeeds, it is
# a recent odML format file and can be exported
# to RDF right away. Otherwise it needs to be
# converted to the latest odML version first.
try:
odml.load(file_path, source_format)
report.write("[Info] RDF conversion of '%s'\n" % file_path)
run_rdf_export(file_path, rdf_dir)
except Exception as exc:
out_name = os.path.splitext(os.path.basename(file_path))[0]
outfile = os.path.join(output_dir, "%s_conv.xml" % out_name)
try:
VerConf(file_path).write_to_file(outfile, source_format)
try:
report.write("[Info] RDF conversion of '%s'\n" % outfile)
run_rdf_export(outfile, rdf_dir)
except Exception as exc:
report.write("[Error] converting '%s' to RDF: '%s'\n" %
(file_path, exc))
except Exception as exc:
# Ignore files we cannot parse or convert
report.write("[Error] version converting file '%s': '%s'\n" %
(file_path, exc))
def test_bool(self):
val_in = True
vals_in = [None, "", [], {}, False, True, "TRUE"]
vals_odml = [False, False, False, False, False, True, True]
parent_sec = self.doc.sections[0]
_ = odml.Property(name="bool test single", dtype="boolean",
value=val_in, parent=parent_sec)
_ = odml.Property(name="bool test", dtype="boolean",
value=vals_in, parent=parent_sec)
# Test correct json save and load.
odml.save(self.doc, self.json_file, "JSON")
jdoc = odml.load(self.json_file, "JSON")
self.assertEqual(jdoc.sections[0].properties[0].dtype, odml.dtypes.DType.boolean)
self.assertEqual(jdoc.sections[0].properties[0].values, [val_in])
self.assertEqual(jdoc.sections[0].properties[1].dtype, odml.dtypes.DType.boolean)
self.assertEqual(jdoc.sections[0].properties[1].values, vals_odml)
self.assertEqual(self.doc, jdoc)
# Test correct xml save and load.
odml.save(self.doc, self.xml_file)
xdoc = odml.load(self.xml_file)
self.assertEqual(xdoc.sections[0].properties[0].dtype, odml.dtypes.DType.boolean)
self.assertEqual(xdoc.sections[0].properties[0].values, [val_in])
self.assertEqual(xdoc.sections[0].properties[1].dtype, odml.dtypes.DType.boolean)
self.assertEqual(xdoc.sections[0].properties[1].values, vals_odml)
self.assertEqual(self.doc, xdoc)
# Test correct yaml save and load.
odml.save(self.doc, self.yaml_file, "YAML")
ydoc = odml.load(self.yaml_file, "YAML")
self.assertEqual(ydoc.sections[0].properties[0].dtype, odml.dtypes.DType.boolean)
self.assertEqual(ydoc.sections[0].properties[0].values, [val_in])
self.assertEqual(ydoc.sections[0].properties[1].dtype, odml.dtypes.DType.boolean)
self.assertEqual(ydoc.sections[0].properties[1].values, vals_odml)
self.assertEqual(self.doc, ydoc)
def test_str(self):
val_in = "single value"
vals_in = [None, "", [], {}, 1, True, "text"]
vals_odml = ["", "", "", "", "1", "True", "text"]
parent_sec = self.doc.sections[0]
_ = odml.Property(name="string test single", dtype="string",
value=val_in, parent=parent_sec)
_ = odml.Property(name="string test", dtype="string",
value=vals_in, parent=parent_sec)
# Test correct json save and load.
odml.save(self.doc, self.json_file, "JSON")
jdoc = odml.load(self.json_file, "JSON")
self.assertEqual(jdoc.sections[0].properties[0].dtype, odml.dtypes.DType.string)
self.assertEqual(jdoc.sections[0].properties[0].values, [val_in])
self.assertEqual(jdoc.sections[0].properties[1].dtype, odml.dtypes.DType.string)
self.assertEqual(jdoc.sections[0].properties[1].values, vals_odml)
self.assertEqual(self.doc, jdoc)
# Test correct xml save and load.
odml.save(self.doc, self.xml_file)
xdoc = odml.load(self.xml_file)
self.assertEqual(xdoc.sections[0].properties[0].dtype, odml.dtypes.DType.string)
self.assertEqual(xdoc.sections[0].properties[0].values, [val_in])
self.assertEqual(xdoc.sections[0].properties[1].dtype, odml.dtypes.DType.string)
self.assertEqual(xdoc.sections[0].properties[1].values, vals_odml)
self.assertEqual(self.doc, xdoc)
# Test correct yaml save and load.
odml.save(self.doc, self.yaml_file, "YAML")
ydoc = odml.load(self.yaml_file, "YAML")
self.assertEqual(ydoc.sections[0].properties[0].dtype, odml.dtypes.DType.string)
self.assertEqual(ydoc.sections[0].properties[0].values, [val_in])
self.assertEqual(ydoc.sections[0].properties[1].dtype, odml.dtypes.DType.string)
self.assertEqual(ydoc.sections[0].properties[1].values, vals_odml)
self.assertEqual(self.doc, ydoc)
def test_display(self):
doc = odml.load(self.file)
odml.display(doc)
def test_load_save(self):
doc = odml.load(self.file)
self.assertTrue(isinstance(doc, odml.doc.BaseDocument))
odml.save(doc, self.file + '_copy')
os.remove(self.file + '_copy')
