def test_invalid_load_prefix(self):
schema_file = '../data/hed_pairs/HED8.0.0t.xml'
hed_xml = os.path.join(os.path.dirname(os.path.abspath(__file__)), schema_file)
hed_schema1 = schema.load_schema(hed_xml)
hed_schema2 = schema.load_schema(hed_xml)
self.assertRaises(HedFileError, HedSchemaGroup, [hed_schema1, hed_schema2])
def get_schema(schema_path=None,
schema_url=None,
schema_string=None,
file_type=".xml"):
"""Return a schema object.
Parameters
----------
schema_path: str
A string representing a path to a schema
schema_url: str
A string representing a URL of a schema
schema_string: str
A string representing a URL of a schema
file_type: str
A string representing the file extension including the .
Returns
-------
HedSchema
The HedSchema object that as loaded.
"""
if schema_path:
hed_schema = hedschema.load_schema(schema_path)
elif schema_url:
hed_schema = hedschema.load_schema(schema_url)
elif schema_string:
hed_schema = hedschema.from_string(schema_string, file_type=file_type)
else:
raise HedFileError("HedSchemaNotFound",
"A HED schema could not be located", "")
return hed_schema
def test_process_services_sidecar(self):
from hedweb.services import process
json_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/bids_events.json')
with open(json_path) as f:
data = json.load(f)
json_text = json.dumps(data)
fb = io.StringIO(json_text)
schema_url = 'https://raw.githubusercontent.com/hed-standard/hed-specification/master/' \
+ 'hedxml/HED8.0.0.xml'
hed_schema = hedschema.load_schema(schema_url)
json_sidecar = models.Sidecar(file=fb, name='JSON_Sidecar')
arguments = {base_constants.SERVICE: 'sidecar_validate', base_constants.SCHEMA: hed_schema,
base_constants.COMMAND: 'validate', base_constants.COMMAND_TARGET: 'sidecar',
base_constants.JSON_SIDECAR: json_sidecar}
with self.app.app_context():
response = process(arguments)
self.assertFalse(response['error_type'],
'sidecar_validation services should not have a fatal error when file is invalid')
results = response['results']
self.assertEqual('success', results['msg_category'],
"sidecar_validation services has success on bids_events.json")
self.assertEqual('8.0.0', results[base_constants.SCHEMA_VERSION], 'Version 8.0.0 was used')
schema_url = 'https://raw.githubusercontent.com/hed-standard/hed-specification/master/' \
+ 'hedxml/HED7.2.0.xml'
arguments[base_constants.SCHEMA] = hedschema.load_schema(schema_url)
with self.app.app_context():
response = process(arguments)
self.assertFalse(response['error_type'],
'sidecar_validation services should not have a error when file is valid')
results = response['results']
self.assertTrue(results['data'], 'sidecar_validation produces errors when file not valid')
self.assertEqual('warning', results['msg_category'], "sidecar_validation did not valid with 7.2.0")
self.assertEqual('7.2.0', results['schema_version'], 'Version 7.2.0 was used')
def test_schema_convert(self):
from hedweb.schema import schema_convert
from hed import schema as hedschema
from hed.errors.exceptions import HedFileError
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
display_name = 'HED8.0.0.4.xml'
with self.app.app_context():
results = schema_convert(hed_schema, display_name)
self.assertTrue(results['data'],
"HED 8.0.0.xml can be converted to mediawiki")
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'data/HEDbad.xml')
display_name = 'HEDbad.xml'
with self.app.app_context():
try:
hed_schema = hedschema.load_schema(schema_path)
schema_convert(hed_schema, display_name)
except HedFileError:
pass
except Exception as ex:
self.fail(
f"schema_convert threw {type(ex).__name__} for invalid schema file header"
)
else:
self.fail(
'process should throw HedFileError when the schema file header was invalid'
)
def setUpClass(cls):
cls.xml_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
cls.xml_file)
cls.hed_schema_xml = schema.load_schema(cls.xml_file)
cls.wiki_file = os.path.join(
os.path.dirname(os.path.abspath(__file__)), cls.wiki_file)
cls.hed_schema_wiki = schema.load_schema(cls.wiki_file)
def setUpClass(cls):
schema_file = '../data/hed_pairs/HED8.0.0t.xml'
hed_xml = os.path.join(os.path.dirname(os.path.abspath(__file__)), schema_file)
hed_schema1 = schema.load_schema(hed_xml)
hed_schema2 = schema.load_schema(hed_xml, library_prefix="tl:")
cls.hed_schema = HedSchemaGroup([hed_schema1, hed_schema2])
cls.error_handler = error_reporter.ErrorHandler()
super().setUpClass()
def test_name(self):
invalid_xml_file = "invalidxmlfile.xml"
name = "PrettyDisplayName.xml"
try:
schema.load_schema(invalid_xml_file)
# We should have an error before we reach here.
self.assertTrue(False)
except HedFileError as e:
self.assertTrue(name in e.format_error_message(return_string_only=True, name=name))
def test_spreadsheet_validate_invalid_excel(self):
from hedweb.spreadsheet import spreadsheet_validate
spreadsheet_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'data/ExcelMultipleSheets.xlsx')
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
prefix_dict = {
2: "Property/Informational-property/Label/",
4: "Property/Informational-property/Description/"
}
spreadsheet = models.HedInput(spreadsheet_path,
worksheet_name='LKT Events',
tag_columns=[5],
has_column_names=True,
column_prefix_dictionary=prefix_dict,
name=spreadsheet_path)
with self.app.app_context():
results = spreadsheet_validate(hed_schema, spreadsheet)
self.assertTrue(
results['data'],
'spreadsheet_validate results should have empty data when errors'
)
self.assertEqual(
'warning', results['msg_category'],
'spreadsheet_validate msg_category should be warning when no errors'
)
def test_spreadsheet_validate_valid_excel(self):
from hedweb.spreadsheet import spreadsheet_validate
spreadsheet_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'data/ExcelMultipleSheets.xlsx')
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'data/HED8.0.0.xml')
prefix_dict = {
2: "Property/Informational-property/Label/",
4: "Property/Informational-property/Description/"
}
hed_schema = hedschema.load_schema(schema_path)
spreadsheet = models.HedInput(spreadsheet_path,
worksheet_name='LKT 8HED3A',
tag_columns=[5],
has_column_names=True,
column_prefix_dictionary=prefix_dict,
name=spreadsheet_path)
with self.app.app_context():
results = spreadsheet_validate(hed_schema, spreadsheet)
print(results['data'])
self.assertFalse(
results['data'],
'spreadsheet_validate results should not have a data key when no validation errors'
)
self.assertEqual(
'success', results["msg_category"],
'spreadsheet_validate msg_category should be success when no errors'
)
def get_hed_schema_from_pull_down(request):
"""Creates a HedSchema object from a section of form that uses a pull-down box and hed_cache
Parameters
----------
request: Request object
A Request object containing user data from a form.
Returns
-------
tuple: str
A HedSchema object
"""
if base_constants.SCHEMA_VERSION not in request.form:
raise HedFileError("NoSchemaError",
"Must provide a valid schema or schema version", "")
elif request.form[base_constants.
SCHEMA_VERSION] != base_constants.OTHER_VERSION_OPTION:
hed_file_path = hedschema.get_path_from_hed_version(
request.form[base_constants.SCHEMA_VERSION])
hed_schema = hedschema.load_schema(hed_file_path)
elif request.form[base_constants.SCHEMA_VERSION] == \
base_constants.OTHER_VERSION_OPTION and base_constants.SCHEMA_PATH in request.files:
f = request.files[base_constants.SCHEMA_PATH]
hed_schema = hedschema.from_string(
f.read(file_constants.BYTE_LIMIT).decode('ascii'),
file_type=secure_filename(f.filename))
else:
raise HedFileError(
"NoSchemaFile",
"Must provide a valid schema for upload if other chosen", "")
return hed_schema
def test_complex_file_validation_invalid(self):
schema_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'../data/validator_tests/bids_schema.mediawiki')
events_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'../data/validator_tests/bids_events.tsv')
hed_schema = schema.load_schema(schema_path)
json_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"../data/validator_tests/bids_events_bad_defs.json")
validator = HedValidator(hed_schema=hed_schema)
sidecar = Sidecar(json_path)
issues = sidecar.validate_entries(validators=validator,
check_for_warnings=True)
self.assertEqual(len(issues), 4)
input_file = EventsInput(events_path, sidecars=sidecar)
validation_issues = input_file.validate_file_sidecars(
validator, check_for_warnings=True)
self.assertEqual(len(validation_issues), 4)
validation_issues = input_file.validate_file(validator,
check_for_warnings=True)
self.assertEqual(len(validation_issues), 42)
def test_spreadsheet_process_validate_valid(self):
from hedweb.spreadsheet import process
spreadsheet_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'data/ExcelMultipleSheets.xlsx')
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
prefix_dict = {
2: "Property/Informational-property/Label/",
4: "Property/Informational-property/Description/"
}
spreadsheet = models.HedInput(spreadsheet_path,
worksheet_name='LKT 8HED3A',
tag_columns=[5],
has_column_names=True,
column_prefix_dictionary=prefix_dict,
name=spreadsheet_path)
arguments = {
base_constants.SCHEMA: hed_schema,
base_constants.SPREADSHEET: spreadsheet,
base_constants.COMMAND: base_constants.COMMAND_VALIDATE,
base_constants.CHECK_FOR_WARNINGS: True
}
arguments[base_constants.SCHEMA] = hed_schema
with self.app.app_context():
results = process(arguments)
self.assertTrue(isinstance(results, dict),
'process should return a dict when no errors')
self.assertEqual('success', results['msg_category'],
'process should return success if validated')
def test_schema2xml(self):
saved_filename = self.hed_schema_xml.save_as_xml()
try:
loaded_schema = schema.load_schema(saved_filename)
finally:
os.remove(saved_filename)
self.assertEqual(loaded_schema, self.hed_schema_xml)
def setUpClass(cls):
if cls.schema_file:
hed_xml = os.path.join(os.path.dirname(os.path.abspath(__file__)),
cls.schema_file)
cls.hed_schema = schema.load_schema(hed_xml)
elif not cls.hed_schema:
raise ValueError("No schema set for test case")
cls.error_handler = error_reporter.ErrorHandler()
def test_wikischema2wiki(self):
saved_filename = self.hed_schema_wiki.save_as_mediawiki()
try:
loaded_schema = schema.load_schema(saved_filename)
finally:
os.remove(saved_filename)
self.assertEqual(loaded_schema, self.hed_schema_wiki)
def test_hed_group_summary_str(self):
hed_url_path = 'https://raw.githubusercontent.com/hed-standard/hed-specification/master/hedxml/HED8.0.0.xml'
hed_schema = load_schema(hed_url_path)
sidecar = Sidecar(self.bids_path)
x = sidecar._column_data['hed_def_conds'].def_dict._defs
y = x['scrambled-face-cond']
test_summary = HedGroupSummary(y.contents, hed_schema, name=y.name, keep_all_values=True)
self.assertTrue(str(test_summary).startswith('Scrambled-face-cond'), 'HedGroupSummary has a string method')
def get_input_schema(parameters):
if base_constants.SCHEMA_STRING in parameters and parameters[
base_constants.SCHEMA_STRING]:
the_schema = hedschema.from_string(
parameters[base_constants.SCHEMA_STRING])
elif base_constants.SCHEMA_URL in parameters and parameters[
base_constants.SCHEMA_URL]:
schema_url = parameters[base_constants.SCHEMA_URL]
the_schema = hedschema.load_schema(schema_url)
elif base_constants.SCHEMA_VERSION in parameters and parameters[
base_constants.SCHEMA_VERSION]:
hed_file_path = hedschema.get_path_from_hed_version(
parameters[base_constants.SCHEMA_VERSION])
the_schema = hedschema.load_schema(hed_file_path)
else:
the_schema = []
return the_schema
def test_hed_group_summary_constructor(self):
hed_url_path = 'https://raw.githubusercontent.com/hed-standard/hed-specification/master/hedxml/HED8.0.0.xml'
hed_schema = load_schema(hed_url_path)
sidecar = Sidecar(self.bids_path)
x = sidecar._column_data['hed_def_conds'].def_dict._defs
y = x['scrambled-face-cond']
test_summary = HedGroupSummary(y.contents, hed_schema, name=y.name, keep_all_values=True)
self.assertEqual(test_summary.name, 'Scrambled-face-cond', 'HedGroupSummary has the right name')
self.assertEqual(len(test_summary.tag_dict), 5, 'HedGroupSummary has the right number of tags')
def test_schema_check(self):
from hedweb.schema import schema_validate
from hed import schema as hedschema
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
display_name = 'HED8.0.0.xml'
with self.app.app_context():
results = schema_validate(hed_schema, display_name)
self.assertTrue(results['data'],
"HED 8.0.0 is not fully HED-3G compliant")
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'data/HED8.0.1.xml')
hed_schema = hedschema.load_schema(schema_path)
display_name = 'HED8.0.1.xml'
with self.app.app_context():
results = schema_validate(hed_schema, display_name)
self.assertFalse(results['data'], "HED8.0.0 is HED-3G compliant")
def test_wikischema2xml(self):
saved_filename = self.hed_schema_wiki.save_as_xml()
try:
loaded_schema = schema.load_schema(saved_filename)
finally:
os.remove(saved_filename)
wiki_schema_copy = copy.deepcopy(self.hed_schema_wiki)
self.assertEqual(loaded_schema, wiki_schema_copy)
def test_string_validate(self):
from hedweb.strings import validate
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
string_list = [HedString('Red'), HedString('Blech')]
with self.app.app_context():
results = validate(hed_schema, string_list)
self.assertEqual('warning', results['msg_category'],
"validate has warning if validation errors")
string_list = [HedString('Red'), HedString('Blue')]
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
with self.app.app_context():
results = validate(hed_schema, string_list)
self.assertEqual('success', results['msg_category'],
"validate should return success if converted")
def test_string_convert_to_short_invalid(self):
from hedweb.strings import convert
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
string_list = [HedString('Red, Blech')]
with self.app.app_context():
results = convert(hed_schema, string_list)
self.assertEqual(
'warning', results['msg_category'],
"hedstring_convert issue warning if unsuccessful")
def test_sidecar_convert_to_short_invalid(self):
from hed import models
from hedweb.sidecar import sidecar_convert
json_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/bids_events_bad.json')
json_sidecar = models.Sidecar(file=json_path, name='bids_events_bad')
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
with self.app.app_context():
results = sidecar_convert(hed_schema, json_sidecar)
self.assertTrue(results['data'], 'sidecar_convert results should have data key')
self.assertEqual('warning', results['msg_category'],
'sidecar_convert msg_category should be warning for errors')
def test_def_mapping_single_line(self):
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'../data/hed_pairs/HED8.0.0.mediawiki')
hed_schema = schema.load_schema(schema_path)
validator = HedValidator(hed_schema=hed_schema)
def_mapper = DefinitionMapper()
string_with_def = \
'(Definition/TestDefPlaceholder/#,(Item/TestDef1/#,Item/TestDef2)), def/TestDefPlaceholder/2471'
test_string = HedString(string_with_def)
issues = test_string.validate([validator, def_mapper],
check_for_definitions=True)
self.assertEqual(len(issues), 0)
def test_sidecar_validate_valid(self):
from hed import models
from hedweb.sidecar import sidecar_validate
json_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/bids_events.json')
json_sidecar = models.Sidecar(file=json_path, name='bids_events')
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
with self.app.app_context():
results = sidecar_validate(hed_schema, json_sidecar)
self.assertFalse(results['data'],
'sidecar_validate results should not have a data key when no validation errors')
self.assertEqual('success', results["msg_category"],
'sidecar_validate msg_category should be success when no errors')
def test_invalid_schema(self):
for filename, error in self.files_and_errors.items():
full_filename = self.full_base_folder + filename
try:
loaded_schema = schema.load_schema(full_filename)
# all of these should produce exceptions.
self.assertFalse(True)
except HedFileError as e:
self.assertEqual(e.error_type, error)
if filename in self.expected_count:
self.assertEqual(len(e.issues), self.expected_count[filename])
pass
def test_sidecar_convert_to_long_valid(self):
from hed import models
from hedweb.sidecar import sidecar_convert
from hedweb.constants import base_constants
json_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/bids_events.json')
json_sidecar = models.Sidecar(file=json_path, name='bids_events')
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
with self.app.app_context():
results = sidecar_convert(hed_schema, json_sidecar, command=base_constants.COMMAND_TO_LONG)
self.assertTrue(results['data'],
'sidecar_convert to long results should have data key')
self.assertEqual('success', results["msg_category"],
'sidecar_convert to long msg_category should be success when no errors')
def test_events_assemble_valid(self):
from hedweb.events import assemble
events_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/bids_events.tsv')
json_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/bids_events.json')
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
json_sidecar = models.Sidecar(file=json_path, name='bids_json')
events = models.EventsInput(file=events_path, sidecars=json_sidecar, name='bids_events')
with self.app.app_context():
results = assemble(hed_schema, events, expand_defs=True)
self.assertTrue(results['data'],
'assemble results should have a data key when no errors')
self.assertEqual('success', results['msg_category'],
'assemble msg_category should be success when no errors')
def test_events_validate_invalid(self):
from hedweb.events import validate
events_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/bids_events.tsv')
json_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/bids_events_bad.json')
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
json_sidecar = models.Sidecar(file=json_path, name='bids_events_bad')
events = models.EventsInput(file=events_path, sidecars=json_sidecar, name='bids_events')
with self.app.app_context():
results = validate(hed_schema, events)
self.assertTrue(results['data'],
'validate results should have a data key when validation errors')
self.assertEqual('warning', results["msg_category"],
'validate msg_category should be warning when errors')
def test_string_convert_to_long(self):
from hedweb.strings import convert
string_list = [HedString('Red'), HedString('Blue')]
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
with self.app.app_context():
results = convert(hed_schema,
string_list,
command=base_constants.COMMAND_TO_LONG)
self.assertEqual(
'success', results['msg_category'],
"hedstring_convert should return success if converted")