def validate(hed_schema, events, sidecar=None, check_for_warnings=False):
"""Validates and events input object and returns the results.
Parameters
----------
hed_schema: str or HedSchema
Version number or path or HedSchema object to be used
events: EventsInput
Events input object to be validated
sidecar: Sidecar
Representation of a BIDS JSON sidecar object
check_for_warnings: bool
If true, validation should include warnings
Returns
-------
dict
A dictionary containing results of validation in standard format
"""
schema_version = hed_schema.header_attributes.get('version',
'Unknown version')
display_name = events.name
validator = HedValidator(hed_schema=hed_schema)
issue_str = ''
if sidecar:
issues = sidecar.validate_entries(
validator, check_for_warnings=check_for_warnings)
if issues:
issue_str = issue_str + get_printable_issue_string(
issues, title="Sidecar definition errors:")
issues = events.validate_file(validator,
check_for_warnings=check_for_warnings)
if issues:
issue_str = issue_str + get_printable_issue_string(
issues, title="Event file errors:")
if issue_str:
file_name = generate_filename(display_name,
name_suffix='_validation_errors',
extension='.txt')
return {
base_constants.COMMAND: base_constants.COMMAND_VALIDATE,
'data': issue_str,
"output_display_name": file_name,
base_constants.SCHEMA_VERSION: schema_version,
"msg_category": "warning",
'msg': f"Events file {display_name} had validation errors"
}
else:
return {
base_constants.COMMAND: base_constants.COMMAND_VALIDATE,
'data': '',
base_constants.SCHEMA_VERSION: schema_version,
'msg_category': 'success',
'msg': f"Events file {display_name} had no validation errors"
}
def validate(hed_schema, string_list, check_for_warnings=False):
"""Validates a list of strings and returns a dictionary containing the issues or a no errors message
Parameters
----------
hed_schema: HedSchema
The HED schema to be used in processing
string_list: list
A list of string to be processed
check_for_warnings: bool
Indicates whether validation should check for warnings as well as errors
Returns
-------
dict
A dictionary with results
"""
schema_version = hed_schema.header_attributes.get('version',
'Unknown version')
hed_validator = HedValidator(hed_schema=hed_schema)
validation_errors = []
for pos, h_string in enumerate(string_list, start=1):
issues = h_string.validate(hed_validator,
check_for_warnings=check_for_warnings)
if issues:
validation_errors.append(
get_printable_issue_string(issues,
f"Errors for HED string {pos}:"))
if validation_errors:
return {
base_constants.COMMAND: base_constants.COMMAND_VALIDATE,
'data': validation_errors,
base_constants.SCHEMA_VERSION: schema_version,
'msg_category': 'warning',
'msg': 'Strings had validation errors'
}
else:
return {
base_constants.COMMAND: base_constants.COMMAND_VALIDATE,
'data': '',
base_constants.SCHEMA_VERSION: schema_version,
'msg_category': 'success',
'msg': 'Strings validated successfully...'
}
def sidecar_validate(hed_schema, json_sidecar, check_for_warnings=False):
""" Validates the sidecar and returns the errors and/or a message in a dictionary
Parameters
----------
hed_schema: str or HedSchema
Version number or path or HedSchema object to be used
json_sidecar: Sidecar
Dictionary object
check_for_warnings: bool
Indicates whether validation should check for warnings as well as errors
Returns
-------
dict
dictionary of response values.
"""
schema_version = hed_schema.header_attributes.get('version',
'Unknown version')
display_name = json_sidecar.name
validator = HedValidator(hed_schema)
issues = json_sidecar.validate_entries(
validator, check_for_warnings=check_for_warnings)
if issues:
issue_str = get_printable_issue_string(
issues, f"JSON dictionary {display_name } validation errors")
file_name = generate_filename(display_name,
name_suffix='validation_errors',
extension='.txt')
return {
base_constants.COMMAND: base_constants.COMMAND_VALIDATE,
'data': issue_str,
'output_display_name': file_name,
base_constants.SCHEMA_VERSION: schema_version,
'msg_category': 'warning',
'msg': f'JSON sidecar {display_name} had validation errors'
}
else:
return {
base_constants.COMMAND: base_constants.COMMAND_VALIDATE,
'data': '',
base_constants.SCHEMA_VERSION: schema_version,
'msg_category': 'success',
'msg': f'JSON file {display_name} had no validation errors'
}
def spreadsheet_validate(hed_schema, spreadsheet, check_for_warnings=False):
""" Validates the spreadsheet.
Parameters
----------
hed_schema: str or HedSchema
Version number or path or HedSchema object to be used
spreadsheet: HedFileInput
Spreadsheet input object to be validated
check_for_warnings: bool
Indicates whether validation should check for warnings as well as errors
Returns
-------
dict
A dictionary containing results of validation in standard format
"""
schema_version = hed_schema.header_attributes.get('version',
'Unknown version')
validator = HedValidator(hed_schema=hed_schema)
issues = spreadsheet.validate_file(validator,
check_for_warnings=check_for_warnings)
display_name = spreadsheet.name
if issues:
issue_str = get_printable_issue_string(
issues, f"Spreadsheet {display_name} validation errors")
file_name = generate_filename(display_name,
name_suffix='_validation_errors',
extension='.txt')
return {
base_constants.COMMAND: base_constants.COMMAND_VALIDATE,
'data': issue_str,
"output_display_name": file_name,
base_constants.SCHEMA_VERSION: schema_version,
"msg_category": "warning",
'msg': f"Spreadsheet {display_name} had validation errors"
}
else:
return {
base_constants.COMMAND: base_constants.COMMAND_VALIDATE,
'data': '',
base_constants.SCHEMA_VERSION: schema_version,
'msg_category': 'success',
'msg': f'Spreadsheet {display_name} had no validation errors'
}
def extract(events, columns_selected):
"""Extracts a JSON sidecar template from a BIDS-style events file.
Parameters
----------
events: EventInput
An events input object
columns_selected: dict
dictionary of columns selected
Returns
-------
dict
A dictionary pointing to extracted JSON file.
"""
columns_info = get_columns_info(events.dataframe)
sr = SidecarMap()
hed_dict, issues = sr.get_sidecar_dict(columns_info, columns_selected)
display_name = events.name
if issues:
issue_str = get_printable_issue_string(
issues, f"{display_name} HED validation errors")
file_name = generate_filename(display_name,
name_suffix='_errors',
extension='.txt')
return {
base_constants.COMMAND: base_constants.COMMAND_VALIDATE,
'data': issue_str,
"output_display_name": file_name,
"msg_category": "warning",
'msg': f"Events file {display_name} had extraction errors"
}
else:
file_name = generate_filename(display_name,
name_suffix='_extracted',
extension='.json')
return {
base_constants.COMMAND: base_constants.COMMAND_EXTRACT,
'data': json.dumps(hed_dict, indent=4),
'output_display_name': file_name,
'msg_category': 'success',
'msg': 'Events extraction to JSON complete'
}
def schema_validate(hed_schema, display_name):
"""Run schema compliance for HED-3G
Parameters
----------
hed_schema: HedSchema object
A HedSchema object containing the schema to be processed.
display_name: str
The display name associated with this schema object.
Returns
-------
result: dict
A dictionary in the standard results format containing the results of the operation
"""
schema_version = hed_schema.header_attributes.get('version', 'Unknown')
issues = hed_schema.check_compliance()
if issues:
issue_str = get_printable_issue_string(
issues, f"Schema HED 3G compliance errors for {display_name}")
file_name = generate_filename(
display_name,
name_suffix='schema_3G_compliance_errors',
extension='.txt')
return {
'command': base_constants.COMMAND_VALIDATE,
'data': issue_str,
'output_display_name': file_name,
'schema_version': schema_version,
'msg_category': 'warning',
'msg': 'Schema is not HED 3G compliant'
}
else:
return {
'command': base_constants.COMMAND_VALIDATE,
'data': '',
'output_display_name': display_name,
'schema_version': schema_version,
'msg_category': 'success',
'msg': 'Schema had no HED-3G validation errors'
}
if __name__ == '__main__':
hed_xml_url = 'https://raw.githubusercontent.com/hed-standard/hed-specification/master/hedxml/HED8.0.0.xml'
hed_schema = load_schema(hed_xml_url)
hed_validator = HedValidator(hed_schema)
hed_validator_no_semantic = HedValidator(run_semantic_validation=False)
hed_string_1 = \
"Sensory-event,Visual-presentation,Experimental-stimulus,Green,Non-target," + \
"(Letter/D, (Center-of, Computer-screen))"
string_obj_1 = HedString(hed_string_1)
validation_issues = string_obj_1.validate(hed_validator)
print(
get_printable_issue_string(
validation_issues,
title=
'[Example 1] hed_string_1 should have no issues with HEDv8.0.0'))
string_1_long, issues = string_obj_1.convert_to_short(hed_schema)
print(
get_printable_issue_string(
issues,
title=
'[Example 2] hed_string_1 should convert to long without errors'))
string_1_long_obj = HedString(string_1_long)
validation_issues = string_1_long_obj.validate(hed_validator)
print(
get_printable_issue_string(
validation_issues,
title=
'[Example 3] hed_string_1 should validate after conversion to long'
multiple_sheet_xlsx_file = os.path.join(example_data_path,
'ExcelMultipleSheets.xlsx')
hed_schema_cached = load_schema_version(xml_version_number='7.1.1')
hed_validator_old = HedValidator(hed_schema=hed_schema_cached)
hed_schema_local = load_schema(local_hed_file)
hed_validator_local = HedValidator(hed_schema=hed_schema_local)
hed_validator_local_warnings = HedValidator(hed_schema=hed_schema_local)
# Example 1a: Valid TSV file with default version of HED
print(valid_tsv_file)
input_file = HedInput(valid_tsv_file, tag_columns=[2])
validation_issues = input_file.validate_file(hed_validator_old,
check_for_warnings=True)
the_title = '[Example 1a] ValidTwoColumnHED7_1_1 is probably okay with default version of HED'
print(get_printable_issue_string(validation_issues, title=the_title))
# Example 1b: Valid TSV file with specified local version of HED
print(valid_tsv_file)
input_file = HedInput(valid_tsv_file, tag_columns=[2])
validation_issues = input_file.validate_file(hed_validator_local)
print(
get_printable_issue_string(
validation_issues,
title=
'[Example 1b] ValidTwoColumnHED7_1_1 should have no issues with local '
+ 'version 7.1.1'))
# Example 1c: Valid TSV file with specified local version of HED and no column headers
print(valid_tsv_file_no_header)
input_file = HedInput(valid_tsv_file_no_header,
if issues:
return issues
for event_obj in self.events_dict.values():
contents = event_obj.contents
if not contents:
contents = EventsInput(file=event_obj.file_path, sidecars=event_obj.sidecars)
if keep_events:
event_obj.my_contents = contents
issues += contents.validate_file(validators=validators, check_for_warnings=check_for_warnings)
return issues
if __name__ == '__main__':
path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'../../../tests/data/bids/eeg_ds003654s_hed')
bids = BidsEventFiles(path)
for file_obj in bids.sidecar_dict.values():
print(file_obj)
for file_obj in bids.events_dict.values():
print(file_obj)
print("Now validating.....")
hed_schema = \
load_schema('https://raw.githubusercontent.com/hed-standard/hed-specification/master/hedxml/HED8.0.0.xml')
validator = HedValidator(hed_schema=hed_schema)
validation_issues = bids.validate(validators=[validator], check_for_warnings=False)
issue_str = get_printable_issue_string(validation_issues, skip_filename=False)
print(f"Issues: {issue_str}")
import os
from hed.errors.error_reporter import get_printable_issue_string
from hed.tools.bids.bids_dataset import BidsDataset
if __name__ == '__main__':
path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'../../../datasets/eeg_ds003654s_hed_library')
bids = BidsDataset(path)
# Validate and include warnings
print("Validating and include warnings...")
issue_list1 = bids.validate(check_for_warnings=True)
if issue_list1:
issue_str = get_printable_issue_string(issue_list1, "HED_library issues with warnings")
print(f"Issues (including warnings):\n{issue_str}")
else:
print("...No issues even when warnings included")
# Validate and don't include warnings
print("Validating and don't include warnings...")
issue_list2 = bids.validate(check_for_warnings=False)
if issue_list2:
issue_str = get_printable_issue_string(issue_list2, "HED_library issues with no warnings")
print(f"Issues (including warnings):\n{issue_str}")
else:
print("...No issues when warnings not included")
def convert(hed_schema,
string_list,
command=base_constants.COMMAND_TO_SHORT,
check_for_warnings=False):
"""Converts a list of strings from long to short or long to short then converts to short
Parameters
----------
hed_schema: HedSchema
The HED schema to be used in processing
string_list: list of HedString
A list of HedString to be processed
command: str
Name of the command to execute (default to short if unrecognized)
check_for_warnings: bool
Indicates whether validation should check for warnings as well as errors
Returns
-------
dict
A dictionary with the results of string processing in standard format.
"""
schema_version = hed_schema.header_attributes.get('version',
'Unknown version')
results = validate(hed_schema,
string_list,
check_for_warnings=check_for_warnings)
if results['data']:
return results
strings = []
conversion_errors = []
for pos, hed_string_obj in enumerate(string_list, start=1):
if command == base_constants.COMMAND_TO_LONG:
converted_string, issues = hed_string_obj.convert_to_long(
hed_schema)
else:
converted_string, issues = hed_string_obj.convert_to_short(
hed_schema)
if issues:
conversion_errors.append(
get_printable_issue_string(issues,
f"Errors for HED string {pos}:"))
strings.append(converted_string)
if conversion_errors:
return {
base_constants.COMMAND:
command,
'data':
conversion_errors,
'additional_info':
string_list,
base_constants.SCHEMA_VERSION:
schema_version,
'msg_category':
'warning',
'msg':
'Some strings had conversion errors, results of conversion in additional_info'
}
else:
return {
base_constants.COMMAND: command,
'data': strings,
base_constants.SCHEMA_VERSION: schema_version,
'msg_category': 'success',
'msg': 'Strings converted successfully'
}
from hed.models.sidecar import Sidecar
from hed.schema.hed_schema_io import load_schema
from hed.validator.hed_validator import HedValidator
if __name__ == '__main__':
hed_xml_url = 'https://raw.githubusercontent.com/hed-standard/hed-specification/master/hedxml/HED8.0.0.xml'
hed_schema = load_schema(hed_xml_url)
json_filename = "../../../datasets/eeg_ds003654s_hed/task-FacePerception_events.json"
# Example 1
print("\nExample 1 demonstrating Sidecar validation....")
sidecar = Sidecar(json_filename)
validator = HedValidator(hed_schema)
issues = sidecar.validate_entries(validator, check_for_warnings=True)
if issues:
print(get_printable_issue_string(issues),
"JSON dictionary from eeg_ds003654s_hed has validation errors")
else:
print(
"JSON dictionary from eeg_ds003654s_hed has no validation errors")
# Example 2: Convert JSON to long and output it.
print("\n\nExample 2 converting a Sidecar to long in place ...")
tag_form = 'long_tag'
validator = HedValidator(hed_schema)
for hed_string_obj, position_info, issue_items in sidecar.hed_string_iter(
validators=validator, expand_defs=False, allow_placeholder=True):
converted_string = hed_string_obj.get_as_form(tag_form)
issues = issues + issue_items
sidecar.set_hed_string(converted_string, position_info)
print(f"Outputting the converted results:\n{sidecar.get_as_json_string()}")
def sidecar_convert(hed_schema,
json_sidecar,
command=base_constants.COMMAND_TO_SHORT,
expand_defs=False):
"""Converts a sidecar from long to short or short to long
Parameters
----------
hed_schema:HedSchema
HedSchema object to be used
json_sidecar: Sidecar
Previously created Sidecar
command: str
Name of the command to execute (default to short if unrecognized)
expand_defs: bool
Indicates whether to expand definitions when converting
Returns
-------
dict
A downloadable dictionary file or a file containing warnings
"""
schema_version = hed_schema.header_attributes.get('version',
'Unknown version')
results = sidecar_validate(hed_schema,
json_sidecar,
check_for_warnings=False)
if results['data']:
return results
if command == base_constants.COMMAND_TO_LONG:
tag_form = 'long_tag'
else:
tag_form = 'short_tag'
issues = []
for hed_string_obj, position_info, issue_items in json_sidecar.hed_string_iter(
validators=hed_schema,
expand_defs=expand_defs,
remove_definitions=False):
converted_string = hed_string_obj.get_as_form(tag_form)
issues = issues + issue_items
json_sidecar.set_hed_string(converted_string, position_info)
# issues = ErrorHandler.filter_issues_by_severity(issues, ErrorSeverity.ERROR)
display_name = json_sidecar.name
if issues:
issue_str = get_printable_issue_string(
issues, f"JSON conversion for {display_name} was unsuccessful")
file_name = generate_filename(
display_name,
name_suffix=f"_{tag_form}_conversion_errors",
extension='.txt')
return {
base_constants.COMMAND: command,
'data': issue_str,
'output_display_name': file_name,
base_constants.SCHEMA_VERSION: schema_version,
'msg_category': 'warning',
'msg': f'JSON file {display_name} had validation errors'
}
else:
file_name = generate_filename(display_name,
name_suffix=f"_{tag_form}",
extension='.json')
data = json_sidecar.get_as_json_string()
return {
base_constants.COMMAND: command,
'data': data,
'output_display_name': file_name,
base_constants.SCHEMA_VERSION: schema_version,
'msg_category': 'success',
'msg': f'JSON sidecar {display_name} was successfully converted'
}
hed_xml_url = 'https://raw.githubusercontent.com/hed-standard/hed-specification/master/hedxml/HED8.0.0.xml'
hed_library_url1 = \
'https://raw.githubusercontent.com/hed-standard/hed-schema-library/main/hedxml/HED_score_0.0.1.xml'
hed_library_url2 = \
'https://raw.githubusercontent.com/hed-standard/hed-schema-library/main/hedxml/HED_test_1.0.2.xml'
hed_schema = load_schema(hed_xml_url)
hed_schema_lib1 = load_schema(hed_library_url1)
hed_schema_lib1.set_library_prefix("sc")
hed_schema_lib2 = load_schema(hed_library_url2)
hed_schema_lib2.set_library_prefix("test")
events_file = os.path.join(
'../../../datasets/eeg_ds003654s_hed_library/sub-003/eeg/sub-003_task-FacePerception_run-2_events.tsv'
)
json_file = os.path.join(
'../../../datasets/eeg_ds003654s_hed_library/task-FacePerception_events.json'
)
schema_group = HedSchemaGroup(
[hed_schema, hed_schema_lib1, hed_schema_lib2])
validator = HedValidator(hed_schema=schema_group)
sidecar = Sidecar(json_file)
input_file = EventsInput(events_file, sidecars=sidecar)
issues = input_file.validate_file_sidecars(validator,
check_for_warnings=False)
issues += input_file.validate_file(validator, check_for_warnings=False)
print(
get_printable_issue_string(
issues, "Validating a Bids event file with its JSON sidecar"))