def validate(hed_schema, events, sidecar=None, check_for_warnings=False): """Validates and events input object and returns the results. Parameters ---------- hed_schema: str or HedSchema Version number or path or HedSchema object to be used events: EventsInput Events input object to be validated sidecar: Sidecar Representation of a BIDS JSON sidecar object check_for_warnings: bool If true, validation should include warnings Returns ------- dict A dictionary containing results of validation in standard format """ schema_version = hed_schema.header_attributes.get('version', 'Unknown version') display_name = events.name validator = HedValidator(hed_schema=hed_schema) issue_str = '' if sidecar: issues = sidecar.validate_entries( validator, check_for_warnings=check_for_warnings) if issues: issue_str = issue_str + get_printable_issue_string( issues, title="Sidecar definition errors:") issues = events.validate_file(validator, check_for_warnings=check_for_warnings) if issues: issue_str = issue_str + get_printable_issue_string( issues, title="Event file errors:") if issue_str: file_name = generate_filename(display_name, name_suffix='_validation_errors', extension='.txt') return { base_constants.COMMAND: base_constants.COMMAND_VALIDATE, 'data': issue_str, "output_display_name": file_name, base_constants.SCHEMA_VERSION: schema_version, "msg_category": "warning", 'msg': f"Events file {display_name} had validation errors" } else: return { base_constants.COMMAND: base_constants.COMMAND_VALIDATE, 'data': '', base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'success', 'msg': f"Events file {display_name} had no validation errors" }
def validate(hed_schema, string_list, check_for_warnings=False): """Validates a list of strings and returns a dictionary containing the issues or a no errors message Parameters ---------- hed_schema: HedSchema The HED schema to be used in processing string_list: list A list of string to be processed check_for_warnings: bool Indicates whether validation should check for warnings as well as errors Returns ------- dict A dictionary with results """ schema_version = hed_schema.header_attributes.get('version', 'Unknown version') hed_validator = HedValidator(hed_schema=hed_schema) validation_errors = [] for pos, h_string in enumerate(string_list, start=1): issues = h_string.validate(hed_validator, check_for_warnings=check_for_warnings) if issues: validation_errors.append( get_printable_issue_string(issues, f"Errors for HED string {pos}:")) if validation_errors: return { base_constants.COMMAND: base_constants.COMMAND_VALIDATE, 'data': validation_errors, base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'warning', 'msg': 'Strings had validation errors' } else: return { base_constants.COMMAND: base_constants.COMMAND_VALIDATE, 'data': '', base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'success', 'msg': 'Strings validated successfully...' }
def sidecar_validate(hed_schema, json_sidecar, check_for_warnings=False): """ Validates the sidecar and returns the errors and/or a message in a dictionary Parameters ---------- hed_schema: str or HedSchema Version number or path or HedSchema object to be used json_sidecar: Sidecar Dictionary object check_for_warnings: bool Indicates whether validation should check for warnings as well as errors Returns ------- dict dictionary of response values. """ schema_version = hed_schema.header_attributes.get('version', 'Unknown version') display_name = json_sidecar.name validator = HedValidator(hed_schema) issues = json_sidecar.validate_entries( validator, check_for_warnings=check_for_warnings) if issues: issue_str = get_printable_issue_string( issues, f"JSON dictionary {display_name } validation errors") file_name = generate_filename(display_name, name_suffix='validation_errors', extension='.txt') return { base_constants.COMMAND: base_constants.COMMAND_VALIDATE, 'data': issue_str, 'output_display_name': file_name, base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'warning', 'msg': f'JSON sidecar {display_name} had validation errors' } else: return { base_constants.COMMAND: base_constants.COMMAND_VALIDATE, 'data': '', base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'success', 'msg': f'JSON file {display_name} had no validation errors' }
def spreadsheet_validate(hed_schema, spreadsheet, check_for_warnings=False): """ Validates the spreadsheet. Parameters ---------- hed_schema: str or HedSchema Version number or path or HedSchema object to be used spreadsheet: HedFileInput Spreadsheet input object to be validated check_for_warnings: bool Indicates whether validation should check for warnings as well as errors Returns ------- dict A dictionary containing results of validation in standard format """ schema_version = hed_schema.header_attributes.get('version', 'Unknown version') validator = HedValidator(hed_schema=hed_schema) issues = spreadsheet.validate_file(validator, check_for_warnings=check_for_warnings) display_name = spreadsheet.name if issues: issue_str = get_printable_issue_string( issues, f"Spreadsheet {display_name} validation errors") file_name = generate_filename(display_name, name_suffix='_validation_errors', extension='.txt') return { base_constants.COMMAND: base_constants.COMMAND_VALIDATE, 'data': issue_str, "output_display_name": file_name, base_constants.SCHEMA_VERSION: schema_version, "msg_category": "warning", 'msg': f"Spreadsheet {display_name} had validation errors" } else: return { base_constants.COMMAND: base_constants.COMMAND_VALIDATE, 'data': '', base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'success', 'msg': f'Spreadsheet {display_name} had no validation errors' }
def extract(events, columns_selected): """Extracts a JSON sidecar template from a BIDS-style events file. Parameters ---------- events: EventInput An events input object columns_selected: dict dictionary of columns selected Returns ------- dict A dictionary pointing to extracted JSON file. """ columns_info = get_columns_info(events.dataframe) sr = SidecarMap() hed_dict, issues = sr.get_sidecar_dict(columns_info, columns_selected) display_name = events.name if issues: issue_str = get_printable_issue_string( issues, f"{display_name} HED validation errors") file_name = generate_filename(display_name, name_suffix='_errors', extension='.txt') return { base_constants.COMMAND: base_constants.COMMAND_VALIDATE, 'data': issue_str, "output_display_name": file_name, "msg_category": "warning", 'msg': f"Events file {display_name} had extraction errors" } else: file_name = generate_filename(display_name, name_suffix='_extracted', extension='.json') return { base_constants.COMMAND: base_constants.COMMAND_EXTRACT, 'data': json.dumps(hed_dict, indent=4), 'output_display_name': file_name, 'msg_category': 'success', 'msg': 'Events extraction to JSON complete' }
def schema_validate(hed_schema, display_name): """Run schema compliance for HED-3G Parameters ---------- hed_schema: HedSchema object A HedSchema object containing the schema to be processed. display_name: str The display name associated with this schema object. Returns ------- result: dict A dictionary in the standard results format containing the results of the operation """ schema_version = hed_schema.header_attributes.get('version', 'Unknown') issues = hed_schema.check_compliance() if issues: issue_str = get_printable_issue_string( issues, f"Schema HED 3G compliance errors for {display_name}") file_name = generate_filename( display_name, name_suffix='schema_3G_compliance_errors', extension='.txt') return { 'command': base_constants.COMMAND_VALIDATE, 'data': issue_str, 'output_display_name': file_name, 'schema_version': schema_version, 'msg_category': 'warning', 'msg': 'Schema is not HED 3G compliant' } else: return { 'command': base_constants.COMMAND_VALIDATE, 'data': '', 'output_display_name': display_name, 'schema_version': schema_version, 'msg_category': 'success', 'msg': 'Schema had no HED-3G validation errors' }
if __name__ == '__main__': hed_xml_url = 'https://raw.githubusercontent.com/hed-standard/hed-specification/master/hedxml/HED8.0.0.xml' hed_schema = load_schema(hed_xml_url) hed_validator = HedValidator(hed_schema) hed_validator_no_semantic = HedValidator(run_semantic_validation=False) hed_string_1 = \ "Sensory-event,Visual-presentation,Experimental-stimulus,Green,Non-target," + \ "(Letter/D, (Center-of, Computer-screen))" string_obj_1 = HedString(hed_string_1) validation_issues = string_obj_1.validate(hed_validator) print( get_printable_issue_string( validation_issues, title= '[Example 1] hed_string_1 should have no issues with HEDv8.0.0')) string_1_long, issues = string_obj_1.convert_to_short(hed_schema) print( get_printable_issue_string( issues, title= '[Example 2] hed_string_1 should convert to long without errors')) string_1_long_obj = HedString(string_1_long) validation_issues = string_1_long_obj.validate(hed_validator) print( get_printable_issue_string( validation_issues, title= '[Example 3] hed_string_1 should validate after conversion to long'
multiple_sheet_xlsx_file = os.path.join(example_data_path, 'ExcelMultipleSheets.xlsx') hed_schema_cached = load_schema_version(xml_version_number='7.1.1') hed_validator_old = HedValidator(hed_schema=hed_schema_cached) hed_schema_local = load_schema(local_hed_file) hed_validator_local = HedValidator(hed_schema=hed_schema_local) hed_validator_local_warnings = HedValidator(hed_schema=hed_schema_local) # Example 1a: Valid TSV file with default version of HED print(valid_tsv_file) input_file = HedInput(valid_tsv_file, tag_columns=[2]) validation_issues = input_file.validate_file(hed_validator_old, check_for_warnings=True) the_title = '[Example 1a] ValidTwoColumnHED7_1_1 is probably okay with default version of HED' print(get_printable_issue_string(validation_issues, title=the_title)) # Example 1b: Valid TSV file with specified local version of HED print(valid_tsv_file) input_file = HedInput(valid_tsv_file, tag_columns=[2]) validation_issues = input_file.validate_file(hed_validator_local) print( get_printable_issue_string( validation_issues, title= '[Example 1b] ValidTwoColumnHED7_1_1 should have no issues with local ' + 'version 7.1.1')) # Example 1c: Valid TSV file with specified local version of HED and no column headers print(valid_tsv_file_no_header) input_file = HedInput(valid_tsv_file_no_header,
if issues: return issues for event_obj in self.events_dict.values(): contents = event_obj.contents if not contents: contents = EventsInput(file=event_obj.file_path, sidecars=event_obj.sidecars) if keep_events: event_obj.my_contents = contents issues += contents.validate_file(validators=validators, check_for_warnings=check_for_warnings) return issues if __name__ == '__main__': path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../tests/data/bids/eeg_ds003654s_hed') bids = BidsEventFiles(path) for file_obj in bids.sidecar_dict.values(): print(file_obj) for file_obj in bids.events_dict.values(): print(file_obj) print("Now validating.....") hed_schema = \ load_schema('https://raw.githubusercontent.com/hed-standard/hed-specification/master/hedxml/HED8.0.0.xml') validator = HedValidator(hed_schema=hed_schema) validation_issues = bids.validate(validators=[validator], check_for_warnings=False) issue_str = get_printable_issue_string(validation_issues, skip_filename=False) print(f"Issues: {issue_str}")
import os from hed.errors.error_reporter import get_printable_issue_string from hed.tools.bids.bids_dataset import BidsDataset if __name__ == '__main__': path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../datasets/eeg_ds003654s_hed_library') bids = BidsDataset(path) # Validate and include warnings print("Validating and include warnings...") issue_list1 = bids.validate(check_for_warnings=True) if issue_list1: issue_str = get_printable_issue_string(issue_list1, "HED_library issues with warnings") print(f"Issues (including warnings):\n{issue_str}") else: print("...No issues even when warnings included") # Validate and don't include warnings print("Validating and don't include warnings...") issue_list2 = bids.validate(check_for_warnings=False) if issue_list2: issue_str = get_printable_issue_string(issue_list2, "HED_library issues with no warnings") print(f"Issues (including warnings):\n{issue_str}") else: print("...No issues when warnings not included")
def convert(hed_schema, string_list, command=base_constants.COMMAND_TO_SHORT, check_for_warnings=False): """Converts a list of strings from long to short or long to short then converts to short Parameters ---------- hed_schema: HedSchema The HED schema to be used in processing string_list: list of HedString A list of HedString to be processed command: str Name of the command to execute (default to short if unrecognized) check_for_warnings: bool Indicates whether validation should check for warnings as well as errors Returns ------- dict A dictionary with the results of string processing in standard format. """ schema_version = hed_schema.header_attributes.get('version', 'Unknown version') results = validate(hed_schema, string_list, check_for_warnings=check_for_warnings) if results['data']: return results strings = [] conversion_errors = [] for pos, hed_string_obj in enumerate(string_list, start=1): if command == base_constants.COMMAND_TO_LONG: converted_string, issues = hed_string_obj.convert_to_long( hed_schema) else: converted_string, issues = hed_string_obj.convert_to_short( hed_schema) if issues: conversion_errors.append( get_printable_issue_string(issues, f"Errors for HED string {pos}:")) strings.append(converted_string) if conversion_errors: return { base_constants.COMMAND: command, 'data': conversion_errors, 'additional_info': string_list, base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'warning', 'msg': 'Some strings had conversion errors, results of conversion in additional_info' } else: return { base_constants.COMMAND: command, 'data': strings, base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'success', 'msg': 'Strings converted successfully' }
from hed.models.sidecar import Sidecar from hed.schema.hed_schema_io import load_schema from hed.validator.hed_validator import HedValidator if __name__ == '__main__': hed_xml_url = 'https://raw.githubusercontent.com/hed-standard/hed-specification/master/hedxml/HED8.0.0.xml' hed_schema = load_schema(hed_xml_url) json_filename = "../../../datasets/eeg_ds003654s_hed/task-FacePerception_events.json" # Example 1 print("\nExample 1 demonstrating Sidecar validation....") sidecar = Sidecar(json_filename) validator = HedValidator(hed_schema) issues = sidecar.validate_entries(validator, check_for_warnings=True) if issues: print(get_printable_issue_string(issues), "JSON dictionary from eeg_ds003654s_hed has validation errors") else: print( "JSON dictionary from eeg_ds003654s_hed has no validation errors") # Example 2: Convert JSON to long and output it. print("\n\nExample 2 converting a Sidecar to long in place ...") tag_form = 'long_tag' validator = HedValidator(hed_schema) for hed_string_obj, position_info, issue_items in sidecar.hed_string_iter( validators=validator, expand_defs=False, allow_placeholder=True): converted_string = hed_string_obj.get_as_form(tag_form) issues = issues + issue_items sidecar.set_hed_string(converted_string, position_info) print(f"Outputting the converted results:\n{sidecar.get_as_json_string()}")
def sidecar_convert(hed_schema, json_sidecar, command=base_constants.COMMAND_TO_SHORT, expand_defs=False): """Converts a sidecar from long to short or short to long Parameters ---------- hed_schema:HedSchema HedSchema object to be used json_sidecar: Sidecar Previously created Sidecar command: str Name of the command to execute (default to short if unrecognized) expand_defs: bool Indicates whether to expand definitions when converting Returns ------- dict A downloadable dictionary file or a file containing warnings """ schema_version = hed_schema.header_attributes.get('version', 'Unknown version') results = sidecar_validate(hed_schema, json_sidecar, check_for_warnings=False) if results['data']: return results if command == base_constants.COMMAND_TO_LONG: tag_form = 'long_tag' else: tag_form = 'short_tag' issues = [] for hed_string_obj, position_info, issue_items in json_sidecar.hed_string_iter( validators=hed_schema, expand_defs=expand_defs, remove_definitions=False): converted_string = hed_string_obj.get_as_form(tag_form) issues = issues + issue_items json_sidecar.set_hed_string(converted_string, position_info) # issues = ErrorHandler.filter_issues_by_severity(issues, ErrorSeverity.ERROR) display_name = json_sidecar.name if issues: issue_str = get_printable_issue_string( issues, f"JSON conversion for {display_name} was unsuccessful") file_name = generate_filename( display_name, name_suffix=f"_{tag_form}_conversion_errors", extension='.txt') return { base_constants.COMMAND: command, 'data': issue_str, 'output_display_name': file_name, base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'warning', 'msg': f'JSON file {display_name} had validation errors' } else: file_name = generate_filename(display_name, name_suffix=f"_{tag_form}", extension='.json') data = json_sidecar.get_as_json_string() return { base_constants.COMMAND: command, 'data': data, 'output_display_name': file_name, base_constants.SCHEMA_VERSION: schema_version, 'msg_category': 'success', 'msg': f'JSON sidecar {display_name} was successfully converted' }
hed_xml_url = 'https://raw.githubusercontent.com/hed-standard/hed-specification/master/hedxml/HED8.0.0.xml' hed_library_url1 = \ 'https://raw.githubusercontent.com/hed-standard/hed-schema-library/main/hedxml/HED_score_0.0.1.xml' hed_library_url2 = \ 'https://raw.githubusercontent.com/hed-standard/hed-schema-library/main/hedxml/HED_test_1.0.2.xml' hed_schema = load_schema(hed_xml_url) hed_schema_lib1 = load_schema(hed_library_url1) hed_schema_lib1.set_library_prefix("sc") hed_schema_lib2 = load_schema(hed_library_url2) hed_schema_lib2.set_library_prefix("test") events_file = os.path.join( '../../../datasets/eeg_ds003654s_hed_library/sub-003/eeg/sub-003_task-FacePerception_run-2_events.tsv' ) json_file = os.path.join( '../../../datasets/eeg_ds003654s_hed_library/task-FacePerception_events.json' ) schema_group = HedSchemaGroup( [hed_schema, hed_schema_lib1, hed_schema_lib2]) validator = HedValidator(hed_schema=schema_group) sidecar = Sidecar(json_file) input_file = EventsInput(events_file, sidecars=sidecar) issues = input_file.validate_file_sidecars(validator, check_for_warnings=False) issues += input_file.validate_file(validator, check_for_warnings=False) print( get_printable_issue_string( issues, "Validating a Bids event file with its JSON sidecar"))