def schema_convert(hed_schema, display_name):
"""Return a string representation of hed_schema in the desired format
Parameters
----------
hed_schema: HedSchema object
A HedSchema object containing the schema to be processed.
display_name: str
The display name associated with this schema object.
Returns
-------
result: dict
A dictionary in the standard results format containing the results of the operation
"""
schema_version = hed_schema.header_attributes.get('version', 'Unknown')
schema_format = get_file_extension(display_name)
if schema_format == file_constants.SCHEMA_XML_EXTENSION:
data = hed_schema.get_as_mediawiki_string()
extension = '.mediawiki'
else:
data = hed_schema.get_as_xml_string()
extension = '.xml'
file_name = generate_filename(display_name, extension=extension)
return {
'command': base_constants.COMMAND_CONVERT,
'data': data,
'output_display_name': file_name,
'schema_version': schema_version,
'msg_category': 'success',
'msg': 'Schema was successfully converted'
}
def sidecar_flatten(json_sidecar):
"""Converts a sidecar from long to short unless unless the command is not COMMAND_TO_LONG then converts to short
Parameters
----------
json_sidecar: Sidecar
Previously created Sidecar
Returns
-------
dict
A downloadable dictionary file or a file containing warnings
"""
json_string = json_sidecar.get_as_json_string()
sidecar = json.loads(json_string)
sr = SidecarMap()
df = sr.flatten(sidecar)
data = df.to_csv(None, sep='\t', index=False, header=True)
display_name = json_sidecar.name
file_name = generate_filename(display_name,
name_suffix='flattened',
extension='.tsv')
return {
base_constants.COMMAND: base_constants.COMMAND_FLATTEN,
'data': data,
'output_display_name': file_name,
'msg_category': 'success',
'msg': f'JSON sidecar {display_name} was successfully flattened'
}
def validate(hed_schema, events, sidecar=None, check_for_warnings=False):
"""Validates and events input object and returns the results.
Parameters
----------
hed_schema: str or HedSchema
Version number or path or HedSchema object to be used
events: EventsInput
Events input object to be validated
sidecar: Sidecar
Representation of a BIDS JSON sidecar object
check_for_warnings: bool
If true, validation should include warnings
Returns
-------
dict
A dictionary containing results of validation in standard format
"""
schema_version = hed_schema.header_attributes.get('version',
'Unknown version')
display_name = events.name
validator = HedValidator(hed_schema=hed_schema)
issue_str = ''
if sidecar:
issues = sidecar.validate_entries(
validator, check_for_warnings=check_for_warnings)
if issues:
issue_str = issue_str + get_printable_issue_string(
issues, title="Sidecar definition errors:")
issues = events.validate_file(validator,
check_for_warnings=check_for_warnings)
if issues:
issue_str = issue_str + get_printable_issue_string(
issues, title="Event file errors:")
if issue_str:
file_name = generate_filename(display_name,
name_suffix='_validation_errors',
extension='.txt')
return {
base_constants.COMMAND: base_constants.COMMAND_VALIDATE,
'data': issue_str,
"output_display_name": file_name,
base_constants.SCHEMA_VERSION: schema_version,
"msg_category": "warning",
'msg': f"Events file {display_name} had validation errors"
}
else:
return {
base_constants.COMMAND: base_constants.COMMAND_VALIDATE,
'data': '',
base_constants.SCHEMA_VERSION: schema_version,
'msg_category': 'success',
'msg': f"Events file {display_name} had no validation errors"
}
def extract(events, columns_selected):
"""Extracts a JSON sidecar template from a BIDS-style events file.
Parameters
----------
events: EventInput
An events input object
columns_selected: dict
dictionary of columns selected
Returns
-------
dict
A dictionary pointing to extracted JSON file.
"""
columns_info = get_columns_info(events.dataframe)
sr = SidecarMap()
hed_dict, issues = sr.get_sidecar_dict(columns_info, columns_selected)
display_name = events.name
if issues:
issue_str = get_printable_issue_string(
issues, f"{display_name} HED validation errors")
file_name = generate_filename(display_name,
name_suffix='_errors',
extension='.txt')
return {
base_constants.COMMAND: base_constants.COMMAND_VALIDATE,
'data': issue_str,
"output_display_name": file_name,
"msg_category": "warning",
'msg': f"Events file {display_name} had extraction errors"
}
else:
file_name = generate_filename(display_name,
name_suffix='_extracted',
extension='.json')
return {
base_constants.COMMAND: base_constants.COMMAND_EXTRACT,
'data': json.dumps(hed_dict, indent=4),
'output_display_name': file_name,
'msg_category': 'success',
'msg': 'Events extraction to JSON complete'
}
def spreadsheet_convert(hed_schema,
spreadsheet,
command=base_constants.COMMAND_TO_LONG,
check_for_warnings=False):
"""Converts a spreadsheet long to short unless unless the command is not COMMAND_TO_LONG then converts to short
Parameters
----------
hed_schema:HedSchema
HedSchema object to be used
spreadsheet: HedInput
Previously created HedInput object
command: str
Name of the command to execute if not COMMAND_TO_LONG
check_for_warnings: bool
If True, check for war
Returns
-------
dict
A downloadable dictionary file or a file containing warnings
"""
schema_version = hed_schema.header_attributes.get('version',
'Unknown version')
results = spreadsheet_validate(hed_schema,
spreadsheet,
check_for_warnings=check_for_warnings)
if results['data']:
return results
display_name = spreadsheet.name
display_ext = os.path.splitext(secure_filename(display_name))[1]
if command == base_constants.COMMAND_TO_LONG:
suffix = '_to_long'
spreadsheet.convert_to_long(hed_schema)
else:
suffix = '_to_short'
spreadsheet.convert_to_short(hed_schema)
file_name = generate_filename(display_name,
name_suffix=suffix,
extension=display_ext)
return {
base_constants.COMMAND: command,
'data': '',
base_constants.SPREADSHEET: spreadsheet,
'output_display_name': file_name,
base_constants.SCHEMA_VERSION: schema_version,
'msg_category': 'success',
'msg': f'Spreadsheet {display_name} converted_successfully'
}
def sidecar_validate(hed_schema, json_sidecar, check_for_warnings=False):
""" Validates the sidecar and returns the errors and/or a message in a dictionary
Parameters
----------
hed_schema: str or HedSchema
Version number or path or HedSchema object to be used
json_sidecar: Sidecar
Dictionary object
check_for_warnings: bool
Indicates whether validation should check for warnings as well as errors
Returns
-------
dict
dictionary of response values.
"""
schema_version = hed_schema.header_attributes.get('version',
'Unknown version')
display_name = json_sidecar.name
validator = HedValidator(hed_schema)
issues = json_sidecar.validate_entries(
validator, check_for_warnings=check_for_warnings)
if issues:
issue_str = get_printable_issue_string(
issues, f"JSON dictionary {display_name } validation errors")
file_name = generate_filename(display_name,
name_suffix='validation_errors',
extension='.txt')
return {
base_constants.COMMAND: base_constants.COMMAND_VALIDATE,
'data': issue_str,
'output_display_name': file_name,
base_constants.SCHEMA_VERSION: schema_version,
'msg_category': 'warning',
'msg': f'JSON sidecar {display_name} had validation errors'
}
else:
return {
base_constants.COMMAND: base_constants.COMMAND_VALIDATE,
'data': '',
base_constants.SCHEMA_VERSION: schema_version,
'msg_category': 'success',
'msg': f'JSON file {display_name} had no validation errors'
}
def assemble(hed_schema, events, expand_defs=True):
"""Creates a two-column event file with first column Onset and second column HED tags.
Parameters
----------
hed_schema: HedSchema
A HED schema
events: model.EventsInput
An events input object
expand_defs: bool
True if definitions should be expanded during assembly
Returns
-------
dict
A dictionary pointing to assembled string or errors
"""
schema_version = hed_schema.header_attributes.get('version',
'Unknown version')
results = validate(hed_schema, events)
if results['data']:
return results
hed_tags = []
onsets = []
for row_number, row_dict in events.iter_dataframe(return_row_dict=True,
expand_defs=expand_defs,
remove_definitions=True):
hed_tags.append(str(row_dict.get("HED", "")))
onsets.append(row_dict.get("onset", "n/a"))
data = {'onset': onsets, 'HED': hed_tags}
df = pd.DataFrame(data)
csv_string = df.to_csv(None, sep='\t', index=False, header=True)
display_name = events.name
file_name = generate_filename(display_name,
name_suffix='_expanded',
extension='.tsv')
return {
base_constants.COMMAND: base_constants.COMMAND_ASSEMBLE,
'data': csv_string,
'output_display_name': file_name,
'schema_version': schema_version,
'msg_category': 'success',
'msg': 'Events file successfully expanded'
}
def spreadsheet_validate(hed_schema, spreadsheet, check_for_warnings=False):
""" Validates the spreadsheet.
Parameters
----------
hed_schema: str or HedSchema
Version number or path or HedSchema object to be used
spreadsheet: HedFileInput
Spreadsheet input object to be validated
check_for_warnings: bool
Indicates whether validation should check for warnings as well as errors
Returns
-------
dict
A dictionary containing results of validation in standard format
"""
schema_version = hed_schema.header_attributes.get('version',
'Unknown version')
validator = HedValidator(hed_schema=hed_schema)
issues = spreadsheet.validate_file(validator,
check_for_warnings=check_for_warnings)
display_name = spreadsheet.name
if issues:
issue_str = get_printable_issue_string(
issues, f"Spreadsheet {display_name} validation errors")
file_name = generate_filename(display_name,
name_suffix='_validation_errors',
extension='.txt')
return {
base_constants.COMMAND: base_constants.COMMAND_VALIDATE,
'data': issue_str,
"output_display_name": file_name,
base_constants.SCHEMA_VERSION: schema_version,
"msg_category": "warning",
'msg': f"Spreadsheet {display_name} had validation errors"
}
else:
return {
base_constants.COMMAND: base_constants.COMMAND_VALIDATE,
'data': '',
base_constants.SCHEMA_VERSION: schema_version,
'msg_category': 'success',
'msg': f'Spreadsheet {display_name} had no validation errors'
}
def schema_validate(hed_schema, display_name):
"""Run schema compliance for HED-3G
Parameters
----------
hed_schema: HedSchema object
A HedSchema object containing the schema to be processed.
display_name: str
The display name associated with this schema object.
Returns
-------
result: dict
A dictionary in the standard results format containing the results of the operation
"""
schema_version = hed_schema.header_attributes.get('version', 'Unknown')
issues = hed_schema.check_compliance()
if issues:
issue_str = get_printable_issue_string(
issues, f"Schema HED 3G compliance errors for {display_name}")
file_name = generate_filename(
display_name,
name_suffix='schema_3G_compliance_errors',
extension='.txt')
return {
'command': base_constants.COMMAND_VALIDATE,
'data': issue_str,
'output_display_name': file_name,
'schema_version': schema_version,
'msg_category': 'warning',
'msg': 'Schema is not HED 3G compliant'
}
else:
return {
'command': base_constants.COMMAND_VALIDATE,
'data': '',
'output_display_name': display_name,
'schema_version': schema_version,
'msg_category': 'success',
'msg': 'Schema had no HED-3G validation errors'
}
def test_generate_file_name(self):
file1 = generate_filename('mybase')
self.assertEqual(file1, "mybase", "generate_file_name should return the base when other arguments not set")
file2 = generate_filename('mybase', name_prefix="prefix")
self.assertEqual(file2, "prefixmybase", "generate_file_name should return correct name when prefix set")
file3 = generate_filename('mybase', name_prefix="prefix", extension=".json")
self.assertEqual(file3, "prefixmybase.json", "generate_file_name should return correct name for extension")
file4 = generate_filename('mybase', name_suffix="suffix")
self.assertEqual(file4, "mybasesuffix", "generate_file_name should return correct name when suffix set")
file5 = generate_filename('mybase', name_suffix="suffix", extension=".json")
self.assertEqual(file5, "mybasesuffix.json", "generate_file_name should return correct name for extension")
file6 = generate_filename('mybase', name_prefix="prefix", name_suffix="suffix", extension=".json")
self.assertEqual(file6, "prefixmybasesuffix.json",
"generate_file_name should return correct name for all set")
filename = generate_filename(None, name_prefix=None, name_suffix=None, extension=None)
self.assertEqual('', filename, "Return empty when all arguments are none")
filename = generate_filename(None, name_prefix=None, name_suffix=None, extension='.txt')
self.assertEqual('', filename,
"Return empty when base_name, prefix, and suffix are None, but extension is not")
filename = generate_filename('c:/temp.json', name_prefix=None, name_suffix=None, extension='.txt')
self.assertEqual('c_temp.txt', filename,
"Returns stripped base_name + extension when prefix, and suffix are None")
filename = generate_filename('temp.json', name_prefix='prefix_', name_suffix='_suffix', extension='.txt')
self.assertEqual('prefix_temp_suffix.txt', filename,
"Return stripped base_name + extension when prefix, and suffix are None")
filename = generate_filename(None, name_prefix='prefix_', name_suffix='suffix', extension='.txt')
self.assertEqual('prefix_suffix.txt', filename,
"Returns correct string when no base_name")
filename = generate_filename('event-strategy-v3_task-matchingpennies_events.json',
name_suffix='_blech', extension='.txt')
self.assertEqual('event-strategy-v3_task-matchingpennies_events_blech.txt', filename,
"Returns correct string when base_name with hyphens")
filename = generate_filename('HED7.2.0.xml', name_suffix='_blech', extension='.txt')
self.assertEqual('HED7.2.0_blech.txt', filename, "Returns correct string when base_name has periods")
def sidecar_convert(hed_schema,
json_sidecar,
command=base_constants.COMMAND_TO_SHORT,
expand_defs=False):
"""Converts a sidecar from long to short or short to long
Parameters
----------
hed_schema:HedSchema
HedSchema object to be used
json_sidecar: Sidecar
Previously created Sidecar
command: str
Name of the command to execute (default to short if unrecognized)
expand_defs: bool
Indicates whether to expand definitions when converting
Returns
-------
dict
A downloadable dictionary file or a file containing warnings
"""
schema_version = hed_schema.header_attributes.get('version',
'Unknown version')
results = sidecar_validate(hed_schema,
json_sidecar,
check_for_warnings=False)
if results['data']:
return results
if command == base_constants.COMMAND_TO_LONG:
tag_form = 'long_tag'
else:
tag_form = 'short_tag'
issues = []
for hed_string_obj, position_info, issue_items in json_sidecar.hed_string_iter(
validators=hed_schema,
expand_defs=expand_defs,
remove_definitions=False):
converted_string = hed_string_obj.get_as_form(tag_form)
issues = issues + issue_items
json_sidecar.set_hed_string(converted_string, position_info)
# issues = ErrorHandler.filter_issues_by_severity(issues, ErrorSeverity.ERROR)
display_name = json_sidecar.name
if issues:
issue_str = get_printable_issue_string(
issues, f"JSON conversion for {display_name} was unsuccessful")
file_name = generate_filename(
display_name,
name_suffix=f"_{tag_form}_conversion_errors",
extension='.txt')
return {
base_constants.COMMAND: command,
'data': issue_str,
'output_display_name': file_name,
base_constants.SCHEMA_VERSION: schema_version,
'msg_category': 'warning',
'msg': f'JSON file {display_name} had validation errors'
}
else:
file_name = generate_filename(display_name,
name_suffix=f"_{tag_form}",
extension='.json')
data = json_sidecar.get_as_json_string()
return {
base_constants.COMMAND: command,
'data': data,
'output_display_name': file_name,
base_constants.SCHEMA_VERSION: schema_version,
'msg_category': 'success',
'msg': f'JSON sidecar {display_name} was successfully converted'
}