def test_check_for_definitions_placeholder(self):
def_dict = DefDict()
original_def_count = len(def_dict._defs)
hed_string_obj = HedString(self.placeholder_def_string)
hed_string_obj.validate(def_dict)
new_def_count = len(def_dict._defs)
self.assertGreater(new_def_count, original_def_count)
def validator_base(self,
test_strings,
expected_results,
expected_issues,
test_function,
hed_schema=None):
for test_key in test_strings:
hed_string_obj = HedString(test_strings[test_key])
error_handler = ErrorHandler()
error_handler.push_error_context(ErrorContext.HED_STRING,
hed_string_obj,
increment_depth_after=False)
test_issues = []
if self.compute_forms:
test_issues += hed_string_obj.convert_to_canonical_forms(
hed_schema)
if not test_issues:
test_issues += test_function(hed_string_obj)
test_result = not test_issues
expected_params = expected_issues[test_key]
expected_result = expected_results[test_key]
expected_issue = self.format_errors_fully(
error_handler,
hed_string=hed_string_obj,
params=expected_params)
error_handler.add_context_to_issues(test_issues)
# print(test_key)
# print(str(expected_issue))
# print(str(test_issues))
error_handler.pop_error_context()
self.assertEqual(test_result, expected_result,
test_strings[test_key])
self.assertCountEqual(test_issues, expected_issue,
test_strings[test_key])
def _hed_iter(self, also_return_bad_types=False):
"""
Iterates over all the hed string entries, returning HedString objects.
Parameters
----------
also_return_bad_types: bool
If true, this will also yield types other than HedString
Returns
-------
hed_string: HedString
position: str
The position to pass back to set this string.
"""
hed_strings = self._hed_dict.get("HED", None)
if isinstance(hed_strings, dict):
for key, hed_string in hed_strings.items():
if isinstance(hed_string, str):
hed_string = HedString(hed_string)
elif not also_return_bad_types:
continue
yield hed_string, key
elif isinstance(hed_strings, str):
hed_string = HedString(hed_strings)
yield hed_string, None
def check_def_base(self, test_strings, expected_issues):
for test_key in test_strings:
def_dict = DefDict()
hed_string_obj = HedString(test_strings[test_key])
hed_string_obj.convert_to_canonical_forms(None)
test_issues = def_dict.check_for_definitions(hed_string_obj)
expected_issue = expected_issues[test_key]
self.assertCountEqual(test_issues, expected_issue,
HedString(test_strings[test_key]))
def test_string_convert_to_long(self):
from hedweb.strings import convert
string_list = [HedString('Red'), HedString('Blue')]
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
with self.app.app_context():
results = convert(hed_schema,
string_list,
command=base_constants.COMMAND_TO_LONG)
self.assertEqual(
'success', results['msg_category'],
"hedstring_convert should return success if converted")
def get_input_from_form(request):
"""Gets input arguments from a request object associated with the string form.
Parameters
----------
request: Request object
A Request object containing user data from the string form.
Returns
-------
dict
A dictionary containing input arguments for calling the underlying string processing functions.
"""
hed_schema = get_hed_schema_from_pull_down(request)
hed_string = request.form.get(base_constants.STRING_INPUT, None)
if hed_string:
string_list = [HedString(hed_string)]
else:
raise HedFileError('EmptyHedString', 'Must enter a HED string', '')
arguments = {
base_constants.COMMAND:
request.form.get(base_constants.COMMAND_OPTION, ''),
base_constants.SCHEMA:
hed_schema,
base_constants.STRING_LIST:
string_list,
base_constants.CHECK_FOR_WARNINGS:
form_has_option(request, base_constants.CHECK_FOR_WARNINGS, 'on')
}
return arguments
def _expand_column(self, column_number, input_text):
"""
Expand the given text based on the rules for expanding this column
Parameters
----------
column_number : int
The column number this text should be treated as from
input_text : str
The text to expand, generally a single cell of a spreadsheet.
Returns
-------
expanded_text : str or None
The text after expansion. Returns None if this column is undefined or the given text is null.
attribute_name_or_error_message: False or str
Depends on the value of first return value.
If None, this is an error message.
If string, this is an attribute name, that should be stored separately..
"""
# Default 1-1 mapping if we don't have specific behavior.
if not self._final_column_map:
return HedString(input_text), False
# If no entry, ignore this column.
if column_number not in self._final_column_map:
return None, False
if not input_text or input_text in self._na_patterns:
return None, False
column_entry = self._final_column_map[column_number]
return column_entry.expand(input_text)
def expand(self, input_text):
"""
Expands the input_text based on the rules for this column.
Eg adding name_prefix, inserting a column hed_string from key, etc.
Parameters
----------
input_text : str
Text to expand(generally from a single cell in a spreadsheet)
Returns
-------
hed_string: str
The expanded column as a hed_string
attribute_name_or_error_message: str or {}
If this is a string, contains the name of this column as an attribute.
If the first return value is None, this is an error message dict.
"""
column_type = self.column_type
if column_type == ColumnType.Categorical:
final_text = self._get_category_hed_string(input_text)
if final_text:
return HedString(final_text), False
else:
return None, ErrorHandler.format_error(
ValidationErrors.HED_SIDECAR_KEY_MISSING,
invalid_key=input_text,
category_keys=list(self._hed_dict["HED"].keys()))
elif column_type == ColumnType.Value:
prelim_text = self._get_value_hed_string()
final_text = prelim_text.replace("#", input_text)
return HedString(final_text), False
elif column_type == ColumnType.HEDTags:
hed_string_obj = HedString(input_text)
final_text = self._prepend_prefix_to_required_tag_column_if_needed(
hed_string_obj, self.column_prefix)
return final_text, False
elif column_type == ColumnType.Ignore:
return None, False
elif column_type == ColumnType.Attribute:
return input_text, self.column_name
return None, {"error_type": "INTERNAL_ERROR"}
def test_string_validate(self):
from hedweb.strings import validate
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
string_list = [HedString('Red'), HedString('Blech')]
with self.app.app_context():
results = validate(hed_schema, string_list)
self.assertEqual('warning', results['msg_category'],
"validate has warning if validation errors")
string_list = [HedString('Red'), HedString('Blue')]
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
with self.app.app_context():
results = validate(hed_schema, string_list)
self.assertEqual('success', results['msg_category'],
"validate should return success if converted")
def test_string_convert_to_short_invalid(self):
from hedweb.strings import convert
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
string_list = [HedString('Red, Blech')]
with self.app.app_context():
results = convert(hed_schema, string_list)
self.assertEqual(
'warning', results['msg_category'],
"hedstring_convert issue warning if unsuccessful")
def test_string_convert_to_short_valid(self):
from hedweb.strings import convert
schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'data/HED8.0.0.xml')
hed_schema = hedschema.load_schema(schema_path)
string_list = [
HedString(
'Property/Informational-property/Description/Blech, Blue')
]
with self.app.app_context():
results = convert(hed_schema, string_list,
base_constants.COMMAND_TO_SHORT)
data = results['data']
self.assertTrue(data, 'convert string to short returns data')
self.assertIsInstance(
data, list, "convert string to short returns data in a list")
self.assertEqual(
"Description/Blech,Blue", data[0],
"convert string to short returns the correct short form.")
self.assertEqual(
'success', results['msg_category'],
"hedstring_convert should return success if converted")
from hed.errors.error_reporter import get_printable_issue_string
from hed.models.hed_string import HedString
from hed.schema.hed_schema_io import load_schema
from hed.validator.hed_validator import HedValidator
if __name__ == '__main__':
hed_xml_url = 'https://raw.githubusercontent.com/hed-standard/hed-specification/master/hedxml/HED8.0.0.xml'
hed_schema = load_schema(hed_xml_url)
hed_validator = HedValidator(hed_schema)
hed_validator_no_semantic = HedValidator(run_semantic_validation=False)
hed_string_1 = \
"Sensory-event,Visual-presentation,Experimental-stimulus,Green,Non-target," + \
"(Letter/D, (Center-of, Computer-screen))"
string_obj_1 = HedString(hed_string_1)
validation_issues = string_obj_1.validate(hed_validator)
print(
get_printable_issue_string(
validation_issues,
title=
'[Example 1] hed_string_1 should have no issues with HEDv8.0.0'))
string_1_long, issues = string_obj_1.convert_to_short(hed_schema)
print(
get_printable_issue_string(
issues,
title=
'[Example 2] hed_string_1 should convert to long without errors'))
string_1_long_obj = HedString(string_1_long)
validation_issues = string_1_long_obj.validate(hed_validator)
def iter_dataframe(self,
mapper=None,
return_row_dict=False,
validators=None,
run_string_ops_on_columns=False,
error_handler=None,
expand_defs=False,
remove_definitions=True,
**kwargs):
"""
Generates a list of parsed rows based on the given column mapper.
Parameters
----------
mapper : ColumnMapper
The column name to column number mapper
return_row_dict: bool
If True, this returns the full row_dict including issues.
If False, returns just the HedStrings for each column
error_handler : ErrorHandler
The error handler to use for context, uses a default one if none.
validators : [func or validator like] or func or validator like
A validator or list of validators to apply to the hed strings before returning
run_string_ops_on_columns: bool
If true, run all tag and string ops on columns, rather than columns then rows.
expand_defs: bool
If True, this will fully remove all definitions found and expand all def tags to def-expand tags
remove_definitions: bool
If true, this will remove all definition tags found.
kwargs:
See util.translate_ops or the specific validators for additional options
Yields
-------
row_number: int
The current row number
row_dict: dict
A dict containing the parsed row, including: "HED", "column_to_hed_tags", and possibly "column_issues"
"""
if error_handler is None:
error_handler = ErrorHandler()
if mapper is None:
mapper = self._mapper
tag_ops, string_ops = self._translate_ops(
validators,
run_string_ops_on_columns=run_string_ops_on_columns,
expand_defs=expand_defs,
remove_definitions=remove_definitions,
error_handler=error_handler,
**kwargs)
start_at_one = 1
if self._has_column_names:
start_at_one += 1
for row_number, text_file_row in self._dataframe.iterrows():
# Skip any blank lines.
if all(text_file_row.isnull()):
continue
row_dict = mapper.expand_row_tags(text_file_row)
column_to_hed_tags = row_dict[model_constants.COLUMN_TO_HED_TAGS]
expansion_column_issues = row_dict.get(
model_constants.COLUMN_ISSUES, {})
error_handler.push_error_context(ErrorContext.ROW, row_number)
row_issues = []
if tag_ops:
row_issues += self._run_column_ops(column_to_hed_tags, tag_ops,
expansion_column_issues,
error_handler)
if return_row_dict:
final_hed_string = HedString.create_from_other(
column_to_hed_tags.values())
if string_ops:
row_issues += self._run_row_ops(final_hed_string,
string_ops, error_handler)
row_dict[model_constants.ROW_ISSUES] = row_issues
row_dict[model_constants.ROW_HED_STRING] = final_hed_string
yield row_number + start_at_one, row_dict
else:
yield row_number + start_at_one, column_to_hed_tags
error_handler.pop_error_context()
def short_to_long_string(input_string, hed_schema):
hed_string_obj = HedString(input_string)
return hed_string_obj.convert_to_long(hed_schema)