def _validate_explorerConfig_helper(explorer_config, type_prop_map, errors):
datatype = explorer_config["guppyConfig"]["dataType"]
props = type_prop_map.get(datatype, [])
if not props:
errors.append(
FieldError(
"Field [{}] in explorerConfig.guppyConfig.dataType not found in etlMapping"
.format(datatype)))
tabs = explorer_config["filters"]["tabs"]
for tab in tabs:
check_field_value("explorerConfig.filters.tabs.fields",
tab.get("fields", []), props, errors)
table = explorer_config["table"]
if table["enabled"]:
check_field_value("explorerConfig.table.fields",
table.get("fields", []), props, errors)
manifest_map = explorer_config.get("manifestMapping")
if manifest_map and manifest_map.get("resourceIndexType"):
resource_props = type_prop_map.get(
manifest_map.get("resourceIndexType"))
if not resource_props:
errors.append(
FieldError(
"Field [{}] in manifestMapping.resourceIndexType not found in etlMapping"
.format(manifest_map.get("resourceIndexType"))))
elif manifest_map.get("resourceIdField") not in resource_props:
errors.append(
FieldError(
"Field [{}] in manifestMapping.resourceIdField not found in etlMapping"
.format(manifest_map.get("resourceIdField"))))
def test_validate_against_etl(gitops_json):
errors = validate_against_etl(gitops_json,
"tests/data/etlMapping_gitops.yaml")
assert errors == [
FieldError(
"Field [investigator_affiliation] in explorerConfig.filters.tabs.fields not found in etlMapping"
),
FieldError(
"Field [investigator_affiliation] in explorerConfig.table.fields not found in etlMapping"
),
]
def _validate_studyViewer_datatypes(viewer, datatype, type_prop_map,
row_accessor, errors):
dtype = viewer.get(datatype)
props = type_prop_map.get(dtype, [])
if not props:
errors.append(
FieldError(
"Field [{}] in studyViewerConfig.{} not found in etlMapping".
format(dtype, datatype)))
if row_accessor not in props:
errors.append(
FieldError(
"rowAccessor [{}] not found in index with type {}".format(
row_accessor, dtype)))
def validate_joining_src(json_obj, recorded_errors, joining_props):
src = json_obj.get("src", json_obj.get("name"))
alternate_src = json_obj.get("alternate_src", json_obj.get("name"))
if src is not None:
if src not in joining_props:
if alternate_src in joining_props:
print("WARN: Utilizing alternate src")
return
recorded_errors.append(
FieldError(
'src field "{}" (declared in "{}") is not found in joining index.'
.format(src, json_obj)))
else:
recorded_errors.append(
FieldError('Missing source field for "{}"'.format(json_obj)))
def test_validate_explorerConfig(gitops_json, etl_prop_type_map):
errors = validate_explorerConfig(gitops_json, etl_prop_type_map, [])
assert errors == [
FieldError(
"Field [investigator_affiliation] in explorerConfig.filters.tabs.fields not found in etlMapping"
),
FieldError(
"Field [investigator_affiliation] in explorerConfig.table.fields not found in etlMapping"
),
]
gitops_json["explorerConfig"][0]["table"]["enabled"] = False
errors = validate_explorerConfig(gitops_json, etl_prop_type_map, [])
assert errors == [
FieldError(
"Field [investigator_affiliation] in explorerConfig.filters.tabs.fields not found in etlMapping"
)
]
def test_validate_studyViewerConfig(gitops_json, etl_prop_type_map):
study_viewer = gitops_json["studyViewerConfig"]
errors = []
validate_studyViewerConfig(study_viewer, etl_prop_type_map, errors)
assert errors == []
# valid datatype but invalid fields
study_viewer[0]["dataType"] = "dataset"
errors = []
validate_studyViewerConfig(study_viewer, etl_prop_type_map, errors)
assert errors == [
FieldError(
"Field [project_id] in studyViewerConfig.blockFields.listItemConfig not found in etlMapping"
),
FieldError(
"Field [study_doi] in studyViewerConfig.tableFields.singleItemConfig not found in etlMapping"
),
]
# invalid datatype
study_viewer[0]["dataType"] = "error"
errors = []
validate_studyViewerConfig(study_viewer, etl_prop_type_map, errors)
assert errors == [
FieldError(
"Field [error] in studyViewerConfig.dataType not found in etlMapping"
),
FieldError("rowAccessor [code] not found in index with type error"),
FieldError(
"Field [project_id] in studyViewerConfig.blockFields.listItemConfig not found in etlMapping"
),
FieldError(
"Field [study_doi] in studyViewerConfig.tableFields.singleItemConfig not found in etlMapping"
),
]
def validate_name_src(
json_obj,
path,
recorded_errors,
nodes_with_props,
labels_to_back_refs=None,
nodes_for_category=None,
):
if not labels_to_back_refs:
labels_to_back_refs = {}
if not nodes_for_category:
nodes_for_category = []
name = validate_name(json_obj, recorded_errors)
fn = validate_fn(json_obj, recorded_errors)
src = json_obj.get("src", name)
if not src:
return name
if not path and not nodes_for_category:
recorded_errors.append(
FieldError(
'src field must be specified with a path for "{}"'.format(
json_obj)))
else:
valid_fields = {"source_node"} # built-in fields
if path:
path_items = path.split(".")
valid_fields.update(nodes_with_props.get(path_items[-1], []))
for node_name in nodes_for_category:
node_backref = labels_to_back_refs[node_name]
node_props = nodes_with_props.get(node_backref, [])
valid_fields.update(node_props)
if fn != "count" and src not in valid_fields:
recorded_errors.append(
FieldError(
'src field "{}" (declared in {} "{}") is not found in given dictionary.'
.format(src, f'"{path}"' if path else "a collector index",
json_obj)))
return name
def test_check_field_value():
checks = ["check1", "check4"]
accepted_vals = ["check1", "check2", "check3"]
errors = []
check_field_value("path", checks, accepted_vals, errors)
assert errors == [
FieldError("Field [check4] in path not found in etlMapping")
]
checks = ["check1", "check2"]
accepted_vals = ["check1", "check2", "check3"]
errors = ["error"]
check_field_value("path", checks, accepted_vals, errors)
assert errors == ["error"]
def test_val_gitops():
# syntax error
with pytest.raises(AssertionError):
val_gitops(data_dict, etlMapping,
"tests/data/gitops_syntax_error.json")
# etl mapping errors
errors, ok = val_gitops(data_dict, etlMapping,
"tests/data/gitops_test.json")
assert errors == [
FieldError(
"Field [investigator_affiliation] in explorerConfig.filters.tabs.fields not found in etlMapping"
),
FieldError(
"Field [investigator_affiliation] in explorerConfig.table.fields not found in etlMapping"
),
]
assert ok
# dictionary error
_, ok = val_gitops(data_dict, etlMapping,
"tests/data/gitops_dict_error.json")
assert not ok
def validate_gitops_syntax(gitops):
"""
Validates the syntax of gitops.json by checking for required fields
Args:
gitops (dict): gitops.json config
Returns:
Error: Returns the error if it exists, else returns None.
TODO
Currently only checks syntax needed for etl mapping and dictionary validation.
"""
graphql = gitops.get("graphql")
ok = assert_and_log(graphql, FieldSyntaxError("graphql"))
if graphql:
ok = ok and assert_and_log("boardCounts" in graphql,
FieldSyntaxError("graphql.boardCounts"))
boardcounts = graphql.get("boardCounts", [])
if boardcounts:
for item in boardcounts:
checks = ["graphql", "name", "plural"]
ok = check_required_fields("graphql.boardCounts", checks, item,
ok)
ok = ok and assert_and_log("chartCounts" in graphql,
FieldSyntaxError("graphql.chartCounts"))
components = gitops.get("components")
ok = ok and assert_and_log(components, FieldSyntaxError("components"))
if components:
index = components.get("index")
ok = ok and assert_and_log(index, FieldSyntaxError("components.index"))
if index:
homepage = index.get("homepageChartNodes", [])
ok = ok and assert_and_log(
index, FieldSyntaxError("components.homepageChartNodes"))
for item in homepage:
checks = ["node", "name"]
ok = check_required_fields("components.index.homepage", checks,
item, ok)
# footerLogos is optional, but when present, it must be an array (list)
footerLogos = components.get("footerLogos", [])
ok = ok and assert_and_log(
isinstance(footerLogos, list),
FieldError(
"footerLogos must be an array if presents in the config"),
)
explorer_enabled = gitops.get("featureFlags", {}).get("explorer", True)
explorerconfig = gitops.get("explorerConfig")
configs = []
if not explorerconfig:
dataConfig = gitops.get("dataExplorerConfig")
fileConfig = gitops.get("fileExplorerConfig")
ok = ok and assert_and_log(dataConfig or not explorer_enabled,
FieldSyntaxError("(data)explorerConfig"))
if dataConfig:
configs.append(dataConfig)
if fileConfig:
configs.append(fileConfig)
else:
configs = explorerconfig
for exp_config in configs:
filters = exp_config.get("filters")
ok = ok and assert_and_log(filters,
FieldSyntaxError("explorerConfig.filters"))
if filters:
tabs = filters.get("tabs", [])
ok = ok and assert_and_log(
tabs, FieldSyntaxError("explorerConfig.filters.tabs"))
for tab in tabs:
checks = ["title", "fields"]
ok = check_required_fields("explorerConfig.filters.tab",
checks, tab, ok)
guppy = exp_config.get("guppyConfig")
ok = ok and assert_and_log(
guppy, FieldSyntaxError("explorerConfig.guppyConfig"))
buttons = exp_config.get("buttons", [])
manifest_mapping = None
if guppy:
ok = ok and assert_and_log(
guppy.get("dataType"),
FieldSyntaxError("explorerConfig.guppyConfig.dataType"),
)
manifest_mapping = guppy.get("manifestMapping")
val_mapping = False
for button in buttons:
if button.get("enabled") and button.get("type") == "manifest":
val_mapping = True
if manifest_mapping and val_mapping:
checks = [
"resourceIndexType",
"resourceIdField",
"referenceIdFieldInResourceIndex",
"referenceIdFieldInDataIndex",
]
ok = check_required_fields(
"explorerConfig.guppyConfig.manifestMapping",
checks,
manifest_mapping,
ok,
)
study_viewer = gitops.get("studyViewerConfig")
if study_viewer:
checks = [
"dataType",
"listItemConfig",
"rowAccessor",
]
for viewer in study_viewer:
ok = check_required_fields("studyViewerConfig", checks, viewer, ok)
return ok
def check_field_value(path, checks, accepted_values, errors):
for check in checks:
if check not in accepted_values:
errors.append(
FieldError("Field [{}] in {} not found in etlMapping".format(
check, path)))