def run_convert(cns_model, item, types, primary_keys, report=None):
"""
property_alias => property_name
create @id
assert @type
"""
assert types
if primary_keys:
assert type(primary_keys) == list
cns_item = {
"@type": types,
}
for p, v in item.items():
px = cns_model.index_property_alias.get(p)
if px:
cns_item[px] = v
else:
bug = {
"category": "warn_convert_cns",
"text": "property not defined in schema",
"property": p
}
if report is not None:
write_report(report, bug)
if item.get("@id"):
cns_item["@id"] = item["@id"]
xid = gen_cns_id(cns_item, primary_keys)
cns_item["@id"] = xid
return cns_item
def _validate_entity_ref(c, p, v, range_actual, range_config, report):
xtype = json_get_list(v, "@type")
if not xtype:
bug = {
"category": "warn_validate_entity_ref",
"text": "value type missing",
"value": v,
"class": c,
"property": p,
"expected" : range_config["text"],
"actual" : None,
}
write_report(report, bug)
return
xtype_main = xtype[0]
if range_config["cns_range_entity"] and range_config["cns_range_entity"][0] in xtype:
pass
elif xtype_main in range_config["cns_range_entity"]:
pass
else:
bug = {
"category": "warn_validate_entity_ref",
"text": "value type/entity mismatch",
"value": v,
"class": c,
"property": p,
"expected" : range_config["cns_range_entity"],
"actual" : xtype,
}
write_report(report, bug)
def _validate_system_property(loaded_schema, cns_item, report):
# system property
types = cns_item["@type"]
if "Thing" in types:
if not "@id" in cns_item:
bug = {
"category": "warn_validate_system_property",
"text": "instance of [Thing] missing @id",
"item": cns_item
}
write_report(report, bug)
def _validate_datastructure(c, p, v, range_actual, range_config, report):
types = range_config["cns_range_datastructure"]
if len(types) == 0:
bug = {
"category": "warn_validate_datastructure",
"text": "value range not specified as datastructure",
"value": v,
"class": c,
"property": p,
"expected" : range_config["text"],
"actual" : None,
}
write_report(report, bug)
return types
def run_validate(loaded_schema, cns_item, report):
"""
validate the following
* template restriction (class-property binding)
* range of property
"""
#stats
report["stats"]["items_validate"] += 1
if XTEMPLATE not in report:
report[XTEMPLATE] = collections.Counter()
#check types
types = cns_item.get("@type")
if types is None:
bug = {
"category": "warn_validate",
"text": "missing @type and no expected @type",
"item": cns_item
}
write_report(report, bug)
return report
_rewrite_item(loaded_schema, cns_item, report)
_count_cnslink(loaded_schema, cns_item, report)
_validate_system_property(loaded_schema, cns_item, report)
_validate_template(loaded_schema, cns_item, cns_item["@type"], report)
#_validate_range(loaded_schema, cns_item, report)
#_validate_domain(loaded_schema, cns_item, report)
return report
def _validate_datatype(c, p, v, range_actual, range_config, report):
if p in ["in","out"]:
# do not validate system property
return
if not range_actual in range_config["python_type_value_list"]:
bug = {
"category": "warn_validate_datatype",
"text": "range value datatype mismatch",
"actualValue": v,
"class": c,
"property": p,
"expected" : range_config["text"],
"actual" : str(range_actual),
}
write_report(report, bug)
#logging.info(json4debug(range_config["text"]))
if range_config["text"].lower() == "date":
ret = iso8601_date_parse(v)
if not ret:
bug = {
"category": "warn_validate_datatype",
"text": "range value is valid Date string",
"actualValue": v,
"class": c,
"property": p,
"expected" : range_config["text"],
"actual" : str(range_actual),
}
write_report(report, bug)
#assert False
elif range_config["text"].lower() == "datetime":
ret = iso8601_datetime_parse(v)
if not ret:
bug = {
"category": "warn_validate_datatype",
"text": "range value is valid DateTime string",
"actualValue": v,
"class": c,
"property": p,
"expected" : range_config["text"],
"actual" : str(range_actual),
}
write_report(report, bug)
def _validate_template_regular(loaded_schema, cns_item, types, report, validated_property):
#regular validation
main_type = types[0]
for idx, xtype in enumerate(types):
# only count main type's template
key_c = u"type_all_{}".format(xtype)
report[XTEMPLATE][key_c] += 1
if idx == 0:
key_c = u"type_top_{}".format(xtype)
report[XTEMPLATE][key_c] += 1
#find templates
template_map = loaded_schema.index_validate_template.get(xtype)
if template_map is None or len(template_map)==0:
# bug = {
# "category": "warn_validate_template",
# "text": "no template found",
# "class": xtype,
# "item": cns_item,
# }
# write_report(report, bug)
continue
#validate one by one
for template in template_map.values():
p = template["refProperty"]
# only count main type's template
key_cp = u"cp_{}_{}_{}".format(main_type, xtype, p)
if p in cns_item:
report[XTEMPLATE][key_cp] += 1
else:
report[XTEMPLATE][key_cp] += 0
if p in validated_property:
# validated, no need to be validated in other templates
continue
else:
validated_property.add(p)
#validate cardinality
values = json_get_list(cns_item, p)
card_actual = len(values)
range_config = template["propertyRange"]
if len(range_config["python_type_value_list"])>0 or len(range_config["cns_range_datastructure"])>0:
if card_actual < template["minCardinality"]:
# logging.info(json4debug(template))
# logging.info(json4debug(cns_item))
# assert False
bug = {
"category": "warn_validate_template_regular",
"text": "minCardinality",
"class": xtype,
"property": p,
"expected": template["minCardinality"],
"actual": card_actual,
"item_name": cns_item.get("name"),
"item_value": cns_item.get(p),
}
write_report(report, bug)
if "maxCardinality" in template:
if card_actual > template["maxCardinality"]:
bug = {
"category": "warn_validate_template_regular",
"text": "maxCardinality",
"class": xtype,
"property": p,
"expected": template["maxCardinality"],
"actual": card_actual,
"item_name": cns_item.get("name"),
"item_value": cns_item.get(p),
}
write_report(report, bug)
if card_actual == 0:
# no further validation on range
continue
#logging.info(template)
for v in values:
range_actual = type(v)
if range_actual in [dict]:
#CnsDataStructure
if p in ["in","out"]:
_validate_entity_ref(xtype, p, v, range_actual, range_config, report)
elif "@id" in v:
_validate_entity_ref(xtype, p, v, range_actual, range_config, report)
v_types = json_get_list(v, "@type")
if v_types:
_validate_template(loaded_schema, v, v_types, report)
else:
v_types = _validate_datastructure(xtype, p, v, range_actual, range_config, report)
if v_types:
if len(v_types) == 1:
v_types = loaded_schema.index_inheritance["rdfs:subClassOf"].get(v_types[0])
_validate_template(loaded_schema, v, v_types, report)
elif range_actual in [list]:
assert False #unexpected
else:
type_actual = _validate_datatype(xtype, p, v, range_actual, range_config, report)
#properties not validate by main template
all_property = set(cns_item.keys())
all_property = all_property.difference(get_system_property())
all_property = all_property.difference(validated_property)
c = types[0]
for p in all_property:
if p.startswith("rdfs:"):
continue
#if p in ["in","out"]:
# continue
bug = {
"category": "warn_validate_template_regular",
"text": "property not validated by main template",
"value": cns_item,
"class": c,
"property": p,
}
#logging.info(bug)
write_report(report, bug)
# not validated properties for main type
key_cp = u"ucp_{}_{}".format(c, p)
report[XTEMPLATE][key_cp] += 1
def _rewrite_item(loaded_schema, cns_item, report):
"""
fix bugs in item, keep on validation
"""
# rewrite type
types = cns_item.get("@type")
if not isinstance(types, list):
bug = {
"category": "warn_rewrite_item",
"text": " @type got string value",
"item": cns_item
}
write_report(report, bug)
types = convert_cns_type_string(types)
#rewrite type
cns_item["@type"] = types
types = cns_item.get("@type")
if types is None:
return
#remove undefined type
types_new = []
for xtype in types:
has_definition = False
for schema in loaded_schema.loaded_schema_list:
the_definition = schema.get_definition_by_alias(xtype)
if the_definition:
has_definition =True
break
if not has_definition:
bug = {
"category": "warn_rewrite_item",
"text": "class not defined",
"class" : xtype,
#"item": cns_item
}
write_report(report, bug)
else:
types_new.append(xtype)
cns_item["@type"] = types_new
#undefined property
bad_property_list = []
for property in cns_item:
if property in get_system_property():
continue
has_definition = False
for schema in loaded_schema.loaded_schema_list:
the_definition = schema.get_definition_by_alias(property)
if the_definition:
has_definition =True
break
if not has_definition:
bug = {
"category": "warn_rewrite_item",
"text": "property not defined",
"property" : property,
#"item": cns_item
}
write_report(report, bug)
bad_property_list.append(property)
for p in bad_property_list:
del cns_item[p]