def addReferencedFile(docElt, elt):
if elt.tag in ("a", "img", "{http://www.w3.org/1999/xhtml}a", "{http://www.w3.org/1999/xhtml}img"):
for attrTag, attrValue in elt.items():
if (attrTag in ("href", "src") and
scheme(attrValue) not in ("data", "javascript") and (
not localFilesOnly or
(not isHttpUrl(attrValue) and not os.path.isabs(attrValue)))):
attrValue = attrValue.partition('#')[0] # remove anchor
if attrValue: # ignore anchor references to base document
base = docElt.modelDocument.baseForElement(docElt)
normalizedUri = docElt.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(attrValue, base)
if not docElt.modelXbrl.fileSource.isInArchive(normalizedUri):
normalizedUri = docElt.modelXbrl.modelManager.cntlr.webCache.getfilename(normalizedUri)
if modelXbrl.fileSource.isInArchive(normalizedUri, checkExistence=True) or modelXbrl.fileSource.exists(normalizedUri):
referencedFiles.add(attrValue) # add file name within source directory
def addReferencedFile(docElt, elt):
if elt.tag in ("a", "img", "{http://www.w3.org/1999/xhtml}a", "{http://www.w3.org/1999/xhtml}img"):
for attrTag, attrValue in elt.items():
if (attrTag in ("href", "src") and
scheme(attrValue) not in ("data", "javascript") and (
not localFilesOnly or
(not isHttpUrl(attrValue) and not os.path.isabs(attrValue)))):
attrValue = attrValue.partition('#')[0].strip() # remove anchor
if attrValue not in ("", "."): # ignore anchor references to base document
base = docElt.modelDocument.baseForElement(docElt)
normalizedUri = docElt.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(attrValue, base)
if not docElt.modelXbrl.fileSource.isInArchive(normalizedUri):
normalizedUri = docElt.modelXbrl.modelManager.cntlr.webCache.getfilename(normalizedUri)
if modelXbrl.fileSource.isInArchive(normalizedUri, checkExistence=True) or modelXbrl.fileSource.exists(normalizedUri):
referencedFiles.add(attrValue) # add file name within source directory
def validateHtmlContent(modelXbrl, referenceElt, htmlEltTree, validatedObjectLabel, messageCodePrefix, isInline=False):
checkedGraphicsFiles = set() # only check any graphics file reference once per footnote
_xhtmlNs = "{{{}}}".format(xhtml)
_xhtmlNsLen = len(_xhtmlNs)
_tableTags = ("table", _xhtmlNs + "table")
_anchorAncestorTags = set(_xhtmlNs + tag for tag in ("html", "body", "div"))
allowedExternalHrefPattern = modelXbrl.modelManager.disclosureSystem.allowedExternalHrefPattern
for elt in htmlEltTree.iter():
if isinstance(elt, ModelObject) and elt.namespaceURI == xhtml:
eltTag = elt.localName
elif isinstance(elt, (_ElementTree, _Comment, _ProcessingInstruction)):
continue # comment or other non-parsed element
else:
eltTag = elt.tag
if eltTag.startswith(_xhtmlNs):
eltTag = eltTag[_xhtmlNsLen:]
if isInline:
if eltTag in efmBlockedInlineHtmlElements:
modelXbrl.error("EFM.5.02.05.disallowedElement",
_("%(validatedObjectLabel)s has disallowed element <%(element)s>"),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
element=eltTag)
if eltTag == "a" and "href" not in elt.keys() and any(a.tag not in _anchorAncestorTags for a in elt.iterancestors()):
modelXbrl.warning("EFM.5.02.05.anchorElementPosition",
_("If element <a> does not have attribute @href, it should not have any ancestors other than html, body, or div. Disallowed ancestors: %(disallowedAncestors)s"),
modelObject=elt, disallowedAncestors=", ".join(a.tag.rpartition('}')[2] for a in elt.iterancestors() if a.tag not in _anchorAncestorTags))
for attrTag, attrValue in elt.items():
if isInline:
if attrTag in efmBlockedInlineHtmlElementAttributes.get(eltTag,()):
modelXbrl.error("EFM.5.02.05.disallowedAttribute",
_("%(validatedObjectLabel)s has disallowed attribute on element <%(element)s>: %(attribute)s=\"%(value)s\""),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
element=eltTag, attribute=attrTag, value=attrValue)
elif attrTag == "{http://www.w3.org/XML/1998/namespace}base":
modelXbrl.error("EFM.5.02.05.xmlBaseDisallowed",
_("%(validatedObjectLabel)s has disallowed attribute on element <%(element)s>: xml:base=\"%(value)s\""),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
element=eltTag, value=attrValue)
elif attrTag == "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation":
modelXbrl.warning("EFM.5.02.05.schemaLocationDisallowed",
_("%(validatedObjectLabel)s has disallowed attribute on element <%(element)s>: xsi:schemaLocation=\"%(value)s\""),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
element=eltTag, value=attrValue)
if ((attrTag == "href" and eltTag == "a") or
(attrTag == "src" and eltTag == "img")):
if "javascript:" in attrValue:
modelXbrl.error(messageCodePrefix + "activeContent",
_("%(validatedObjectLabel)s has javascript in '%(attribute)s' for <%(element)s>"),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
attribute=attrTag, element=eltTag,
messageCodes=("EFM.6.05.34.activeContent", "EFM.5.02.05.activeContent"))
elif eltTag == "a" and (not allowedExternalHrefPattern or allowedExternalHrefPattern.match(attrValue)):
pass
elif scheme(attrValue) in ("http", "https", "ftp"):
modelXbrl.error(messageCodePrefix + "externalReference",
_("%(validatedObjectLabel)s has an invalid external reference in '%(attribute)s' for <%(element)s>: %(value)s"),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
attribute=attrTag, element=eltTag, value=attrValue,
messageCodes=("EFM.6.05.34.externalReference", "EFM.5.02.05.externalReference"))
if attrTag == "src" and attrValue not in checkedGraphicsFiles:
if scheme(attrValue) == "data":
modelXbrl.error(messageCodePrefix + "graphicDataUrl",
_("%(validatedObjectLabel)s references a graphics data URL which isn't accepted '%(attribute)s' for <%(element)s>"),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
attribute=attrValue[:32], element=eltTag)
elif attrValue.lower()[-4:] not in ('.jpg', '.gif'):
modelXbrl.error(messageCodePrefix + "graphicFileType",
_("%(validatedObjectLabel)s references a graphics file which isn't .gif or .jpg '%(attribute)s' for <%(element)s>"),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
attribute=attrValue, element=eltTag,
messageCodes=("EFM.6.05.34.graphicFileType", "EFM.5.02.05.graphicFileType"))
else: # test file contents
try:
if validateGraphicFile(referenceElt, attrValue) != attrValue.lower()[-3:]:
modelXbrl.error(messageCodePrefix +"graphicFileContent",
_("%(validatedObjectLabel)s references a graphics file which doesn't have expected content '%(attribute)s' for <%(element)s>"),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
attribute=attrValue, element=eltTag,
messageCodes=("EFM.6.05.34.graphicFileContent", "EFM.5.02.05.graphicFileContent"))
except IOError as err:
modelXbrl.error(messageCodePrefix + "graphicFileError",
_("%(validatedObjectLabel)s references a graphics file which isn't openable '%(attribute)s' for <%(element)s>, error: %(error)s"),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
attribute=attrValue, element=eltTag, error=err,
messageCodes=("EFM.6.05.34.graphicFileError", "EFM.5.02.05.graphicFileError"))
checkedGraphicsFiles.add(attrValue)
if eltTag == "meta" and attrTag == "content" and not attrValue.startswith("text/html"):
modelXbrl.error(messageCodePrefix + "disallowedMetaContent",
_("%(validatedObjectLabel)s <meta> content is \"%(metaContent)s\" but must be \"text/html\""),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
metaContent=attrValue,
messageCodes=("EFM.6.05.34.disallowedMetaContent", "EFM.5.02.05.disallowedMetaContent"))
if eltTag == "table" and any(a.tag in _tableTags
for a in elt.iterancestors()):
modelXbrl.error(messageCodePrefix + "nestedTable",
_("%(validatedObjectLabel)s has nested <table> elements."),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
messageCodes=("EFM.6.05.34.nestedTable", "EFM.5.02.05.nestedTable"))
def validateTextBlockFacts(modelXbrl):
#handler = TextBlockHandler(modelXbrl)
loadDTD(modelXbrl)
checkedGraphicsFiles = set() # only check any graphics file reference once per fact
allowedExternalHrefPattern = modelXbrl.modelManager.disclosureSystem.allowedExternalHrefPattern
if isInlineDTD:
htmlBodyTemplate = "<body><div>\n{0}\n</div></body>\n"
else:
htmlBodyTemplate = "<body>\n{0}\n</body>\n"
_xhtmlNs = "{{{}}}".format(xhtml)
_xhtmlNsLen = len(_xhtmlNs)
for f1 in modelXbrl.facts:
# build keys table for 6.5.14
concept = f1.concept
if f1.xsiNil != "true" and \
concept is not None and \
concept.isTextBlock and \
XMLpattern.match(f1.value):
#handler.fact = f1
# test encoded entity tags
for match in namedEntityPattern.finditer(f1.value):
entity = match.group()
if not entity in xhtmlEntities:
modelXbrl.error(("EFM.6.05.16", "GFM.1.2.15"),
_("Fact %(fact)s contextID %(contextID)s has disallowed entity %(entity)s"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID, entity=entity, error=entity)
# test html
for xmltext in [f1.value] + CDATApattern.findall(f1.value):
'''
try:
xml.sax.parseString(
"<?xml version='1.0' encoding='utf-8' ?>\n<body>\n{0}\n</body>\n".format(
removeEntities(xmltext)).encode('utf-8'),handler,handler)
except (xml.sax.SAXParseException,
xml.sax.SAXException,
UnicodeDecodeError) as err:
# ignore errors which are not errors (e.g., entity codes checked previously
if not err.endswith("undefined entity"):
handler.modelXbrl.error(("EFM.6.05.15", "GFM.1.02.14"),
_("Fact %(fact)s contextID %(contextID)s has text which causes the XML error %(error)s"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID, error=err)
'''
xmlBodyWithoutEntities = htmlBodyTemplate.format(removeEntities(xmltext))
try:
textblockXml = XML(xmlBodyWithoutEntities)
if not edbodyDTD.validate( textblockXml ):
errors = edbodyDTD.error_log.filter_from_errors()
htmlError = any(e.type_name in ("DTD_INVALID_CHILD", "DTD_UNKNOWN_ATTRIBUTE")
for e in errors)
modelXbrl.error("EFM.6.05.16" if htmlError else ("EFM.6.05.15.dtdError", "GFM.1.02.14"),
_("Fact %(fact)s contextID %(contextID)s has text which causes the XML error %(error)s"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID,
error=', '.join(e.message for e in errors),
messageCodes=("EFM.6.05.16", "EFM.6.05.15.dtdError", "GFM.1.02.14"))
for elt in textblockXml.iter():
eltTag = elt.tag
if isinstance(elt, ModelObject) and elt.namespaceURI == xhtml:
eltTag = elt.localName
elif isinstance(elt, (_ElementTree, _Comment, _ProcessingInstruction)):
continue # comment or other non-parsed element
else:
eltTag = elt.tag
if eltTag.startswith(_xhtmlNs):
eltTag = eltTag[_xhtmlNsLen:]
if isInlineDTD and eltTag in efmBlockedInlineHtmlElements:
modelXbrl.error("EFM.5.02.05.disallowedElement",
_("%(validatedObjectLabel)s has disallowed element <%(element)s>"),
modelObject=elt, validatedObjectLabel=f1.qname,
element=eltTag)
for attrTag, attrValue in elt.items():
if isInlineDTD:
if attrTag in efmBlockedInlineHtmlElementAttributes.get(eltTag,()):
modelXbrl.error("EFM.5.02.05.disallowedAttribute",
_("%(validatedObjectLabel)s has disallowed attribute on element <%(element)s>: %(attribute)s=\"%(value)s\""),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
element=eltTag, attribute=attrTag, value=attrValue)
if ((attrTag == "href" and eltTag == "a") or
(attrTag == "src" and eltTag == "img")):
if "javascript:" in attrValue:
modelXbrl.error("EFM.6.05.16.activeContent",
_("Fact %(fact)s of context %(contextID)s has javascript in '%(attribute)s' for <%(element)s>"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID,
attribute=attrTag, element=eltTag)
elif eltTag == "a" and (not allowedExternalHrefPattern or allowedExternalHrefPattern.match(attrValue)):
pass
elif scheme(attrValue) in ("http", "https", "ftp"):
modelXbrl.error("EFM.6.05.16.externalReference",
_("Fact %(fact)s of context %(contextID)s has an invalid external reference in '%(attribute)s' for <%(element)s>"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID,
attribute=attrTag, element=eltTag)
if attrTag == "src" and attrValue not in checkedGraphicsFiles:
if scheme(attrValue) == "data":
modelXbrl.error("EFM.6.05.16.graphicDataUrl",
_("Fact %(fact)s of context %(contextID)s references a graphics data URL which isn't accepted '%(attribute)s' for <%(element)s>"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID,
attribute=attrValue[:32], element=eltTag)
elif attrValue.lower()[-4:] not in ('.jpg', '.gif'):
modelXbrl.error("EFM.6.05.16.graphicFileType",
_("Fact %(fact)s of context %(contextID)s references a graphics file which isn't .gif or .jpg '%(attribute)s' for <%(element)s>"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID,
attribute=attrValue, element=eltTag)
else: # test file contents
try:
if validateGraphicFile(f1, attrValue) != attrValue.lower()[-3:]:
modelXbrl.error("EFM.6.05.16.graphicFileContent",
_("Fact %(fact)s of context %(contextID)s references a graphics file which doesn't have expected content '%(attribute)s' for <%(element)s>"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID,
attribute=attrValue, element=eltTag)
except IOError as err:
modelXbrl.error("EFM.6.05.16.graphicFileError",
_("Fact %(fact)s of context %(contextID)s references a graphics file which isn't openable '%(attribute)s' for <%(element)s>, error: %(error)s"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID,
attribute=attrValue, element=eltTag, error=err)
checkedGraphicsFiles.add(attrValue)
if eltTag == "table" and any(a is not None for a in elt.iterancestors("table")):
modelXbrl.error("EFM.6.05.16.nestedTable",
_("Fact %(fact)s of context %(contextID)s has nested <table> elements."),
modelObject=f1, fact=f1.qname, contextID=f1.contextID)
except (XMLSyntaxError,
UnicodeDecodeError) as err:
#if not err.endswith("undefined entity"):
modelXbrl.error(("EFM.6.05.15", "GFM.1.02.14"),
_("Fact %(fact)s contextID %(contextID)s has text which causes the XML error %(error)s"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID, error=err)
checkedGraphicsFiles.clear()
def checkImageContents(modelXbrl, imgElt, imgType, isFile, data):
if "svg" in imgType:
try:
rootElement = True
for elt in XML(data).iter():
if rootElement:
if elt.tag != "{http://www.w3.org/2000/svg}svg":
modelXbrl.error(
"ESEF.2.5.1.imageFileCannotBeLoaded",
_("Image SVG has root element which is not svg"),
modelObject=imgElt)
rootElement = False
eltTag = elt.tag.rpartition("}")[2] # strip namespace
if ((eltTag in ("object", "script"))
or (eltTag in ("audio", "foreignObject", "iframe",
"image", "use", "video"))):
href = elt.get("href", "")
if eltTag in ("object", "script") or "javascript:" in href:
modelXbrl.error(
"ESEF.2.5.1.executableCodePresent",
_("Inline XBRL images MUST NOT contain executable code: %(element)s"
),
modelObject=imgElt,
element=eltTag)
elif scheme(href) in ("http", "https", "ftp"):
modelXbrl.error(
"ESEF.2.5.1.referencesPointingOutsideOfTheReportingPackagePresent",
_("Inline XBRL instance document [image] MUST NOT contain any reference pointing to resources outside the reporting package: %(element)s"
),
modelObject=imgElt,
element=eltTag)
except (XMLSyntaxError, UnicodeDecodeError) as err:
modelXbrl.error("ESEF.2.5.1.imageFileCannotBeLoaded",
_("Image SVG has XML error %(error)s"),
modelObject=imgElt,
error=err)
elif not any(t in imgType for t in ("gif", "jpg", "jpeg", "png")):
modelXbrl.error(
"ESEF.2.5.1.imageFormatNotSupported",
_("Images included in the XHTML document MUST be saved in PNG, GIF, SVG or JPG/JPEG formats: %(imgType)s is not supported"
),
modelObject=imgElt,
imgType=imgType)
else:
if data[:3] == b"GIF" and data[3:6] in (b'89a', b'89b', b'87a'):
headerType = "gif"
elif ((data[:4] == b'\xff\xd8\xff\xe0' and data[6:11] == b'JFIF\x00')
or
(data[:4] == b'\xff\xd8\xff\xe1' and data[6:11] == b'Exif\x00')):
headerType = "jpg"
elif data[:8] == b"\x89PNG\r\n\x1a\n":
headerType = "png"
elif data[:2] in (b"MM", b"II"):
headerType = "tiff"
elif data[:2] in (b"BM", b"BA"):
headerType = "bmp"
elif data[:4] == b"\x00\x00\x01\x00":
headerType = "ico"
elif data[:4] == b"\x00\x00\x02\x00":
headerType = "cur"
elif len(data) == 0:
headerType = "none"
else:
headerType = "unrecognized"
if (("gif" in imgType and headerType != "gif") or
(("jpg" in imgType or "jpeg" in imgType) and headerType != "jpg")
or ("png" in imgType and headerType != "png")):
modelXbrl.error(
"ESEF.2.5.1.imageDoesNotMatchItsFileExtension"
if isFile else "ESEF.2.5.1.incorrectMIMETypeSpecified",
_("Image type %(imgType)s has wrong header type: %(headerType)s"
),
modelObject=imgElt,
imgType=imgType,
headerType=headerType,
messageCodes=("ESEF.2.5.1.imageDoesNotMatchItsFileExtension",
"ESEF.2.5.1.incorrectMIMETypeSpecified"))
def validateXbrlFinally(val, *args, **kwargs):
if not (val.validateEFMHTMplugin):
return
modelXbrl = val.modelXbrl
allowedExternalHrefPattern = modelXbrl.modelManager.disclosureSystem.allowedExternalHrefPattern
#efmHtmDTD = None
#with open(os.path.join(os.path.dirname(__file__), "resources", "efm-htm.dtd")) as fh:
# efmHtmDTD = DTD(fh)
#if efmHtmDTD and not efmHtmDTD.validate( modelXbrl.modelDocument.xmlRootElement ):
# for e in efmHtmDTD.error_log.filter_from_errors():
# if "declared in the external subset contains white spaces nodes" not in e.message:
# modelXbrl.error("html.syntax",
# _("HTML error %(error)s"),
# error=e.message)
numHtmlTags = 0
inBody = False
for elt in modelXbrl.modelDocument.xmlRootElement.iter():
if isinstance(elt, (_ElementTree, _Comment, _ProcessingInstruction)):
continue # comment or other non-parsed element
eltTag = elt.tag.lower()
for attrTag, attrValue in elt.items():
if ((attrTag == "href" and eltTag == "a")
or (attrTag == "src" and eltTag == "img")):
if "javascript:" in attrValue:
modelXbrl.error(
"EFM.5.02.02.10.activeContent",
_("Element has javascript in '%(attribute)s' for <%(element)s>"
),
modelObject=elt,
attribute=attrTag,
element=eltTag)
elif eltTag == "a" and (
not allowedExternalHrefPattern
or allowedExternalHrefPattern.match(attrValue)):
pass
elif scheme(attrValue) in ("http", "https", "ftp", "mailto"):
modelXbrl.error(
"EFM.6.05.16.externalReference",
_("Element has an invalid external reference in '%(attribute)s' for <%(element)s>"
),
modelObject=elt,
attribute=attrTag,
element=eltTag)
if attrTag == "src":
if scheme(attrValue) == "data":
modelXbrl.error(
"EFM.5.02.02.10.graphicDataUrl",
_("Element references a graphics data URL which isn't accepted '%(attribute)s' for <%(element)s>"
),
modelObject=elt,
attribute=attrValue[:32],
element=eltTag)
elif attrValue.lower()[-4:] not in ('.jpg', '.gif'):
modelXbrl.error(
"EFM.5.02.02.10.graphicFileType",
_("Element references a graphics file which isn't .gif or .jpg '%(attribute)s' for <%(element)s>"
),
modelObject=elt,
attribute=attrValue,
element=eltTag)
elif attrTag in disallowedElementAttrs.get(
eltTag, ()) or attrTag in disallowedElementAttrs["*"]:
modelXbrl.error(
"EFM.5.02.02.05.disallowedAttribute",
_("Element has disallowed attribute '%(attribute)s' for <%(element)s>"
),
modelObject=elt,
attribute=attrTag,
element=eltTag)
elif attrTag.startswith("xmlns"):
modelXbrl.error(
"EFM.5.02.02.05.xmlns",
_("Element has disallowed xmlns declaration '%(attribute)s' for <%(element)s>"
),
modelObject=elt,
attribute=attrTag,
element=eltTag)
if eltTag == "html":
numHtmlTags += 1
if numHtmlTags > 1:
modelXbrl.error("EFM.5.02.02.02.htmlTags",
_("Document can only have one html tag"),
modelObject=elt)
elif eltTag in ("head", "meta", "isindex", "title"):
pass # these are allowed
elif eltTag == "body":
inBody = True
elif eltTag == "table":
if any(a is not None for a in elt.iterancestors("table")):
modelXbrl.error("EFM.5.02.02.10.nestedTable",
_("Element is a disallowed nested <table>."),
modelObject=elt)
elif eltTag in disallowedElements:
modelXbrl.error("EFM.5.02.02.04.disallowedElement",
_("Element is disallowed: <%(element)s>"),
modelObject=elt,
element=eltTag)
elif eltTag not in recognizedElements:
modelXbrl.error("EFM.5.02.02.03.unrecognizedElement",
_("Element is not recognized: <%(element)s>"),
modelObject=elt,
element=eltTag)
elif not inBody:
modelXbrl.error("EFM.5.02.02.03.bodyTags",
_("Element is not in a body: <%(element)s>"),
modelObject=elt,
element=eltTag)
def validateXbrlFinally(val, *args, **kwargs):
if not (val.validateESEFplugin):
return
_xhtmlNs = "{{{}}}".format(xhtml)
_xhtmlNsLen = len(_xhtmlNs)
modelXbrl = val.modelXbrl
modelDocument = modelXbrl.modelDocument
_statusMsg = _("validating {0} filing rules").format(
val.disclosureSystem.name)
modelXbrl.profileActivity()
modelXbrl.modelManager.showStatus(_statusMsg)
reportXmlLang = None
firstRootmostXmlLangDepth = 9999999
_ifrsNs = None
for targetNs in modelXbrl.namespaceDocs.keys():
if ifrsNsPattern.match(targetNs):
_ifrsNs = targetNs
if not _ifrsNs:
modelXbrl.error("ESEF.RTS.ifrsRequired",
_("RTS on ESEF requires IFRS taxonomy."),
modelObject=modelXbrl)
return
esefPrimaryStatementPlaceholders = set(
qname(_ifrsNs, n) for n in esefPrimaryStatementPlaceholderNames)
esefMandatoryElements2020 = set(
qname(_ifrsNs, n) for n in esefMandatoryElementNames2020)
if modelDocument.type == ModelDocument.Type.INSTANCE:
modelXbrl.error("ESEF.I.1.instanceShallBeInlineXBRL",
_("RTS on ESEF requires inline XBRL instances."),
modelObject=modelXbrl)
checkFilingDimensions(
val) # sets up val.primaryItems and val.domainMembers
val.hasExtensionSchema = val.hasExtensionPre = val.hasExtensionCal = val.hasExtensionDef = val.hasExtensionLbl = False
checkFilingDTS(val, modelXbrl.modelDocument, [])
modelXbrl.profileActivity("... filer DTS checks", minTimeToShow=1.0)
if not (val.hasExtensionSchema and val.hasExtensionPre
and val.hasExtensionCal and val.hasExtensionDef
and val.hasExtensionLbl):
missingFiles = []
if not val.hasExtensionSchema: missingFiles.append("schema file")
if not val.hasExtensionPre:
missingFiles.append("presentation linkbase")
if not val.hasExtensionCal: missingFiles.append("calculation linkbase")
if not val.hasExtensionDef: missingFiles.append("definition linkbase")
if not val.hasExtensionLbl: missingFiles.append("label linkbase")
modelXbrl.warning(
"ESEF.3.1.1.extensionTaxonomyWrongFilesStructure",
_("Extension taxonomies MUST consist of at least a schema file and presentation, calculation, definition and label linkbases"
": missing %(missingFiles)s"),
modelObject=modelXbrl,
missingFiles=", ".join(missingFiles))
#if modelDocument.type == ModelDocument.Type.INLINEXBRLDOCUMENTSET:
# # reports only under reports, none elsewhere
# modelXbrl.fileSource.dir
if modelDocument.type in (ModelDocument.Type.INLINEXBRL,
ModelDocument.Type.INLINEXBRLDOCUMENTSET,
ModelDocument.Type.INSTANCE):
footnotesRelationshipSet = modelXbrl.relationshipSet("XBRL-footnotes")
orphanedFootnotes = set()
noLangFootnotes = set()
factLangFootnotes = defaultdict(set)
footnoteRoleErrors = set()
transformRegistryErrors = set()
def checkFootnote(elt, text):
if text: # non-empty footnote must be linked to a fact if not empty
if not any(
isinstance(rel.fromModelObject, ModelFact) for rel in
footnotesRelationshipSet.toModelObject(elt)):
orphanedFootnotes.add(elt)
lang = elt.xmlLang
if not lang:
noLangFootnotes.add(elt)
else:
for rel in footnotesRelationshipSet.toModelObject(elt):
if rel.fromModelObject is not None:
factLangFootnotes[rel.fromModelObject].add(lang)
if elt.role != XbrlConst.footnote or not all(
rel.arcrole == XbrlConst.factFootnote
and rel.linkrole == XbrlConst.defaultLinkRole
for rel in footnotesRelationshipSet.toModelObject(elt)):
footnoteRoleErrors.add(elt)
# check file name of each inline document (which might be below a top-level IXDS)
for doc in modelXbrl.urlDocs.values():
if doc.type == ModelDocument.Type.INLINEXBRL:
_baseName, _baseExt = os.path.splitext(doc.basename)
if _baseExt not in (".xhtml", ".html"):
modelXbrl.warning(
"ESEF.RTS.Art.3.fileNameExtension",
_("FileName SHOULD have the extension .xhtml or .html: %(fileName)s"
),
modelObject=doc,
fileName=doc.basename)
docinfo = doc.xmlRootElement.getroottree().docinfo
if " html" in docinfo.doctype:
modelXbrl.warning(
"ESEF.RTS.Art.3.htmlDoctype",
_("Doctype SHOULD NOT be html: %(fileName)s"),
modelObject=doc,
fileName=doc.basename)
if modelDocument.type in (ModelDocument.Type.INLINEXBRL,
ModelDocument.Type.INLINEXBRLDOCUMENTSET):
hiddenEltIds = {}
presentedHiddenEltIds = defaultdict(list)
eligibleForTransformHiddenFacts = []
requiredToDisplayFacts = []
requiredToDisplayFactIds = {}
firstIxdsDoc = True
for ixdsHtmlRootElt in modelXbrl.ixdsHtmlElements: # ix root elements for all ix docs in IXDS
ixNStag = ixdsHtmlRootElt.modelDocument.ixNStag
ixTags = set(ixNStag + ln
for ln in ("nonNumeric", "nonFraction",
"references", "relationship"))
ixTextTags = set(ixNStag + ln
for ln in ("nonFraction", "continuation",
"footnote"))
ixExcludeTag = ixNStag + "exclude"
ixTupleTag = ixNStag + "tuple"
ixFractionTag = ixNStag + "fraction"
for elt, depth in etreeIterWithDepth(ixdsHtmlRootElt):
eltTag = elt.tag
if isinstance(
elt,
(_ElementTree, _Comment, _ProcessingInstruction)):
continue # comment or other non-parsed element
else:
eltTag = elt.tag
if eltTag.startswith(_xhtmlNs):
eltTag = eltTag[_xhtmlNsLen:]
if firstIxdsDoc and (not reportXmlLang or depth <
firstRootmostXmlLangDepth):
xmlLang = elt.get(
"{http://www.w3.org/XML/1998/namespace}lang"
)
if xmlLang:
reportXmlLang = xmlLang
firstRootmostXmlLangDepth = depth
if ((eltTag in ("object", "script"))
or (eltTag == "a"
and "javascript:" in elt.get("href", ""))
or (eltTag == "img"
and "javascript:" in elt.get("src", ""))):
modelXbrl.error(
"ESEF.2.5.1.executableCodePresent",
_("Inline XBRL documents MUST NOT contain executable code: %(element)s"
),
modelObject=elt,
element=eltTag)
elif eltTag == "img":
src = elt.get("src", "").strip()
hasParentIxTextTag = False # check if image is in an ix text-bearing element
_ancestorElt = elt
while (_ancestorElt is not None):
if _ancestorElt.tag == ixExcludeTag: # excluded from any parent text-bearing ix element
break
if _ancestorElt.tag in ixTextTags:
hasParentIxTextTag = True
break
_ancestorElt = _ancestorElt.getparent()
if scheme(src) in ("http", "https", "ftp"):
modelXbrl.error(
"ESEF.3.5.1.inlinXbrlContainsExternalReferences",
_("Inline XBRL instance documents MUST NOT contain any reference pointing to resources outside the reporting package: %(element)s"
),
modelObject=elt,
element=eltTag)
elif not src.startswith("data:image"):
if hasParentIxTextTag:
modelXbrl.error(
"ESEF.2.5.1.imageInIXbrlElementNotEmbedded",
_("Images appearing within an inline XBRL element MUST be embedded regardless of their size."
),
modelObject=elt)
else:
# presume it to be an image file, check image contents
try:
base = elt.modelDocument.baseForElement(
elt)
normalizedUri = elt.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
src, base)
if not elt.modelXbrl.fileSource.isInArchive(
normalizedUri):
normalizedUri = elt.modelXbrl.modelManager.cntlr.webCache.getfilename(
normalizedUri)
imglen = 0
with elt.modelXbrl.fileSource.file(
normalizedUri,
binary=True)[0] as fh:
imglen += len(fh.read())
if imglen < browserMaxBase64ImageLength:
modelXbrl.error(
"ESEF.2.5.1.embeddedImageNotUsingBase64Encoding",
_("Images MUST be included in the XHTML document as a base64 encoded string unless their size exceeds support of browsers (%(maxImageSize)s): %(file)s."
),
modelObject=elt,
maxImageSize=
browserMaxBase64ImageLength,
file=os.path.basename(
normalizedUri))
except IOError as err:
modelXbrl.error(
"ESEF.2.5.1.imageFileCannotBeLoaded",
_("Image file which isn't openable '%(src)s', error: %(error)s"
),
modelObject=elt,
src=src,
error=err)
elif not any(
src.startswith(m)
for m in allowedImgMimeTypes):
modelXbrl.error(
"ESEF.2.5.1.embeddedImageNotUsingBase64Encoding",
_("Images MUST be included in the XHTML document as a base64 encoded string, encoding disallowed: %(src)s."
),
modelObject=elt,
src=attrValue[:128])
elif eltTag == "a":
href = elt.get("href", "").strip()
if scheme(href) in ("http", "https", "ftp"):
modelXbrl.error(
"ESEF.3.5.1.inlinXbrlContainsExternalReferences",
_("Inline XBRL instance documents MUST NOT contain any reference pointing to resources outside the reporting package: %(element)s"
),
modelObject=elt,
element=eltTag)
elif eltTag == "base" or elt.tag == "{http://www.w3.org/XML/1998/namespace}base":
modelXbrl.error(
"ESEF.2.4.2.htmlOrXmlBaseUsed",
_("The HTML <base> elements and xml:base attributes MUST NOT be used in the Inline XBRL document."
),
modelObject=elt,
element=eltTag)
elif eltTag == "link" and elt.get(
"type") == "text/css":
if len(modelXbrl.ixdsHtmlElements) > 1:
f = elt.get("href")
if not f or isHttpUrl(f) or os.path.isabs(f):
modelXbrl.warning(
"ESEF.2.5.4.externalCssReportPackage",
_("The CSS file should be physically stored within the report package: %{file}s."
),
modelObject=elt,
file=f)
else:
modelXbrl.error(
"ESEF.2.5.4.externalCssFileForSingleIXbrlDocument",
_("Where an Inline XBRL document set contains a single document, the CSS MUST be embedded within the document."
),
modelObject=elt,
element=eltTag)
elif eltTag == "style" and elt.get(
"type") == "text/css":
if len(modelXbrl.ixdsHtmlElements) > 1:
modelXbrl.warning(
"ESEF.2.5.4.embeddedCssForMultiHtmlIXbrlDocumentSets",
_("Where an Inline XBRL document set contains multiple documents, the CSS SHOULD be defined in a separate file."
),
modelObject=elt,
element=eltTag)
if eltTag in ixTags and elt.get("target"):
modelXbrl.error(
"ESEF.2.5.3.targetAttributeUsed",
_("Target attribute MUST not be used: element %(localName)s, target attribute %(target)s."
),
modelObject=elt,
localName=elt.elementQname,
target=elt.get("target"))
if eltTag == ixTupleTag:
modelXbrl.error(
"ESEF.2.4.1.tupleElementUsed",
_("The ix:tuple element MUST not be used in the Inline XBRL document: %(qname)s."
),
modelObject=elt,
qname=elt.qname)
if eltTag == ixFractionTag:
modelXbrl.error(
"ESEF.2.4.1.fractionElementUsed",
_("The ix:fraction element MUST not be used in the Inline XBRL document."
),
modelObject=elt)
if elt.get("{http://www.w3.org/XML/1998/namespace}base"
) is not None:
modelXbrl.error(
"ESEF.2.4.1.xmlBaseUsed",
_("xml:base attributes MUST NOT be used in the Inline XBRL document: element %(localName)s, base attribute %(base)s."
),
modelObject=elt,
localName=elt.elementQname,
base=elt.get(
"{http://www.w3.org/XML/1998/namespace}base"))
if isinstance(elt, ModelInlineFootnote):
checkFootnote(elt, elt.value)
elif isinstance(
elt, ModelResource
) and elt.qname == XbrlConst.qnLinkFootnote:
checkFootnote(elt, elt.value)
elif isinstance(elt, ModelInlineFact):
if elt.format is not None and elt.format.namespaceURI not in IXT_NAMESPACES:
transformRegistryErrors.add(elt)
for ixHiddenElt in ixdsHtmlRootElt.iterdescendants(
tag=ixNStag + "hidden"):
for tag in (ixNStag + "nonNumeric",
ixNStag + "nonFraction"):
for ixElt in ixHiddenElt.iterdescendants(tag=tag):
if (
getattr(ixElt, "xValid",
0) >= VALID # may not be validated
): # add future "and" conditions on elements which can be in hidden
if (ixElt.concept.baseXsdType
not in untransformableTypes
and not ixElt.isNil):
eligibleForTransformHiddenFacts.append(
ixElt)
elif ixElt.id is None:
requiredToDisplayFacts.append(ixElt)
if ixElt.id:
hiddenEltIds[ixElt.id] = ixElt
firstIxdsDoc = False
if eligibleForTransformHiddenFacts:
modelXbrl.warning(
"ESEF.2.4.1.transformableElementIncludedInHiddenSection",
_("The ix:hidden section of Inline XBRL document MUST not include elements eligible for transformation. "
"%(countEligible)s fact(s) were eligible for transformation: %(elements)s"
),
modelObject=eligibleForTransformHiddenFacts,
countEligible=len(eligibleForTransformHiddenFacts),
elements=", ".join(
sorted(
set(
str(f.qname)
for f in eligibleForTransformHiddenFacts))))
for ixdsHtmlRootElt in modelXbrl.ixdsHtmlElements:
for ixElt in ixdsHtmlRootElt.getroottree().iterfind(
"//{http://www.w3.org/1999/xhtml}*[@style]"):
hiddenFactRefMatch = styleIxHiddenPattern.match(
ixElt.get("style", ""))
if hiddenFactRefMatch:
hiddenFactRef = hiddenFactRefMatch.group(2)
if hiddenFactRef not in hiddenEltIds:
modelXbrl.error(
"ESEF.2.4.1.esefIxHiddenStyleNotLinkingFactInHiddenSection",
_("\"-esef-ix-hidden\" style identifies @id, %(id)s of a fact that is not in ix:hidden section."
),
modelObject=ixElt,
id=hiddenFactRef)
else:
presentedHiddenEltIds[hiddenFactRef].append(ixElt)
for hiddenEltId, ixElt in hiddenEltIds.items():
if (hiddenEltId not in presentedHiddenEltIds
and getattr(ixElt, "xValid", 0) >= VALID
and # may not be validated
(ixElt.concept.baseXsdType in untransformableTypes
or ixElt.isNil)):
requiredToDisplayFacts.append(ixElt)
if requiredToDisplayFacts:
modelXbrl.warning(
"ESEF.2.4.1.factInHiddenSectionNotInReport",
_("The ix:hidden section contains %(countUnreferenced)s fact(s) whose @id is not applied on any \"-esef-ix- hidden\" style: %(elements)s"
),
modelObject=requiredToDisplayFacts,
countUnreferenced=len(requiredToDisplayFacts),
elements=", ".join(
sorted(
set(str(f.qname)
for f in requiredToDisplayFacts))))
del eligibleForTransformHiddenFacts, hiddenEltIds, presentedHiddenEltIds, requiredToDisplayFacts
elif modelDocument.type == ModelDocument.Type.INSTANCE:
for elt in modelDocument.xmlRootElement.iter():
if elt.qname == XbrlConst.qnLinkFootnote: # for now assume no private elements extend link:footnote
checkFootnote(elt, elt.stringValue)
contextsWithDisallowedOCEs = []
contextsWithDisallowedOCEcontent = []
contextsWithPeriodTime = []
contextsWithPeriodTimeZone = []
contextIdentifiers = defaultdict(list)
nonStandardTypedDimensions = defaultdict(set)
for context in modelXbrl.contexts.values():
for elt in context.iterdescendants(
"{http://www.xbrl.org/2003/instance}startDate",
"{http://www.xbrl.org/2003/instance}endDate",
"{http://www.xbrl.org/2003/instance}instant"):
m = datetimePattern.match(elt.stringValue)
if m:
if m.group(1):
contextsWithPeriodTime.append(context)
if m.group(3):
contextsWithPeriodTimeZone.append(context)
for elt in context.iterdescendants(
"{http://www.xbrl.org/2003/instance}segment"):
contextsWithDisallowedOCEs.append(context)
break
for elt in context.iterdescendants(
"{http://www.xbrl.org/2003/instance}scenario"):
if isinstance(elt, ModelObject):
if any(True for child in elt.iterchildren()
if isinstance(child, ModelObject)
and child.tag not in (
"{http://xbrl.org/2006/xbrldi}explicitMember",
"{http://xbrl.org/2006/xbrldi}typedMember")):
contextsWithDisallowedOCEcontent.append(context)
# check periods here
contextIdentifiers[context.entityIdentifier].append(context)
if contextsWithDisallowedOCEs:
modelXbrl.error(
"ESEF.2.1.3.segmentUsed",
_("xbrli:segment container MUST NOT be used in contexts: %(contextIds)s"
),
modelObject=contextsWithDisallowedOCEs,
contextIds=", ".join(c.id for c in contextsWithDisallowedOCEs))
if contextsWithDisallowedOCEcontent:
modelXbrl.error(
"ESEF.2.1.3.scenarioContainsNonDimensionalContent",
_("xbrli:scenario in contexts MUST NOT contain any other content than defined in XBRL Dimensions specification: %(contextIds)s"
),
modelObject=contextsWithDisallowedOCEcontent,
contextIds=", ".join(
c.id for c in contextsWithDisallowedOCEcontent))
if len(contextIdentifiers) > 1:
modelXbrl.error(
"ESEF.2.1.4.multipleIdentifiers",
_("All entity identifiers in contexts MUST have identical content: %(contextIdentifiers)s"
),
modelObject=modelXbrl,
contextIds=", ".join(i[1] for i in contextIdentifiers))
for (contextScheme,
contextIdentifier), contextElts in contextIdentifiers.items():
if contextScheme != iso17442:
modelXbrl.warning(
"ESEF.2.1.1.nonLEIContextScheme",
_("The scheme attribute of the xbrli:identifier element should have \"%(leiScheme)s\" as its content: %(contextScheme)s"
),
modelObject=contextElts,
contextScheme=contextScheme,
leiScheme=iso17442)
else:
leiValidity = LeiUtil.checkLei(contextIdentifier)
if leiValidity == LeiUtil.LEI_INVALID_LEXICAL:
modelXbrl.warning(
"ESEF.2.1.1.invalidIdentifierFormat",
_("The LEI context identifier has an invalid format: %(identifier)s"
),
modelObject=contextElts,
identifier=contextIdentifier)
elif leiValidity == LeiUtil.LEI_INVALID_CHECKSUM:
modelXbrl.warning(
"ESEF.2.1.1.invalidIdentifier",
_("The LEI context identifier has checksum error: %(identifier)s"
),
modelObject=contextElts,
identifier=contextIdentifier)
if contextsWithPeriodTime:
modelXbrl.warning(
"ESEF.2.1.2.periodWithTimeContent",
_("Context period startDate, endDate and instant elements should be in whole days without time: %(contextIds)s"
),
modelObject=contextsWithPeriodTime,
contextIds=", ".join(c.id for c in contextsWithPeriodTime))
if contextsWithPeriodTimeZone:
modelXbrl.warning(
"ESEF.2.1.2.periodWithTimeZone",
_("Context period startDate, endDate and instant elements should be in whole days without a timezone: %(contextIds)s"
),
modelObject=contextsWithPeriodTimeZone,
contextIds=", ".join(c.id for c in contextsWithPeriodTimeZone))
# identify unique contexts and units
mapContext = {}
mapUnit = {}
uniqueContextHashes = {}
for context in modelXbrl.contexts.values():
h = context.contextDimAwareHash
if h in uniqueContextHashes:
if context.isEqualTo(uniqueContextHashes[h]):
mapContext[context] = uniqueContextHashes[h]
else:
uniqueContextHashes[h] = context
del uniqueContextHashes
uniqueUnitHashes = {}
utrValidator = ValidateUtr(modelXbrl)
utrUnitIds = set(
u.unitId
for unitItemType in utrValidator.utrItemTypeEntries.values()
for u in unitItemType.values())
for unit in modelXbrl.units.values():
h = unit.hash
if h in uniqueUnitHashes:
if unit.isEqualTo(uniqueUnitHashes[h]):
mapUnit[unit] = uniqueUnitHashes[h]
else:
uniqueUnitHashes[h] = unit
# check if any custom measure is in UTR
for measureTerm in unit.measures:
for measure in measureTerm:
ns = measure.namespaceURI
if ns != XbrlConst.iso4217 and not ns.startswith(
"http://www.xbrl.org/"):
if measure.localName in utrUnitIds:
modelXbrl.error(
"ESEF.RTS.III.1.G1-7-1.customUnitInUtr",
_("Custom measure SHOULD NOT duplicate a UnitID of UTR: %(measure)s"
),
modelObject=unit,
measure=measure)
del uniqueUnitHashes
reportedMandatory = set()
precisionFacts = set()
numFactsByConceptContextUnit = defaultdict(list)
textFactsByConceptContext = defaultdict(list)
footnotesRelationshipSet = modelXbrl.relationshipSet(
XbrlConst.factFootnote, XbrlConst.defaultLinkRole)
noLangFacts = []
textFactsMissingReportLang = []
conceptsUsed = set()
for qn, facts in modelXbrl.factsByQname.items():
if qn in mandatory:
reportedMandatory.add(qn)
for f in facts:
if f.precision is not None:
precisionFacts.add(f)
if f.isNumeric:
numFactsByConceptContextUnit[(f.qname,
mapContext.get(
f.context, f.context),
mapUnit.get(
f.unit,
f.unit))].append(f)
if f.concept is not None and not f.isNil and f.xValid >= VALID and f.xValue > 1 and f.concept.type is not None and (
f.concept.type.qname == PERCENT_TYPE
or f.concept.type.isDerivedFrom(PERCENT_TYPE)):
modelXbrl.warning(
"ESEF.2.2.2.percentGreaterThan100",
_("A percent fact should have value <= 100: %(element)s in context %(context)s value %(value)s"
),
modelObject=f,
element=f.qname,
context=f.context.id,
value=f.xValue)
elif f.concept is not None and f.concept.type is not None:
if f.concept.type.isOimTextFactType:
if not f.xmlLang:
noLangFacts.append(f)
elif f.context is not None:
textFactsByConceptContext[(
f.qname, mapContext.get(f.context,
f.context))].append(f)
conceptsUsed.add(f.concept)
if f.context is not None:
for dim in f.context.qnameDims.values():
conceptsUsed.add(dim.dimension)
if dim.isExplicit:
conceptsUsed.add(dim.member)
elif dim.isTyped:
conceptsUsed.add(dim.typedMember)
if noLangFacts:
modelXbrl.error(
"ESEF.2.5.2.undefinedLanguageForTextFact",
_("Each tagged text fact MUST have the 'xml:lang' attribute assigned or inherited."
),
modelObject=noLangFacts)
# missing report lang text facts
if reportXmlLang:
for fList in textFactsByConceptContext.values():
if not any(f.xmlLang == reportXmlLang for f in fList):
modelXbrl.error(
"ESEF.2.5.2.taggedTextFactOnlyInLanguagesOtherThanLanguageOfAReport",
_("Each tagged text fact MUST have the 'xml:lang' provided in at least the language of the report: %(element)s"
),
modelObject=fList,
element=fList[0].qname)
# 2.2.4 test
for fList in numFactsByConceptContextUnit.values():
if len(fList) > 1:
f0 = fList[0]
if any(f.isNil for f in fList):
_inConsistent = not all(f.isNil for f in fList)
elif all(
inferredDecimals(f) == inferredDecimals(f0)
for f in fList[1:]): # same decimals
v0 = rangeValue(f0.value)
_inConsistent = not all(
rangeValue(f.value) == v0 for f in fList[1:])
else: # not all have same decimals
aMax, bMin = rangeValue(f0.value, inferredDecimals(f0))
for f in fList[1:]:
a, b = rangeValue(f.value, inferredDecimals(f))
if a > aMax: aMax = a
if b < bMin: bMin = b
_inConsistent = (bMin < aMax)
if _inConsistent:
modelXbrl.error(
("ESEF.2.2.4.inconsistentDuplicateNumericFactInInlineXbrlDocument"
),
"Inconsistent duplicate numeric facts MUST NOT appear in the content of an inline XBRL document. %(fact)s that was used more than once in contexts equivalent to %(contextID)s: values %(values)s. ",
modelObject=fList,
fact=f0.qname,
contextID=f0.contextID,
values=", ".join(
strTruncate(f.value, 128) for f in fList))
if precisionFacts:
modelXbrl.warning(
"ESEF.2.2.1.precisionAttributeUsed",
_("The accuracy of numeric facts SHOULD be defined with the 'decimals' attribute rather than the 'precision' attribute: %(elements)s."
),
modelObject=precisionFacts,
elements=", ".join(sorted(
str(e.qname) for e in precisionFacts)))
missingElements = (mandatory - reportedMandatory)
if missingElements:
modelXbrl.error(
"ESEF.???.missingRequiredElements",
_("Required elements missing from document: %(elements)s."),
modelObject=modelXbrl,
elements=", ".join(sorted(str(qn) for qn in missingElements)))
if transformRegistryErrors:
modelXbrl.warning(
"ESEF.2.2.3.transformRegistry",
_("ESMA recommends applying the latest available version of the Transformation Rules Registry marked with 'Recommendation' status for these elements: %(elements)s."
),
modelObject=transformRegistryErrors,
elements=", ".join(
sorted(
str(fact.qname) for fact in transformRegistryErrors)))
if orphanedFootnotes:
modelXbrl.error(
"ESEF.2.3.1.unusedFootnote",
_("Non-empty footnotes must be connected to fact(s)."),
modelObject=orphanedFootnotes)
if noLangFootnotes:
modelXbrl.error(
"ESEF.2.3.1.undefinedLanguageForFootnote",
_("Each footnote MUST have the 'xml:lang' attribute whose value corresponds to the language of the text in the content of the respective footnote."
),
modelObject=noLangFootnotes)
nonDefLangFtFacts = set(f for f, langs in factLangFootnotes.items()
if reportXmlLang not in langs)
if nonDefLangFtFacts:
modelXbrl.error(
"ESEF.2.3.1.footnoteOnlyInLanguagesOtherThanLanguageOfAReport",
_("Each fact MUST have at least one footnote with 'xml:lang' attribute whose value corresponds to the language of the text in the content of the respective footnote: %(qnames)s."
),
modelObject=nonDefLangFtFacts,
qnames=", ".join(
sorted(str(f.qname) for f in nonDefLangFtFacts)))
del nonDefLangFtFacts
if footnoteRoleErrors:
modelXbrl.error(
"ESEF.2.3.1.nonStandardRoleForFootnote",
_("The xlink:role attribute of a link:footnote and link:footnoteLink element as well as xlink:arcrole attribute of a link:footnoteArc MUST be defined in the XBRL Specification 2.1."
),
modelObject=footnoteRoleErrors)
nonStdFootnoteElts = list()
for modelLink in modelXbrl.baseSets[("XBRL-footnotes", None, None,
None)]:
for elt in modelLink.iterchildren():
if isinstance(
elt, (_ElementTree, _Comment, _ProcessingInstruction)):
continue # comment or other non-parsed element
if elt.qname not in FOOTNOTE_LINK_CHILDREN:
nonStdFootnoteElts.append(elt)
if nonStdFootnoteElts:
modelXbrl.error(
"ESEF.2.3.2.nonStandardElementInFootnote",
_("A link:footnoteLink element MUST have no children other than link:loc, link:footnote, and link:footnoteArc."
),
modelObject=nonStdFootnoteElts)
for qn in modelXbrl.qnameDimensionDefaults.values():
conceptsUsed.add(modelXbrl.qnameConcepts.get(qn))
# unused elements in linkbases
for arcroles, err in (
((parentChild, ),
"elementsNotUsedForTaggingAppliedInPresentationLinkbase"),
((summationItem, ),
"elementsNotUsedForTaggingAppliedInCalculationLinkbase"),
((dimensionDomain, domainMember),
"elementsNotUsedForTaggingAppliedInDefinitionLinkbase")):
unreportedLbElts = set()
for arcrole in arcroles:
for rel in modelXbrl.relationshipSet(
arcrole).modelRelationships:
fr = rel.fromModelObject
to = rel.toModelObject
if arcrole in (parentChild, summationItem):
if fr is not None and not fr.isAbstract and fr not in conceptsUsed and isExtension(
val, rel):
unreportedLbElts.add(fr)
if to is not None and not to.isAbstract and to not in conceptsUsed and isExtension(
val, rel):
unreportedLbElts.add(to)
elif arcrole == dimensionDomain: # dimension, always abstract
if fr is not None and fr not in conceptsUsed and isExtension(
val, rel):
unreportedLbElts.add(fr)
if to is not None and rel.isUsable and to not in conceptsUsed and isExtension(
val, rel):
unreportedLbElts.add(to)
elif arcrole == domainMember:
if to is not None and not fr.isAbstract and rel.isUsable and to not in conceptsUsed and isExtension(
val, rel):
unreportedLbElts.add(to)
if unreportedLbElts:
modelXbrl.error(
"ESEF.3.4.6." + err,
_("All usable concepts in extension taxonomy relationships MUST be applied by tagged facts: %(elements)s."
),
modelObject=unreportedLbElts,
elements=", ".join(
sorted((str(c.qname) for c in unreportedLbElts))))
# 3.4.4 check for presentation preferred labels
missingConceptLabels = defaultdict(set) # by role
pfsConceptsRootInPreLB = set()
def checkLabels(parent, relSet, labelrole, visited):
if not parent.label(
labelrole, lang=reportXmlLang, fallbackToQname=False):
if parent.name != "NotesAccountingPoliciesAndMandatoryTags": # TEMPORARY TBD remove
missingConceptLabels[labelrole].add(parent)
visited.add(parent)
conceptRels = defaultdict(
list) # counts for concepts without preferred label role
for rel in relSet.fromModelObject(parent):
child = rel.toModelObject
if child is not None:
labelrole = rel.preferredLabel
if not labelrole:
conceptRels[child].append(rel)
if child not in visited:
checkLabels(child, relSet, labelrole, visited)
for concept, rels in conceptRels.items():
if len(rels) > 1:
modelXbrl.warning(
"ESEF.3.4.4.missingPreferredLabelRole",
_("Preferred label role SHOULD be used when concept is duplicated in same presentation tree location: %(qname)s."
),
modelObject=rels + [concept],
qname=concept.qname)
visited.remove(parent)
for ELR in modelXbrl.relationshipSet(parentChild).linkRoleUris:
relSet = modelXbrl.relationshipSet(parentChild, ELR)
for rootConcept in relSet.rootConcepts:
checkLabels(rootConcept, relSet, None, set())
# check for PFS element which isn't an orphan
if rootConcept.qname in esefPrimaryStatementPlaceholders and relSet.fromModelObject(
rootConcept):
pfsConceptsRootInPreLB.add(rootConcept)
for labelrole, concepts in missingConceptLabels.items():
modelXbrl.warning(
"ESEF.3.4.5.missingLabelForRoleInReportLanguage",
_("Label for %(role)s role SHOULD be available in report language for concepts: %(qnames)s."
),
modelObject=concepts,
qnames=", ".join(str(c.qname) for c in concepts),
role=os.path.basename(labelrole) if labelrole else "standard")
if not pfsConceptsRootInPreLB:
# no PFS statements were recognized
modelXbrl.error(
"ESEF.RTS.Annex.II.Par.1.Par.7.missingPrimaryFinancialStatement",
_("A primary financial statement placeholder element MUST be a root of a presentation linkbase tree."
),
modelObject=modelXbrl)
# dereference
del missingConceptLabels, pfsConceptsRootInPreLB
# mandatory factc RTS Annex II
missingMandatoryElements = esefMandatoryElements2020 - modelXbrl.factsByQname.keys(
)
if missingMandatoryElements:
modelXbrl.error(
"ESEF.RTS.Annex.II.Par.2.missingMandatoryMarkups",
_("Mandatory elements to be marked up are missing: %(qnames)s."
),
modelObject=missingMandatoryElements,
qnames=", ".join(
sorted(str(qn) for qn in missingMandatoryElements)))
# duplicated core taxonomy elements
for name, concepts in modelXbrl.nameConcepts.items():
if len(concepts) > 1:
i = None # ifrs Concept
for c in concepts:
if c.qname.namespaceURI == _ifrsNs:
i = c
break
if i is not None:
for c in concepts:
if c != i and c.balance == i.balance and c.periodType == i.periodType:
modelXbrl.error(
"ESEF.RTS.Annex.IV.Par.4.2.extensionElementDuplicatesCoreElement",
_("Extension elements must not duplicate the existing elements from the core taxonomy and be identifiable %(qname)s."
),
modelObject=(c, i),
qname=c.qname)
modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0)
modelXbrl.modelManager.showStatus(None)
def validateHtmlContent(modelXbrl, referenceElt, htmlEltTree, validatedObjectLabel, messageCodePrefix, isInline=False):
checkedGraphicsFiles = set() # only check any graphics file reference once per footnote
_xhtmlNs = "{{{}}}".format(xhtml)
_xhtmlNsLen = len(_xhtmlNs)
_tableTags = ("table", _xhtmlNs + "table")
_anchorAncestorTags = set(_xhtmlNs + tag for tag in ("html", "body", "div"))
allowedExternalHrefPattern = modelXbrl.modelManager.disclosureSystem.allowedExternalHrefPattern
for elt in htmlEltTree.iter():
if isinstance(elt, ModelObject) and elt.namespaceURI == xhtml:
eltTag = elt.localName
elif isinstance(elt, (_ElementTree, _Comment, _ProcessingInstruction)):
continue # comment or other non-parsed element
else:
eltTag = elt.tag
if eltTag.startswith(_xhtmlNs):
eltTag = eltTag[_xhtmlNsLen:]
if isInline:
if eltTag in efmBlockedInlineHtmlElements:
modelXbrl.error("EFM.5.02.05.disallowedElement",
_("%(validatedObjectLabel)s has disallowed element <%(element)s>"),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
element=eltTag)
if eltTag == "a" and "href" not in elt.keys() and any(a.tag not in _anchorAncestorTags for a in elt.iterancestors()):
modelXbrl.warning("EFM.5.02.05.anchorElementPosition",
_("If element <a> does not have attribute @href, it should not have any ancestors other than html, body, or div. Disallowed ancestors: %(disallowedAncestors)s"),
modelObject=elt, disallowedAncestors=", ".join(a.tag.rpartition('}')[2] for a in elt.iterancestors() if a.tag not in _anchorAncestorTags))
for attrTag, attrValue in elt.items():
if isInline:
if attrTag in efmBlockedInlineHtmlElementAttributes.get(eltTag,()):
modelXbrl.error("EFM.5.02.05.disallowedAttribute",
_("%(validatedObjectLabel)s has disallowed attribute on element <%(element)s>: %(attribute)s=\"%(value)s\""),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
element=eltTag, attribute=attrTag, value=attrValue)
elif attrTag == "{http://www.w3.org/XML/1998/namespace}base":
modelXbrl.error("EFM.5.02.05.xmlBaseDisallowed",
_("%(validatedObjectLabel)s has disallowed attribute on element <%(element)s>: xml:base=\"%(value)s\""),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
element=eltTag, value=attrValue)
elif attrTag == "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation":
modelXbrl.warning("EFM.5.02.05.schemaLocationDisallowed",
_("%(validatedObjectLabel)s has disallowed attribute on element <%(element)s>: xsi:schemaLocation=\"%(value)s\""),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
element=eltTag, value=attrValue)
if ((attrTag == "href" and eltTag == "a") or
(attrTag == "src" and eltTag == "img")):
if "javascript:" in attrValue:
modelXbrl.error(messageCodePrefix + "activeContent",
_("%(validatedObjectLabel)s has javascript in '%(attribute)s' for <%(element)s>"),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
attribute=attrTag, element=eltTag,
messageCodes=("EFM.6.05.34.activeContent", "EFM.5.02.05.activeContent"))
elif eltTag == "a" and (not allowedExternalHrefPattern or allowedExternalHrefPattern.match(attrValue)):
pass
elif scheme(attrValue) in ("http", "https", "ftp"):
modelXbrl.error(messageCodePrefix + "externalReference",
_("%(validatedObjectLabel)s has an invalid external reference in '%(attribute)s' for <%(element)s>: %(value)s"),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
attribute=attrTag, element=eltTag, value=attrValue,
messageCodes=("EFM.6.05.34.externalReference", "EFM.5.02.05.externalReference"))
if attrTag == "src" and attrValue not in checkedGraphicsFiles:
if scheme(attrValue) == "data":
modelXbrl.error(messageCodePrefix + "graphicDataUrl",
_("%(validatedObjectLabel)s references a graphics data URL which isn't accepted '%(attribute)s' for <%(element)s>"),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
attribute=attrValue[:32], element=eltTag)
elif attrValue.lower()[-4:] not in ('.jpg', '.gif'):
modelXbrl.error(messageCodePrefix + "graphicFileType",
_("%(validatedObjectLabel)s references a graphics file which isn't .gif or .jpg '%(attribute)s' for <%(element)s>"),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
attribute=attrValue, element=eltTag,
messageCodes=("EFM.6.05.34.graphicFileType", "EFM.5.02.05.graphicFileType"))
else: # test file contents
try:
if validateGraphicFile(referenceElt, attrValue) != attrValue.lower()[-3:]:
modelXbrl.error(messageCodePrefix +"graphicFileContent",
_("%(validatedObjectLabel)s references a graphics file which doesn't have expected content '%(attribute)s' for <%(element)s>"),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
attribute=attrValue, element=eltTag,
messageCodes=("EFM.6.05.34.graphicFileContent", "EFM.5.02.05.graphicFileContent"))
except IOError as err:
modelXbrl.error(messageCodePrefix + "graphicFileError",
_("%(validatedObjectLabel)s references a graphics file which isn't openable '%(attribute)s' for <%(element)s>, error: %(error)s"),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
attribute=attrValue, element=eltTag, error=err,
messageCodes=("EFM.6.05.34.graphicFileError", "EFM.5.02.05.graphicFileError"))
checkedGraphicsFiles.add(attrValue)
if eltTag == "meta" and attrTag == "content" and not attrValue.startswith("text/html"):
modelXbrl.error(messageCodePrefix + "disallowedMetaContent",
_("%(validatedObjectLabel)s <meta> content is \"%(metaContent)s\" but must be \"text/html\""),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
metaContent=attrValue,
messageCodes=("EFM.6.05.34.disallowedMetaContent", "EFM.5.02.05.disallowedMetaContent"))
if eltTag == "table" and any(a.tag in _tableTags
for a in elt.iterancestors()):
modelXbrl.error(messageCodePrefix + "nestedTable",
_("%(validatedObjectLabel)s has nested <table> elements."),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
messageCodes=("EFM.6.05.34.nestedTable", "EFM.5.02.05.nestedTable"))
def validateTextBlockFacts(modelXbrl):
#handler = TextBlockHandler(modelXbrl)
loadDTD(modelXbrl)
checkedGraphicsFiles = set() # only check any graphics file reference once per fact
allowedExternalHrefPattern = modelXbrl.modelManager.disclosureSystem.allowedExternalHrefPattern
if isInlineDTD:
htmlBodyTemplate = "<body><div>\n{0}\n</div></body>\n"
else:
htmlBodyTemplate = "<body>\n{0}\n</body>\n"
_xhtmlNs = "{{{}}}".format(xhtml)
_xhtmlNsLen = len(_xhtmlNs)
for f1 in modelXbrl.facts:
# build keys table for 6.5.14
concept = f1.concept
if f1.xsiNil != "true" and \
concept is not None and \
concept.isTextBlock and \
XMLpattern.match(f1.value):
#handler.fact = f1
# test encoded entity tags
for match in namedEntityPattern.finditer(f1.value):
entity = match.group()
if not entity in xhtmlEntities:
modelXbrl.error(("EFM.6.05.16", "GFM.1.2.15"),
_("Fact %(fact)s contextID %(contextID)s has disallowed entity %(entity)s"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID, entity=entity, error=entity)
# test html
for xmltext in [f1.value] + CDATApattern.findall(f1.value):
'''
try:
xml.sax.parseString(
"<?xml version='1.0' encoding='utf-8' ?>\n<body>\n{0}\n</body>\n".format(
removeEntities(xmltext)).encode('utf-8'),handler,handler)
except (xml.sax.SAXParseException,
xml.sax.SAXException,
UnicodeDecodeError) as err:
# ignore errors which are not errors (e.g., entity codes checked previously
if not err.endswith("undefined entity"):
handler.modelXbrl.error(("EFM.6.05.15", "GFM.1.02.14"),
_("Fact %(fact)s contextID %(contextID)s has text which causes the XML error %(error)s"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID, error=err)
'''
xmlBodyWithoutEntities = htmlBodyTemplate.format(removeEntities(xmltext))
try:
textblockXml = XML(xmlBodyWithoutEntities)
if not edbodyDTD.validate( textblockXml ):
errors = edbodyDTD.error_log.filter_from_errors()
htmlError = any(e.type_name in ("DTD_INVALID_CHILD", "DTD_UNKNOWN_ATTRIBUTE")
for e in errors)
modelXbrl.error("EFM.6.05.16" if htmlError else ("EFM.6.05.15.dtdError", "GFM.1.02.14"),
_("Fact %(fact)s contextID %(contextID)s has text which causes the XML error %(error)s"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID,
error=', '.join(e.message for e in errors),
messageCodes=("EFM.6.05.16", "EFM.6.05.15.dtdError", "GFM.1.02.14"))
for elt in textblockXml.iter():
eltTag = elt.tag
if isinstance(elt, ModelObject) and elt.namespaceURI == xhtml:
eltTag = elt.localName
elif isinstance(elt, (_ElementTree, _Comment, _ProcessingInstruction)):
continue # comment or other non-parsed element
else:
eltTag = elt.tag
if eltTag.startswith(_xhtmlNs):
eltTag = eltTag[_xhtmlNsLen:]
if isInlineDTD and eltTag in efmBlockedInlineHtmlElements:
modelXbrl.error("EFM.5.02.05.disallowedElement",
_("%(validatedObjectLabel)s has disallowed element <%(element)s>"),
modelObject=elt, validatedObjectLabel=f1.qname,
element=eltTag)
for attrTag, attrValue in elt.items():
if isInlineDTD:
if attrTag in efmBlockedInlineHtmlElementAttributes.get(eltTag,()):
modelXbrl.error("EFM.5.02.05.disallowedAttribute",
_("%(validatedObjectLabel)s has disallowed attribute on element <%(element)s>: %(attribute)s=\"%(value)s\""),
modelObject=elt, validatedObjectLabel=validatedObjectLabel,
element=eltTag, attribute=attrTag, value=attrValue)
if ((attrTag == "href" and eltTag == "a") or
(attrTag == "src" and eltTag == "img")):
if "javascript:" in attrValue:
modelXbrl.error("EFM.6.05.16.activeContent",
_("Fact %(fact)s of context %(contextID)s has javascript in '%(attribute)s' for <%(element)s>"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID,
attribute=attrTag, element=eltTag)
elif eltTag == "a" and (not allowedExternalHrefPattern or allowedExternalHrefPattern.match(attrValue)):
pass
elif scheme(attrValue) in ("http", "https", "ftp"):
modelXbrl.error("EFM.6.05.16.externalReference",
_("Fact %(fact)s of context %(contextID)s has an invalid external reference in '%(attribute)s' for <%(element)s>"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID,
attribute=attrTag, element=eltTag)
if attrTag == "src" and attrValue not in checkedGraphicsFiles:
if scheme(attrValue) == "data":
modelXbrl.error("EFM.6.05.16.graphicDataUrl",
_("Fact %(fact)s of context %(contextID)s references a graphics data URL which isn't accepted '%(attribute)s' for <%(element)s>"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID,
attribute=attrValue[:32], element=eltTag)
elif attrValue.lower()[-4:] not in ('.jpg', '.gif'):
modelXbrl.error("EFM.6.05.16.graphicFileType",
_("Fact %(fact)s of context %(contextID)s references a graphics file which isn't .gif or .jpg '%(attribute)s' for <%(element)s>"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID,
attribute=attrValue, element=eltTag)
else: # test file contents
try:
if validateGraphicFile(f1, attrValue) != attrValue.lower()[-3:]:
modelXbrl.error("EFM.6.05.16.graphicFileContent",
_("Fact %(fact)s of context %(contextID)s references a graphics file which doesn't have expected content '%(attribute)s' for <%(element)s>"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID,
attribute=attrValue, element=eltTag)
except IOError as err:
modelXbrl.error("EFM.6.05.16.graphicFileError",
_("Fact %(fact)s of context %(contextID)s references a graphics file which isn't openable '%(attribute)s' for <%(element)s>, error: %(error)s"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID,
attribute=attrValue, element=eltTag, error=err)
checkedGraphicsFiles.add(attrValue)
if eltTag == "table" and any(a is not None for a in elt.iterancestors("table")):
modelXbrl.error("EFM.6.05.16.nestedTable",
_("Fact %(fact)s of context %(contextID)s has nested <table> elements."),
modelObject=f1, fact=f1.qname, contextID=f1.contextID)
except (XMLSyntaxError,
UnicodeDecodeError) as err:
#if not err.endswith("undefined entity"):
modelXbrl.error(("EFM.6.05.15", "GFM.1.02.14"),
_("Fact %(fact)s contextID %(contextID)s has text which causes the XML error %(error)s"),
modelObject=f1, fact=f1.qname, contextID=f1.contextID, error=err)
checkedGraphicsFiles.clear()
