def resolveHtmlUri(elt, name, value):
if name == "archive": # URILIST
return " ".join(
resolveHtmlUri(elt, "archiveListElement", v)
for v in value.split(" "))
if not UrlUtil.isAbsolute(value):
if elt.localName == "object" and name in (
"classid", "data",
"archiveListElement") and elt.get("codebase"):
base = elt.get("codebase") + "/"
else:
base = getattr(
elt.modelDocument, "htmlBase"
) # None if no htmlBase, empty string if it's not set
if base:
if value.startswith("/"): # add to authority
value = UrlUtil.authority(base) + value
elif value.startswith("#"): # add anchor to base document
value = base + value
else:
value = os.path.dirname(base) + "/" + value
# canonicalize ../ and ./
scheme, sep, pathpart = value.rpartition("://")
if sep:
pathpart = pathpart.replace('\\', '/')
endingSep = '/' if pathpart[
-1] == '/' else '' # normpath drops ending directory separator
_uri = scheme + "://" + posixpath.normpath(pathpart) + endingSep
else:
_uri = posixpath.normpath(value)
return _uri # .replace(" ", "%20") requirement for this is not yet clear
def resolveUri(self, hrefObject=None, uri=None, dtsModelXbrl=None):
if dtsModelXbrl is None:
dtsModelXbrl = self.modelXbrl
doc = None
if hrefObject:
hrefElt,doc,id = hrefObject
elif uri:
from arelle import UrlUtil
url, id = UrlUtil.splitDecodeFragment(uri)
if url == "":
doc = self.modelDocument
else:
normalizedUrl = self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
url,
self.modelDocument.baseForElement(self))
doc = dtsModelXbrl.urlDocs.get(normalizedUrl)
from arelle import ModelDocument
if isinstance(doc, ModelDocument.ModelDocument):
if id is None:
return doc
elif id in doc.idObjects:
return doc.idObjects[id]
else:
from arelle.XmlUtil import xpointerElement
xpointedElement = xpointerElement(doc,id)
# find element
for docModelObject in doc.modelObjects:
if docModelObject == xpointedElement:
doc.idObjects[id] = docModelObject # cache for reuse
return docModelObject
return None
def base_uri(xc, p, contextItem, args):
item = anytypeArg(xc, args, 0, "node()?", missingArgFallback=contextItem)
if item == ():
return ''
if isinstance(item, (ModelObject, ModelDocument)):
return UrlUtil.ensureUrl(item.modelDocument.uri)
return ''
def checkConcept(val, concept):
if concept.element.hasAttributeNS(XbrlConst.xbrldt, "typedDomainRef"):
if concept.isDimensionItem:
typedDomainElement = concept.typedDomainElement
if typedDomainElement is None:
url, id = UrlUtil.splitDecodeFragment(concept.element.getAttributeNS(XbrlConst.xbrldt, "typedDomainRef"))
if len(id) == 0:
val.modelXbrl.error(
_("Concept {0} typedDomainRef has no fragment identifier").format(
concept.qname),
"err", "xbrldte:TypedDimensionURIError")
else:
val.modelXbrl.error(
_("Concept {0} typedDomainRef is not resolved").format(
concept.qname),
"err", "xbrldte:OutOfDTSSchemaError")
elif not isinstance(typedDomainElement, ModelObject.ModelConcept) or \
not typedDomainElement.isGlobalDeclaration or \
typedDomainElement.abstract == "true":
val.modelXbrl.error(
_("Concept {0} typedDomainRef must identify a non-abstract element").format(
concept.qname),
"err", "xbrldte:TypedDimensionError")
else:
val.modelXbrl.error(
_("Concept {0} is not a dimension item but has a typedDomainRef").format(
concept.qname),
"err", "xbrldte:TypedDomainRefError")
def base_uri(xc, p, contextItem, args):
item = anytypeArg(xc, args, 0, "node()?", missingArgFallback=contextItem)
if item == ():
return ''
if isinstance(item, (ModelObject, ModelDocument)):
return UrlUtil.ensureUrl(item.modelDocument.uri)
return ''
def checkConcept(val, concept):
if concept.element.hasAttributeNS(XbrlConst.xbrldt, "typedDomainRef"):
if concept.isDimensionItem:
typedDomainElement = concept.typedDomainElement
if typedDomainElement is None:
url, id = UrlUtil.splitDecodeFragment(
concept.element.getAttributeNS(XbrlConst.xbrldt,
"typedDomainRef"))
if len(id) == 0:
val.modelXbrl.error(
_("Concept {0} typedDomainRef has no fragment identifier"
).format(concept.qname), "err",
"xbrldte:TypedDimensionURIError")
else:
val.modelXbrl.error(
_("Concept {0} typedDomainRef is not resolved").format(
concept.qname), "err",
"xbrldte:OutOfDTSSchemaError")
elif not isinstance(typedDomainElement, ModelObject.ModelConcept) or \
not typedDomainElement.isGlobalDeclaration or \
typedDomainElement.abstract == "true":
val.modelXbrl.error(
_("Concept {0} typedDomainRef must identify a non-abstract element"
).format(concept.qname), "err",
"xbrldte:TypedDimensionError")
else:
val.modelXbrl.error(
_("Concept {0} is not a dimension item but has a typedDomainRef"
).format(concept.qname), "err",
"xbrldte:TypedDomainRefError")
def doc(xc, p, contextItem, args):
if len(args) != 1: raise XPathContext.FunctionNumArgs()
uri = stringArg(xc, args, 0, "xs:string", emptyFallback=None)
if uri is None:
return ()
if xc.progHeader is None or xc.progHeader.element is None:
raise XPathContext.XPathException(
p, 'err:FODC0005',
_('Function xf:doc no formula resource element for {0}').format(
uri))
if not UrlUtil.isValid(uri):
raise XPathContext.XPathException(
p, 'err:FODC0005',
_('Function xf:doc $uri is not valid {0}').format(uri))
normalizedUri = xc.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
uri,
xc.progHeader.element.modelDocument.baseForElement(
xc.progHeader.element))
if normalizedUri in xc.modelXbrl.urlDocs:
return xc.modelXbrl.urlDocs[normalizedUri].xmlDocument
modelDocument = ModelDocument.load(xc.modelXbrl, normalizedUri)
if modelDocument is None:
raise XPathContext.XPathException(
p, 'err:FODC0005',
_('Function xf:doc $uri not successfully loaded {0}').format(uri))
# assure that document is validated
XmlValidate.validate(xc.modelXbrl, modelDocument.xmlRootElement)
return modelDocument.xmlDocument
def resolveUri(self, hrefObject=None, uri=None, dtsModelXbrl=None):
if dtsModelXbrl is None:
dtsModelXbrl = self.modelXbrl
doc = None
if hrefObject:
hrefElt, doc, id = hrefObject
elif uri:
from arelle import UrlUtil
url, id = UrlUtil.splitDecodeFragment(uri)
if url == "":
doc = self.modelDocument
else:
normalizedUrl = self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
url, self.modelDocument.baseForElement(self))
doc = dtsModelXbrl.urlDocs.get(normalizedUrl)
from arelle import ModelDocument
if isinstance(doc, ModelDocument.ModelDocument):
if id is None:
return doc
elif id in doc.idObjects:
return doc.idObjects[id]
else:
from arelle.XmlUtil import xpointerElement
xpointedElement = xpointerElement(doc, id)
# find element
for docModelObject in doc.modelObjects:
if docModelObject == xpointedElement:
doc.idObjects[id] = docModelObject # cache for reuse
return docModelObject
return None
def loadStandardTaxonomiesDict(self):
if self.selection:
self.standardTaxonomiesDict = {}
self.standardLocalHrefs = set()
self.standardAuthorities = set()
if not self.standardTaxonomiesUrl:
return
basename = os.path.basename(self.standardTaxonomiesUrl)
self.modelManager.cntlr.showStatus(
_("parsing {0}").format(basename))
try:
for file in (self.modelManager.cntlr.webCache.getfilename(
self.standardTaxonomiesUrl),
os.path.join(self.modelManager.cntlr.configDir,
"xbrlschemafiles.xml")):
xmldoc = etree.parse(file)
for locElt in xmldoc.iter(tag="Loc"):
href = None
localHref = None
namespaceUri = None
attType = None
family = None
for childElt in locElt.iterchildren():
ln = childElt.tag
value = childElt.text.strip()
if ln == "Href":
href = value
elif ln == "LocalHref":
localHref = value
elif ln == "Namespace":
namespaceUri = value
elif ln == "AttType":
attType = value
elif ln == "Family":
family = value
if href:
if namespaceUri and (attType == "SCH"
or attType == "ENT"):
if namespaceUri not in self.standardTaxonomiesDict:
self.standardTaxonomiesDict[
namespaceUri] = (href, localHref)
authority = UrlUtil.authority(namespaceUri)
self.standardAuthorities.add(authority)
if family == "BASE":
self.baseTaxonomyNamespaces.add(
namespaceUri)
if href not in self.standardTaxonomiesDict:
self.standardTaxonomiesDict[
href] = "Allowed" + attType
if localHref:
self.standardLocalHrefs.add(localHref)
elif attType == "SCH" and family == "BASE":
self.baseTaxonomyNamespaces.add(namespaceUri)
except (EnvironmentError, etree.LxmlError) as err:
self.modelManager.cntlr.addToLog(
"{0}: import error: {1}".format(basename, err))
etree.clear_error_log()
def loadStandardTaxonomiesDict(self):
if self.selection:
self.standardTaxonomiesDict = {}
self.standardAuthorities = set()
if not self.standardTaxonomiesUrl:
return
basename = os.path.basename(self.standardTaxonomiesUrl)
self.modelManager.cntlr.showStatus(
_("parsing {0}").format(basename))
try:
for file in (self.modelManager.cntlr.webCache.getfilename(
self.standardTaxonomiesUrl),
os.path.join(self.modelManager.cntlr.configDir,
"xbrlschemafiles.xml")):
xmldoc = xml.dom.minidom.parse(file)
for locElt in xmldoc.getElementsByTagName("Loc"):
href = None
localHref = None
namespaceUri = None
attType = None
family = None
for childElt in locElt.childNodes:
if childElt.nodeType == 1: #element
ln = childElt.localName
value = XmlUtil.innerText(childElt)
if ln == "Href":
href = value
elif ln == "LocalHref":
localHref = value
elif ln == "Namespace":
namespaceUri = value
elif ln == "AttType":
attType = value
elif ln == "Family":
family = value
if href:
if namespaceUri and (attType == "SCH"
or attType == "ENT"):
if namespaceUri not in self.standardTaxonomiesDict:
self.standardTaxonomiesDict[
namespaceUri] = (href, localHref)
authority = UrlUtil.authority(namespaceUri)
self.standardAuthorities.add(authority)
if family == "BASE":
self.baseTaxonomyNamespaces.add(
namespaceUri)
if href not in self.standardTaxonomiesDict:
self.standardTaxonomiesDict[
href] = "Allowed" + attType
except (EnvironmentError, xml.parsers.expat.ExpatError,
xml.dom.DOMException) as err:
self.modelManager.cntlr.addToLog(
"{0}: import error: {1}".format(basename, err))
def loadStandardTaxonomiesDict(self):
if self.selection:
self.standardTaxonomiesDict = {}
self.standardLocalHrefs = set()
self.standardAuthorities = set()
if not self.standardTaxonomiesUrl:
return
basename = os.path.basename(self.standardTaxonomiesUrl)
self.modelManager.cntlr.showStatus(_("parsing {0}").format(basename))
try:
for file in (self.modelManager.cntlr.webCache.getfilename(self.standardTaxonomiesUrl),
os.path.join(self.modelManager.cntlr.configDir,"xbrlschemafiles.xml")):
xmldoc = etree.parse(file)
for locElt in xmldoc.iter(tag="Loc"):
href = None
localHref = None
namespaceUri = None
attType = None
family = None
for childElt in locElt.iterchildren():
ln = childElt.tag
value = childElt.text.strip()
if ln == "Href":
href = value
elif ln == "LocalHref":
localHref = value
elif ln == "Namespace":
namespaceUri = value
elif ln == "AttType":
attType = value
elif ln == "Family":
family = value
if href:
if namespaceUri and (attType == "SCH" or attType == "ENT"):
if namespaceUri not in self.standardTaxonomiesDict:
self.standardTaxonomiesDict[namespaceUri] = (href, localHref)
authority = UrlUtil.authority(namespaceUri)
self.standardAuthorities.add(authority)
if family == "BASE":
self.baseTaxonomyNamespaces.add(namespaceUri)
if href not in self.standardTaxonomiesDict:
self.standardTaxonomiesDict[href] = "Allowed" + attType
if localHref:
self.standardLocalHrefs.add(localHref)
elif attType == "SCH" and family == "BASE":
self.baseTaxonomyNamespaces.add(namespaceUri)
except (EnvironmentError,
etree.LxmlError) as err:
self.modelManager.cntlr.addToLog("{0}: import error: {1}".format(basename,err))
etree.clear_error_log()
def discoverHref(self, element, nonDTS=False):
if element.hasAttributeNS(XbrlConst.xlink, "href"):
url, id = UrlUtil.splitDecodeFragment(element.getAttributeNS(XbrlConst.xlink, "href"))
if url == "":
doc = self
else:
doc = load(self.modelXbrl, url, base=self.baseForElement(element))
if not nonDTS and doc is not None and self.referencesDocument.get(doc) is None:
self.referencesDocument[doc] = "href"
doc.inDTS = doc.type != Type.Unknown # non-XBRL document is not in DTS
href = (element, doc, id if len(id) > 0 else None)
self.hrefObjects.append(href)
return href
return None
def insertFiling(self, rssItem, g):
self.showStatus("insert filing")
# accession graph -> document vertices
new_filing = {'documents': []}
if self.modelXbrl.modelDocument.creationSoftwareComment:
new_filing['creation_software'] = self.modelXbrl.modelDocument.creationSoftwareComment
datetimeNow = datetime.datetime.now()
datetimeNowStr = XmlUtil.dateunionValue(datetimeNow)
entryUri = modelObjectDocumentUri(self.modelXbrl)
if rssItem is not None: # sec accession
# set self.
new_filing['filingType'] = "SEC filing"
# for an RSS Feed entry from SEC, use rss item's accession information
new_filing['filingNumber'] = filingNumber = rssItem.accessionNumber
new_filing['acceptedTimestamp'] = XmlUtil.dateunionValue(rssItem.acceptanceDatetime)
new_filing['filingDate'] = XmlUtil.dateunionValue(rssItem.filingDate)
new_filing['entityId'] = rssItem.cikNumber
new_filing['entityName'] = rssItem.companyName
new_filing['SICCode'] = rssItem.assignedSic
new_filing['SECHtmlUrl'] = rssItem.htmlUrl
new_filing['entryUrl'] = rssItem.url
self.filingURI = rssItem.htmlUrl
else:
# not an RSS Feed item, make up our own accession ID (the time in seconds of epoch)
intNow = int(time.time())
new_filing['filingNumber'] = filingNumber = str(intNow)
self.filingId = int(time.time()) # only available if entered from an SEC filing
new_filing['filingType'] = "independent filing"
new_filing['acceptedTimestamp'] = datetimeNowStr
new_filing['filingDate'] = datetimeNowStr
new_filing['entryUrl'] = UrlUtil.ensureUrl(self.modelXbrl.fileSource.url)
self.filingURI = filingNumber
g[FILINGS][self.filingURI] = new_filing
self.filing = new_filing
# for now only one report per filing (but SEC may have multiple in future, such as form SD)
self.reportURI = modelObjectDocumentUri(self.modelXbrl)
self.report = {'filing': self.filingURI,
'aspectProxies': {},
'relationshipSets': {},
'dataPoints': {},
'messages': {}}
new_filing['reports'] = {self.reportURI: self.report}
# relationshipSets are a dts property
self.relationshipSets = [(arcrole, ELR, linkqname, arcqname)
for arcrole, ELR, linkqname, arcqname in self.modelXbrl.baseSets.keys()
if ELR and (arcrole.startswith("XBRL-") or (linkqname and arcqname))]
def resolveHtmlUri(elt, name, value):
if name == "archive": # URILIST
return " ".join(resolveHtmlUri(elt, None, v) for v in value.split(" "))
if not UrlUtil.isAbsolute(value) and not value.startswith("/"):
if elt.modelDocument.htmlBase is not None:
value = elt.modelDocument.htmlBase + value
# canonicalize ../ and ./
scheme, sep, pathpart = value.rpartition("://")
if sep:
pathpart = pathpart.replace('\\','/')
endingSep = '/' if pathpart[-1] == '/' else '' # normpath drops ending directory separator
_uri = scheme + "://" + posixpath.normpath(pathpart) + endingSep
else:
_uri = posixpath.normpath(value)
return _uri # .replace(" ", "%20") requirement for this is not yet clear
def loadStandardTaxonomiesDict(self):
if self.selection:
self.standardTaxonomiesDict = {}
self.standardAuthorities = set()
if not self.standardTaxonomiesUrl:
return
basename = os.path.basename(self.standardTaxonomiesUrl)
self.modelManager.cntlr.showStatus(_("parsing {0}").format(basename))
try:
for file in (self.modelManager.cntlr.webCache.getfilename(self.standardTaxonomiesUrl),
os.path.join(self.modelManager.cntlr.configDir,"xbrlschemafiles.xml")):
xmldoc = xml.dom.minidom.parse(file)
for locElt in xmldoc.getElementsByTagName("Loc"):
href = None
localHref = None
namespaceUri = None
attType = None
family = None
for childElt in locElt.childNodes:
if childElt.nodeType == 1: #element
ln = childElt.localName
value = XmlUtil.innerText(childElt)
if ln == "Href":
href = value
elif ln == "LocalHref":
localHref = value
elif ln == "Namespace":
namespaceUri = value
elif ln == "AttType":
attType = value
elif ln == "Family":
family = value
if href:
if namespaceUri and (attType == "SCH" or attType == "ENT"):
if namespaceUri not in self.standardTaxonomiesDict:
self.standardTaxonomiesDict[namespaceUri] = (href, localHref)
authority = UrlUtil.authority(namespaceUri)
self.standardAuthorities.add(authority)
if family == "BASE":
self.baseTaxonomyNamespaces.add(namespaceUri)
if href not in self.standardTaxonomiesDict:
self.standardTaxonomiesDict[href] = "Allowed" + attType
except (EnvironmentError,
xml.parsers.expat.ExpatError,
xml.dom.DOMException) as err:
self.modelManager.cntlr.addToLog("{0}: import error: {1}".format(basename,err))
def discoverHref(self, element, nonDTS=False):
if element.hasAttributeNS(XbrlConst.xlink, "href"):
url, id = UrlUtil.splitDecodeFragment(
element.getAttributeNS(XbrlConst.xlink, "href"))
if url == "":
doc = self
else:
doc = load(self.modelXbrl,
url,
base=self.baseForElement(element))
if not nonDTS and doc is not None and self.referencesDocument.get(
doc) is None:
self.referencesDocument[doc] = "href"
doc.inDTS = doc.type != Type.Unknown # non-XBRL document is not in DTS
href = (element, doc, id if len(id) > 0 else None)
self.hrefObjects.append(href)
return href
return None
def discoverHref(self, element, nonDTS=False):
href = element.get("{http://www.w3.org/1999/xlink}href")
if href:
url, id = UrlUtil.splitDecodeFragment(href)
if url == "":
doc = self
else:
# href discovery only can happein within a DTS
doc = load(self.modelXbrl, url, isDiscovered=not nonDTS, base=self.baseForElement(element), referringElement=element)
if not nonDTS and doc is not None and self.referencesDocument.get(doc) is None:
self.referencesDocument[doc] = "href"
if not doc.inDTS and doc.type != Type.Unknown: # non-XBRL document is not in DTS
doc.inDTS = True # now known to be discovered
if doc.type == Type.SCHEMA: # schema coming newly into DTS
doc.schemaDiscoverChildElements(doc.xmlRootElement)
href = (element, doc, id if len(id) > 0 else None)
self.hrefObjects.append(href)
return href
return None
def doc(xc, p, contextItem, args):
if len(args) != 1: raise XPathContext.FunctionNumArgs()
uri = stringArg(xc, args, 0, "xs:string", emptyFallback=None)
if uri is None:
return ()
if xc.progHeader is None or xc.progHeader.element is None:
raise XPathContext.XPathException(p, 'err:FODC0005', _('Function xf:doc no formula resource element for {0}').format(uri))
if not UrlUtil.isValid(uri):
raise XPathContext.XPathException(p, 'err:FODC0005', _('Function xf:doc $uri is not valid {0}').format(uri))
normalizedUri = xc.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
uri,
xc.progHeader.element.modelDocument.baseForElement(xc.progHeader.element))
if normalizedUri in xc.modelXbrl.urlDocs:
return xc.modelXbrl.urlDocs[normalizedUri].xmlDocument
modelDocument = ModelDocument.load(xc.modelXbrl, normalizedUri)
if modelDocument is None:
raise XPathContext.XPathException(p, 'err:FODC0005', _('Function xf:doc $uri not successfully loaded {0}').format(uri))
# assure that document is validated
XmlValidate.validate(xc.modelXbrl, modelDocument.xmlRootElement)
return modelDocument.xmlDocument
def resolveUri(self, hrefObject=None, uri=None, dtsModelXbrl=None):
"""Returns the modelObject within modelDocment that resolves a URI based on arguments relative
to this element
:param hrefObject: an optional tuple of (hrefElement, modelDocument, id), or
:param uri: An (element scheme pointer), and dtsModelXbrl (both required together if for a multi-instance href)
:type uri: str
:param dtsModelXbrl: DTS of href resolution (default is the element's own modelXbrl)
:type dtsModelXbrl: ModelXbrl
:returns: ModelObject -- Document node corresponding to the href or resolved uri
"""
if dtsModelXbrl is None:
dtsModelXbrl = self.modelXbrl
doc = None
if hrefObject:
hrefElt,doc,id = hrefObject
elif uri:
from arelle import UrlUtil
url, id = UrlUtil.splitDecodeFragment(uri)
if url == "":
doc = self.modelDocument
else:
normalizedUrl = self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
url,
self.modelDocument.baseForElement(self))
doc = dtsModelXbrl.urlDocs.get(normalizedUrl)
from arelle import ModelDocument
if isinstance(doc, ModelDocument.ModelDocument):
if id is None:
return doc
elif id in doc.idObjects:
return doc.idObjects[id]
else:
from arelle.XmlUtil import xpointerElement
xpointedElement = xpointerElement(doc,id)
# find element
for docModelObject in doc.xmlRootElement.iter():
if docModelObject == xpointedElement:
doc.idObjects[id] = docModelObject # cache for reuse
return docModelObject
return None
def identifyPreexistingDocuments(self):
self.existingDocumentUris = set()
if not self.isJsonFile:
docFilters = []
for modelDocument in self.modelXbrl.urlDocs.values():
if modelDocument.type == Type.SCHEMA:
docFilters.append('STR(?doc) = "{}"'.format(UrlUtil.ensureUrl(modelDocument.uri)))
results = self.execute(
# TBD: fix up for Mongo DB query
"select",
query="""
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX DTS: <http://xbrl.org/2013/rdf/DTS/>
SELECT distinct ?doc WHERE { ?doc rdf:type DTS:Document
FILTER( """ + '\n|| '.join(docFilters) + ") .}")
try:
for result in results['results']['bindings']:
doc = result['doc']
if doc.get('type') == 'uri':
self.existingDocumentUris.add(doc['value'])
except KeyError:
pass # no existingDocumentUris
def checkConcept(val, concept):
if concept.get("{http://xbrl.org/2005/xbrldt}typedDomainRef"):
if concept.isDimensionItem:
typedDomainElement = concept.typedDomainElement
if typedDomainElement is None:
url, id = UrlUtil.splitDecodeFragment(concept.get("{http://xbrl.org/2005/xbrldt}typedDomainRef"))
if len(id) == 0:
val.modelXbrl.error(
"xbrldte:TypedDimensionURIError",
_("Concept %(concept)s typedDomainRef has no fragment identifier"),
modelObject=concept,
concept=concept.qname,
)
else:
val.modelXbrl.error(
"xbrldte:OutOfDTSSchemaError",
_("Concept %(concept)s typedDomainRef is not resolved"),
modelObject=concept,
concept=concept.qname,
)
elif (
not isinstance(typedDomainElement, ModelConcept)
or not typedDomainElement.isGlobalDeclaration
or typedDomainElement.abstract == "true"
):
val.modelXbrl.error(
"xbrldte:TypedDimensionError",
_("Concept %(concept)s typedDomainRef must identify a non-abstract element"),
modelObject=concept,
concept=concept.qname,
)
else:
val.modelXbrl.error(
"xbrldte:TypedDomainRefError",
_("Concept %(concept)s is not a dimension item but has a typedDomainRef"),
modelObject=concept,
concept=concept.qname,
)
def checkConcept(val, concept):
if concept.get("{http://xbrl.org/2005/xbrldt}typedDomainRef"):
if concept.isDimensionItem:
typedDomainElement = concept.typedDomainElement
if typedDomainElement is None:
url, id = UrlUtil.splitDecodeFragment(
concept.get("{http://xbrl.org/2005/xbrldt}typedDomainRef"))
if len(id) == 0:
val.modelXbrl.error(
"xbrldte:TypedDimensionURIError",
_("Concept %(concept)s typedDomainRef has no fragment identifier"
),
modelObject=concept,
concept=concept.qname)
else:
val.modelXbrl.error(
"xbrldte:OutOfDTSSchemaError",
_("Concept %(concept)s typedDomainRef is not resolved"
),
modelObject=concept,
concept=concept.qname)
elif not isinstance(typedDomainElement, ModelConcept) or \
not typedDomainElement.isGlobalDeclaration or \
typedDomainElement.abstract == "true":
val.modelXbrl.error(
"xbrldte:TypedDimensionError",
_("Concept %(concept)s typedDomainRef must identify a non-abstract element"
),
modelObject=concept,
concept=concept.qname)
else:
val.modelXbrl.error(
"xbrldte:TypedDomainRefError",
_("Concept %(concept)s is not a dimension item but has a typedDomainRef"
),
modelObject=concept,
concept=concept.qname)
def discoverHref(self, element, nonDTS=False):
href = element.get("{http://www.w3.org/1999/xlink}href")
if href:
url, id = UrlUtil.splitDecodeFragment(href)
if url == "":
doc = self
else:
# href discovery only can happein within a DTS
doc = load(self.modelXbrl,
url,
isDiscovered=not nonDTS,
base=self.baseForElement(element),
referringElement=element)
if not nonDTS and doc is not None and self.referencesDocument.get(
doc) is None:
self.referencesDocument[doc] = "href"
if not doc.inDTS and doc.type > Type.UnknownTypes: # non-XBRL document is not in DTS
doc.inDTS = True # now known to be discovered
if doc.type == Type.SCHEMA: # schema coming newly into DTS
doc.schemaDiscoverChildElements(doc.xmlRootElement)
href = (element, doc, id if len(id) > 0 else None)
self.hrefObjects.append(href)
return href
return None
def final(val):
if not (val.validateEBA or val.validateEIOPA):
return
modelXbrl = val.modelXbrl
modelDocument = modelXbrl.modelDocument
_statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name)
modelXbrl.profileActivity()
modelXbrl.modelManager.showStatus(_statusMsg)
if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA):
if not modelDocument.uri.endswith(".xbrl"):
modelXbrl.warning("EBA.1.1",
_('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'),
modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1])
modelXbrl.error("EIOPA.S.1.1.a",
_('XBRL instance documents MUST use the extension ".xbrl" but it is "%(extension)s"'),
modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1])
if val.isEIOPA_2_0_1: _encodings = ("UTF-8", "utf-8-sig")
else: _encodings = ("utf-8", "UTF-8", "utf-8-sig")
if modelDocument.documentEncoding not in _encodings:
modelXbrl.error(("EBA.1.4", "EIOPA.1.4"),
_('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'),
modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding)
schemaRefElts = []
schemaRefFileNames = []
for doc, docRef in modelDocument.referencesDocument.items():
if docRef.referenceType == "href":
if docRef.referringModelObject.localName == "schemaRef":
schemaRefElts.append(docRef.referringModelObject)
schemaRefFileNames.append(doc.basename)
if not UrlUtil.isAbsolute(doc.uri):
modelXbrl.error(("EBA.2.2", "EIOPA.S.1.5.a" if val.isEIOPAfullVersion else "EIOPA.S.1.5.b"),
_('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'),
modelObject=docRef.referringModelObject, url=doc.uri,
messageCodes=("EBA.2.2", "EIOPA.S.1.5.a","EIOPA.S.1.5.b"))
elif docRef.referringModelObject.localName == "linkbaseRef":
modelXbrl.error(("EBA.2.3","EIOPA.S.1.5.a"),
_('The link:linkbaseRef element is not allowed: %(fileName)s.'),
modelObject=docRef.referringModelObject, fileName=doc.basename)
_numSchemaRefs = len(XmlUtil.children(modelDocument.xmlRootElement, XbrlConst.link, "schemaRef"))
if _numSchemaRefs > 1:
modelXbrl.error(("EIOPA.S.1.5.a", "EBA.1.5"),
_('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'),
modelObject=modelDocument, numEntryPoints=_numSchemaRefs, entryPointNames=', '.join(sorted(schemaRefFileNames)))
### check entry point names appropriate for filing indicator (DPM DB?)
if len(schemaRefElts) != 1:
modelXbrl.error("EBA.2.3",
_('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'),
modelObject=schemaRefElts, entryPointCount=len(schemaRefElts))
# non-streaming EBA checks
if not getattr(modelXbrl, "isStreamingMode", False):
val.qnReportedCurrency = None
if val.isEIOPA_2_0_1 and qnMetReportingCurrency in modelXbrl.factsByQname:
for _multiCurrencyFact in modelXbrl.factsByQname[qnMetReportingCurrency]:
# multi-currency fact
val.qnReportedCurrency = _multiCurrencyFact.xValue
break
validateFacts(val, modelXbrl.facts)
# check sum of fact md5s (otherwise checked in streaming process)
xbrlFactsCheckVersion = None
expectedSumOfFactMd5s = None
for pi in modelDocument.xmlRootElement.getchildren():
if isinstance(pi, etree._ProcessingInstruction) and pi.target == "xbrl-facts-check":
_match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text)
if _match:
_matchGroups = _match.groups()
if len(_matchGroups) == 2:
if _matchGroups[0] == "version":
xbrlFactsCheckVersion = _matchGroups[1]
elif _matchGroups[0] == "sum-of-fact-md5s":
try:
expectedSumOfFactMd5s = Md5Sum(_matchGroups[1])
except ValueError:
modelXbrl.error("EIOPA:xbrlFactsCheckError",
_("Invalid sum-of-md5s %(sumOfMd5)s"),
modelObject=modelXbrl, sumOfMd5=_matchGroups[1])
if xbrlFactsCheckVersion and expectedSumOfFactMd5s:
sumOfFactMd5s = Md5Sum()
for f in modelXbrl.factsInInstance:
sumOfFactMd5s += f.md5sum
if sumOfFactMd5s != expectedSumOfFactMd5s:
modelXbrl.warning("EIOPA:xbrlFactsCheckWarning",
_("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"),
modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s)
else:
modelXbrl.info("info",
_("Successful XBRL facts sum of md5s."),
modelObject=modelXbrl)
if any(badError in modelXbrl.errors
for badError in ("EBA.2.1", "EIOPA.2.1", "EIOPA.S.1.5.a/EIOPA.S.1.5.b")):
pass # skip checking filingIndicators if bad errors
elif not val.filingIndicators:
modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"),
_('Missing filing indicators. Reported XBRL instances MUST include appropriate (positive) filing indicator elements'),
modelObject=modelDocument)
elif all(filed == False for filed in val.filingIndicators.values()):
modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"),
_('All filing indicators are filed="false". Reported XBRL instances MUST include appropriate (positive) filing indicator elements'),
modelObject=modelDocument)
if val.numFilingIndicatorTuples > 1:
modelXbrl.warning(("EBA.1.6.2", "EIOPA.1.6.2"),
_('Multiple filing indicators tuples when not in streaming mode (info).'),
modelObject=modelXbrl.factsByQname[qnFIndicators])
if len(val.cntxDates) > 1:
modelXbrl.error(("EBA.2.13","EIOPA.2.13"),
_('Contexts must have the same date: %(dates)s.'),
# when streaming values are no longer available, but without streaming they can be logged
modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs),
dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True)
for _dt in val.cntxDates.keys()))
if val.unusedCntxIDs:
if val.isEIOPA_2_0_1:
modelXbrl.error("EIOPA.2.7",
_('Unused xbrli:context nodes MUST NOT be present in the instance: %(unusedContextIDs)s.'),
modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts],
unusedContextIDs=", ".join(sorted(val.unusedCntxIDs)))
else:
modelXbrl.warning(("EBA.2.7", "EIOPA.2.7"),
_('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'),
modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts],
unusedContextIDs=", ".join(sorted(val.unusedCntxIDs)))
if len(val.cntxEntities) > 1:
modelXbrl.error(("EBA.2.9", "EIOPA.2.9"),
_('All entity identifiers and schemes MUST be the same, %(count)s found: %(entities)s.'),
modelObject=modelDocument, count=len(val.cntxEntities),
entities=", ".join(sorted(str(cntxEntity) for cntxEntity in val.cntxEntities)))
for _scheme, _LEI in val.cntxEntities:
if (_scheme in ("http://standards.iso.org/iso/17442", "http://standard.iso.org/iso/17442", "LEI") or
(not val.isEIOPAfullVersion and _scheme == "PRE-LEI")):
result = LeiUtil.checkLei(_LEI)
if result == LeiUtil.LEI_INVALID_LEXICAL:
modelXbrl.error("EIOPA.S.2.8.c",
_("Context has lexically invalid LEI %(lei)s."),
modelObject=modelDocument, lei=_LEI)
elif result == LeiUtil.LEI_INVALID_CHECKSUM:
modelXbrl.error("EIOPA.S.2.8.c",
_("Context has LEI checksum error in %(lei)s."),
modelObject=modelDocument, lei=_LEI)
if _scheme == "http://standard.iso.org/iso/17442":
modelXbrl.warning("EIOPA.S.2.8.c",
_("Warning, context has entity scheme %(scheme)s should be plural: http://standards.iso.org/iso/17442."),
modelObject=modelDocument, scheme=_scheme)
elif _scheme == "SC":
pass # anything is ok for Specific Code
else:
modelXbrl.error("EIOPA.S.2.8.c",
_("Context has unrecognized entity scheme %(scheme)s."),
modelObject=modelDocument, scheme=_scheme)
if val.unusedUnitIDs:
if val.isEIOPA_2_0_1:
modelXbrl.error("EIOPA.2.22",
_('Unused xbrli:unit nodes MUST NOT be present in the instance: %(unusedUnitIDs)s.'),
modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units],
unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs)))
else:
modelXbrl.warning(("EBA.2.22", "EIOPA.2.22"),
_('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'),
modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units],
unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs)))
if len(val.currenciesUsed) > 1:
modelXbrl.error(("EBA.3.1","EIOPA.3.1"),
_("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"),
modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join(str(c) for c in val.currenciesUsed.keys()))
elif val.isEIOPA_2_0_1 and any(_measure.localName != val.reportingCurrency for _measure in val.currenciesUsed.keys()):
modelXbrl.error("EIOPA.3.1",
_("There MUST be only one currency but reporting currency %(reportingCurrency)s differs from unit currencies: %(currencies)s.'"),
modelObject=val.currenciesUsed.values(), reportingCurrency=val.reportingCurrency, currencies=", ".join(str(c) for c in val.currenciesUsed.keys()))
if val.prefixesUnused:
modelXbrl.warning(("EBA.3.4", "EIOPA.3.4"),
_("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"),
modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused)))
for ns, prefixes in val.namespacePrefixesUsed.items():
nsDocs = modelXbrl.namespaceDocs.get(ns)
if nsDocs:
for nsDoc in nsDocs:
nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns)
if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None):
modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"),
_("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"),
modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None})))
elif ns in CANONICAL_PREFIXES and any(prefix != CANONICAL_PREFIXES[ns] for prefix in prefixes if prefix is not None):
modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"),
_("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"),
modelObject=modelDocument, namespace=ns, declaredPrefix=CANONICAL_PREFIXES[ns], foundPrefixes=', '.join(sorted(prefixes - {None})))
modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0)
modelXbrl.modelManager.showStatus(None)
del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements
del val.utrValidator, val.firstFact, val.footnotesRelationshipSet
def final(val):
if not (val.validateEBA or val.validateEIOPA):
return
modelXbrl = val.modelXbrl
modelDocument = modelXbrl.modelDocument
_statusMsg = _("validating {0} filing rules").format(
val.disclosureSystem.name)
modelXbrl.profileActivity()
modelXbrl.modelManager.showStatus(_statusMsg)
if modelDocument.type == ModelDocument.Type.INSTANCE and (
val.validateEBA or val.validateEIOPA):
if not modelDocument.uri.endswith(".xbrl"):
modelXbrl.warning(
"EBA.1.1",
_('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'
),
modelObject=modelDocument,
extension=os.path.splitext(modelDocument.basename)[1])
if modelDocument.documentEncoding.lower() not in ("utf-8",
"utf-8-sig"):
modelXbrl.error(
"EBA.1.4",
_('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'
),
modelObject=modelDocument,
xmlEncoding=modelDocument.documentEncoding)
schemaRefElts = []
schemaRefFileNames = []
for doc, docRef in modelDocument.referencesDocument.items():
if docRef.referenceType == "href":
if docRef.referringModelObject.localName == "schemaRef":
schemaRefElts.append(docRef.referringModelObject)
schemaRefFileNames.append(doc.basename)
if not UrlUtil.isAbsolute(doc.uri):
modelXbrl.error(
"EBA.2.2",
_('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'
),
modelObject=docRef.referringModelObject,
url=doc.uri)
elif docRef.referringModelObject.localName == "linkbaseRef":
modelXbrl.error(
"EBA.2.3",
_('The link:linkbaseRef element is not allowed: %(fileName)s.'
),
modelObject=docRef.referringModelObject,
fileName=doc.basename)
if len(schemaRefFileNames) > 1:
modelXbrl.error(
"EBA.1.5",
_('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'
),
modelObject=modelDocument,
numEntryPoints=len(schemaRefFileNames),
entryPointNames=', '.join(sorted(schemaRefFileNames)))
### check entry point names appropriate for filing indicator (DPM DB?)
if len(schemaRefElts) != 1:
modelXbrl.error(
"EBA.2.3",
_('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'
),
modelObject=schemaRefElts,
entryPointCount=len(schemaRefElts))
# non-streaming EBA checks
if not getattr(modelXbrl, "isStreamingMode", False):
validateFacts(val, modelXbrl.facts)
# check sum of fact md5s (otherwise checked in streaming process)
xbrlFactsCheckVersion = None
expectedSumOfFactMd5s = None
for pi in modelDocument.xmlRootElement.getchildren():
if isinstance(pi, etree._ProcessingInstruction
) and pi.target == "xbrl-facts-check":
_match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']",
pi.text)
if _match:
_matchGroups = _match.groups()
if len(_matchGroups) == 2:
if _matchGroups[0] == "version":
xbrlFactsCheckVersion = _matchGroups[1]
elif _matchGroups[0] == "sum-of-fact-md5s":
try:
expectedSumOfFactMd5s = Md5Sum(
_matchGroups[1])
except ValueError:
modelXbrl.error(
"EIOPA:xbrlFactsCheckError",
_("Invalid sum-of-md5s %(sumOfMd5)s"),
modelObject=modelXbrl,
sumOfMd5=_matchGroups[1])
if xbrlFactsCheckVersion and expectedSumOfFactMd5s:
sumOfFactMd5s = Md5Sum()
for f in modelXbrl.factsInInstance:
sumOfFactMd5s += f.md5sum
if sumOfFactMd5s != expectedSumOfFactMd5s:
modelXbrl.warning(
"EIOPA:xbrlFactsCheckWarning",
_("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"
),
modelObject=modelXbrl,
expectedMd5=expectedSumOfFactMd5s,
actualMd5Sum=sumOfFactMd5s)
else:
modelXbrl.info("info",
_("Successful XBRL facts sum of md5s."),
modelObject=modelXbrl)
if not val.filingIndicators:
modelXbrl.error(
"EBA.1.6",
_('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements'
),
modelObject=modelDocument)
if val.numFilingIndicatorTuples > 1:
modelXbrl.info(
"EBA.1.6.2",
_('Multiple filing indicators tuples when not in streaming mode (info).'
),
modelObject=modelXbrl.factsByQname[qnFIndicators])
if len(val.cntxDates) > 1:
modelXbrl.error(
"EBA.2.13",
_('Contexts must have the same date: %(dates)s.'),
# when streaming values are no longer available, but without streaming they can be logged
modelObject=set(_cntx for _cntxs in val.cntxDates.values()
for _cntx in _cntxs),
dates=', '.join(
XmlUtil.dateunionValue(_dt, subtractOneDay=True)
for _dt in val.cntxDates.keys()))
if val.unusedCntxIDs:
modelXbrl.warning(
"EBA.2.7",
_('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'
),
modelObject=[
modelXbrl.contexts[unusedCntxID]
for unusedCntxID in val.unusedCntxIDs
if unusedCntxID in modelXbrl.contexts
],
unusedContextIDs=", ".join(sorted(val.unusedCntxIDs)))
if len(val.cntxEntities) > 1:
modelXbrl.warning(
"EBA.2.9",
_('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.'
),
modelObject=modelDocument,
count=len(val.cntxEntities),
entities=", ".join(
sorted(str(cntxEntity)
for cntxEntity in val.cntxEntities)))
if val.unusedUnitIDs:
modelXbrl.warning(
"EBA.2.21",
_('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'
),
modelObject=[
modelXbrl.units[unusedUnitID]
for unusedUnitID in val.unusedUnitIDs
if unusedUnitID in modelXbrl.units
],
unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs)))
if len(val.currenciesUsed) > 1:
modelXbrl.error(
"EBA.3.1",
_("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"
),
modelObject=val.currenciesUsed.values(),
numCurrencies=len(val.currenciesUsed),
currencies=", ".join(
str(c) for c in val.currenciesUsed.keys()))
if val.prefixesUnused:
modelXbrl.warning(
"EBA.3.4",
_("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"
),
modelObject=modelDocument,
unusedPrefixes=', '.join(sorted(val.prefixesUnused)))
for ns, prefixes in val.namespacePrefixesUsed.items():
nsDocs = modelXbrl.namespaceDocs.get(ns)
if nsDocs:
for nsDoc in nsDocs:
nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns)
if any(prefix != nsDocPrefix for prefix in prefixes
if prefix is not None):
modelXbrl.warning(
"EBA.3.5",
_("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"
),
modelObject=modelDocument,
namespace=ns,
declaredPrefix=nsDocPrefix,
foundPrefixes=', '.join(sorted(prefixes - {None})))
modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0)
modelXbrl.modelManager.showStatus(None)
del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements
del val.utrValidator
def uriAuthorityValid(self, uri):
return UrlUtil.authority(uri) in self.standardAuthorities
def final(val):
if not (val.validateEBA or val.validateEIOPA):
return
modelXbrl = val.modelXbrl
modelDocument = modelXbrl.modelDocument
_statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name)
modelXbrl.profileActivity()
modelXbrl.modelManager.showStatus(_statusMsg)
if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA):
if not modelDocument.uri.endswith(".xbrl"):
modelXbrl.warning("EBA.1.1",
_('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'),
modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1])
if modelDocument.documentEncoding.lower() not in ("utf-8", "utf-8-sig"):
modelXbrl.error("EBA.1.4",
_('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'),
modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding)
schemaRefElts = []
schemaRefFileNames = []
for doc, docRef in modelDocument.referencesDocument.items():
if docRef.referenceType == "href":
if docRef.referringModelObject.localName == "schemaRef":
schemaRefElts.append(docRef.referringModelObject)
schemaRefFileNames.append(doc.basename)
if not UrlUtil.isAbsolute(doc.uri):
modelXbrl.error("EBA.2.2",
_('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'),
modelObject=docRef.referringModelObject, url=doc.uri)
elif docRef.referringModelObject.localName == "linkbaseRef":
modelXbrl.error("EBA.2.3",
_('The link:linkbaseRef element is not allowed: %(fileName)s.'),
modelObject=docRef.referringModelObject, fileName=doc.basename)
if len(schemaRefFileNames) > 1:
modelXbrl.error("EBA.1.5",
_('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'),
modelObject=modelDocument, numEntryPoints=len(schemaRefFileNames), entryPointNames=', '.join(sorted(schemaRefFileNames)))
### check entry point names appropriate for filing indicator (DPM DB?)
if len(schemaRefElts) != 1:
modelXbrl.error("EBA.2.3",
_('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'),
modelObject=schemaRefElts, entryPointCount=len(schemaRefElts))
# non-streaming EBA checks
if not getattr(modelXbrl, "isStreamingMode", False):
validateFacts(val, modelXbrl.facts)
# check sum of fact md5s (otherwise checked in streaming process)
xbrlFactsCheckVersion = None
expectedSumOfFactMd5s = None
for pi in modelDocument.xmlRootElement.getchildren():
if isinstance(pi, etree._ProcessingInstruction) and pi.target == "xbrl-facts-check":
_match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text)
if _match:
_matchGroups = _match.groups()
if len(_matchGroups) == 2:
if _matchGroups[0] == "version":
xbrlFactsCheckVersion = _matchGroups[1]
elif _matchGroups[0] == "sum-of-fact-md5s":
try:
expectedSumOfFactMd5s = Md5Sum(_matchGroups[1])
except ValueError:
modelXbrl.error("EIOPA:xbrlFactsCheckError",
_("Invalid sum-of-md5s %(sumOfMd5)s"),
modelObject=modelXbrl, sumOfMd5=_matchGroups[1])
if xbrlFactsCheckVersion and expectedSumOfFactMd5s:
sumOfFactMd5s = Md5Sum()
for f in modelXbrl.factsInInstance:
sumOfFactMd5s += f.md5sum
if sumOfFactMd5s != expectedSumOfFactMd5s:
modelXbrl.warning("EIOPA:xbrlFactsCheckWarning",
_("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"),
modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s)
else:
modelXbrl.info("info",
_("Successful XBRL facts sum of md5s."),
modelObject=modelXbrl)
if not val.filingIndicators:
modelXbrl.error("EBA.1.6",
_('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements'),
modelObject=modelDocument)
if val.numFilingIndicatorTuples > 1:
modelXbrl.info("EBA.1.6.2",
_('Multiple filing indicators tuples when not in streaming mode (info).'),
modelObject=modelXbrl.factsByQname[qnFIndicators])
if len(val.cntxDates) > 1:
modelXbrl.error("EBA.2.13",
_('Contexts must have the same date: %(dates)s.'),
# when streaming values are no longer available, but without streaming they can be logged
modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs),
dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True)
for _dt in val.cntxDates.keys()))
if val.unusedCntxIDs:
modelXbrl.warning("EBA.2.7",
_('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'),
modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts],
unusedContextIDs=", ".join(sorted(val.unusedCntxIDs)))
if len(val.cntxEntities) > 1:
modelXbrl.warning("EBA.2.9",
_('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.'),
modelObject=modelDocument, count=len(val.cntxEntities),
entities=", ".join(sorted(str(cntxEntity) for cntxEntity in val.cntxEntities)))
if val.unusedUnitIDs:
modelXbrl.warning("EBA.2.21",
_('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'),
modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units],
unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs)))
if len(val.currenciesUsed) > 1:
modelXbrl.error("EBA.3.1",
_("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"),
modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join(str(c) for c in val.currenciesUsed.keys()))
if val.prefixesUnused:
modelXbrl.warning("EBA.3.4",
_("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"),
modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused)))
for ns, prefixes in val.namespacePrefixesUsed.items():
nsDocs = modelXbrl.namespaceDocs.get(ns)
if nsDocs:
for nsDoc in nsDocs:
nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns)
if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None):
modelXbrl.warning("EBA.3.5",
_("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"),
modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None})))
modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0)
modelXbrl.modelManager.showStatus(None)
del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements
del val.utrValidator
def loadStandardTaxonomiesDict(self):
if self.selection:
self.standardTaxonomiesDict = defaultdict(set)
self.familyHrefs = defaultdict(set)
self.standardLocalHrefs = defaultdict(set)
self.standardAuthorities = set()
self.standardPrefixes = {}
if not self.standardTaxonomiesUrl:
return
basename = os.path.basename(self.standardTaxonomiesUrl)
self.modelManager.cntlr.showStatus(_("parsing {0}").format(basename))
try:
from arelle.FileSource import openXmlFileStream
for filepath in (self.standardTaxonomiesUrl,
os.path.join(self.modelManager.cntlr.configDir,"xbrlschemafiles.xml")):
xmldoc = etree.parse(filepath) # must open with file path for xinclude to know base of file
xmldoc.xinclude() # to include elements below root use xpointer(/*/*)
for erxlElt in xmldoc.iter(tag="Erxl"):
v = erxlElt.get("version")
if v and re.match(r"[0-9]+([.][0-9]+)*$", v):
vSplit = v.split('.') # at least 3 digits always!
self.version = tuple(int(n) for n in vSplit) + tuple(0 for n in range(3 - len(vSplit)))
break
for locElt in xmldoc.iter(tag="Loc"):
href = None
localHref = None
namespaceUri = None
prefix = None
attType = None
family = None
elements = None
version = None
for childElt in locElt.iterchildren():
ln = childElt.tag
value = childElt.text.strip()
if ln == "Href":
href = value
elif ln == "LocalHref":
localHref = value
elif ln == "Namespace":
namespaceUri = value
elif ln == "Prefix":
prefix = value
elif ln == "AttType":
attType = value
elif ln == "Family":
family = value
elif ln == "Elements":
elements = value
elif ln == "Version":
version = value
if href:
if namespaceUri and (attType == "SCH" or attType == "ENT"):
self.standardTaxonomiesDict[namespaceUri].add(href)
if localHref:
self.standardLocalHrefs[namespaceUri].add(localHref)
authority = UrlUtil.authority(namespaceUri)
self.standardAuthorities.add(authority)
if family == "BASE":
self.baseTaxonomyNamespaces.add(namespaceUri)
if prefix:
self.standardPrefixes[namespaceUri] = prefix
if href not in self.standardTaxonomiesDict:
self.standardTaxonomiesDict[href] = "Allowed" + attType
if family:
self.familyHrefs[family].add(ErxlLoc(family, version, href, attType, elements, namespaceUri))
elif attType == "SCH" and family == "BASE":
self.baseTaxonomyNamespaces.add(namespaceUri)
except (EnvironmentError,
etree.LxmlError) as err:
self.modelManager.cntlr.addToLog(_("Disclosure System \"%(name)s\" import %(importFile)s, error: %(error)s"),
messageCode="arelle:disclosureSystemImportError",
messageArgs={"error": str(err), "name": self.name, "importFile": basename},
level=logging.ERROR)
etree.clear_error_log()
def checkDTS(val, modelDocument, visited):
global targetNamespaceDatePattern, efmFilenamePattern, roleTypePattern, arcroleTypePattern, \
arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet, \
namespacesConflictPattern
if targetNamespaceDatePattern is None:
targetNamespaceDatePattern = re.compile(r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|"
r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|")
efmFilenamePattern = re.compile(r"^[a-z0-9][a-z0-9_\.\-]*(\.xsd|\.xml)$")
roleTypePattern = re.compile(r"^.*/role/[^/\s]+$")
arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$")
arcroleDefinitionPattern = re.compile(r"^.*[^\\s]+.*$") # at least one non-whitespace character
namePattern = re.compile("[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]") # u20ac=Euro, u00a3=pound sterling
linkroleDefinitionBalanceIncomeSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)",
re.IGNORECASE)
namespacesConflictPattern = re.compile(r"http://(xbrl\.us|fasb\.org|xbrl\.sec\.gov)/(dei|us-types|us-roles|rr)/([0-9]{4}-[0-9]{2}-[0-9]{2})$")
nonDomainItemNameProblemPattern = re.compile(
r"({0})|(FirstQuarter|SecondQuarter|ThirdQuarter|FourthQuarter|[1-4]Qtr|Qtr[1-4]|ytd|YTD|HalfYear)(?:$|[A-Z\W])"
.format(re.sub(r"\W", "", (val.entityRegistrantName or "").title())))
visited.append(modelDocument)
definesLabelLinkbase = False
for referencedDocument in modelDocument.referencesDocument.items():
#6.07.01 no includes
if referencedDocument[1] == "include":
val.modelXbrl.error(("EFM.6.07.01", "GFM.1.03.01", "SBR.NL.2.2.0.18"),
_("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"),
modelObject=modelDocument,
schema=os.path.basename(modelDocument.uri),
include=os.path.basename(referencedDocument[0].uri))
if referencedDocument[0] not in visited:
checkDTS(val, referencedDocument[0], visited)
if val.disclosureSystem.standardTaxonomiesDict is None:
pass
if val.validateEFM:
if modelDocument.uri in val.disclosureSystem.standardTaxonomiesDict:
if modelDocument.targetNamespace:
# check for duplicates of us-types, dei, and rr taxonomies
match = namespacesConflictPattern.match(modelDocument.targetNamespace)
if match is not None:
val.standardNamespaceConflicts[match.group(2)].add(modelDocument)
else:
if len(modelDocument.basename) > 32:
val.modelXbrl.error("EFM.5.01.01.tooManyCharacters",
_("Document file name %(filename)s must not exceed 32 characters."),
modelObject=modelDocument, filename=modelDocument.basename)
if not efmFilenamePattern.match(modelDocument.basename):
val.modelXbrl.error("EFM.5.01.01",
_("Document file name %(filename)s must start with a-z or 0-9, contain lower case letters, ., -, _, and end with .xsd or .xml."),
modelObject=modelDocument, filename=modelDocument.basename)
if (modelDocument.type == ModelDocument.Type.SCHEMA and
modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and
modelDocument.uri.startswith(val.modelXbrl.uriDir)):
# check schema contents types
if val.validateSBRNL:
definesLinkroles = False
definesArcroles = False
definesLinkParts = False
definesAbstractItems = False
definesNonabstractItems = False
definesConcepts = False
definesTuples = False
definesPresentationTuples = False
definesSpecificationTuples = False
definesTypes = False
definesEnumerations = False
definesDimensions = False
definesDomains = False
definesHypercubes = False
# 6.7.3 check namespace for standard authority
targetNamespaceAuthority = UrlUtil.authority(modelDocument.targetNamespace)
if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities:
val.modelXbrl.error(("EFM.6.07.03", "GFM.1.03.03"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s is a disallowed authority"),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, targetNamespaceAuthority=targetNamespaceAuthority)
# 6.7.4 check namespace format
if modelDocument.targetNamespace is None or not modelDocument.targetNamespace.startswith("http://"):
match = None
elif val.validateEFMorGFM:
targetNamespaceDate = modelDocument.targetNamespace[len(targetNamespaceAuthority):]
match = targetNamespaceDatePattern.match(targetNamespaceDate)
else:
match = None
if match is not None:
try:
if match.lastindex == 3:
date = datetime.date(int(match.group(1)),int(match.group(2)),int(match.group(3)))
elif match.lastindex == 6:
date = datetime.date(int(match.group(4)),int(match.group(5)),int(match.group(6)))
else:
match = None
except ValueError:
match = None
if match is None:
val.modelXbrl.error(("EFM.6.07.04", "GFM.1.03.04"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s must have format http://{authority}/{versionDate}"),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
elif val.fileNameDate and date > val.fileNameDate:
val.modelXbrl.info(("EFM.6.07.06", "GFM.1.03.06"),
_("Warning: Taxonomy schema %(schema)s namespace %(targetNamespace)s has date later than document name date %(docNameDate)s"),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace,
docNameDate=val.fileNameDate)
if modelDocument.targetNamespace is not None:
# 6.7.5 check prefix for _
authority = UrlUtil.authority(modelDocument.targetNamespace)
if not re.match(r"(http://|https://|ftp://|urn:)\w+",authority):
val.modelXbrl.error(("EFM.6.07.05", "GFM.1.03.05"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s must be a valid URL with a valid authority for the namespace."),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement,modelDocument.targetNamespace)
if not prefix:
val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s missing prefix for the namespace."),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
elif "_" in prefix:
val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s prefix %(prefix)s must not have an '_'"),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, prefix=prefix)
if val.validateSBRNL:
genrlSpeclRelSet = val.modelXbrl.relationshipSet(XbrlConst.generalSpecial)
for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"):
if isinstance(modelConcept,ModelConcept):
# 6.7.16 name not duplicated in standard taxonomies
name = modelConcept.get("name")
if name is None:
name = ""
if modelConcept.get("ref") is not None:
continue # don't validate ref's here
for c in val.modelXbrl.nameConcepts.get(name, []):
if c.modelDocument != modelDocument:
if (val.validateEFMorGFM and
not c.modelDocument.uri.startswith(val.modelXbrl.uriDir)):
val.modelXbrl.error(("EFM.6.07.16", "GFM.1.03.18"),
_("Concept %(concept)s is also defined in standard taxonomy schema schema %(standardSchema)s"),
modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri))
elif val.validateSBRNL:
if not (genrlSpeclRelSet.isRelated(modelConcept, "child", c) or genrlSpeclRelSet.isRelated(c, "child", modelConcept)):
val.modelXbrl.error("SBR.NL.2.2.2.02",
_("Concept %(concept)s is also defined in standard taxonomy schema %(standardSchema)s without a general-special relationship"),
modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri))
''' removed RH 2011-12-23 corresponding set up of table in ValidateFiling
if val.validateSBRNL and name in val.nameWordsTable:
if not any( any( genrlSpeclRelSet.isRelated(c, "child", modelConcept)
for c in val.modelXbrl.nameConcepts.get(partialWordName, []))
for partialWordName in val.nameWordsTable[name]):
val.modelXbrl.error("SBR.NL.2.3.2.01",
_("Concept %(specialName)s is appears to be missing a general-special relationship to %(generalNames)s"),
modelObject=c, specialName=modelConcept.qname, generalNames=', or to '.join(val.nameWordsTable[name]))
'''
# 6.7.17 id properly formed
id = modelConcept.id
requiredId = (prefix if prefix is not None else "") + "_" + name
if val.validateEFMorGFM and id != requiredId:
val.modelXbrl.error(("EFM.6.07.17", "GFM.1.03.19"),
_("Concept %(concept)s id %(id)s should be $(requiredId)s"),
modelObject=modelConcept, concept=modelConcept.qname, id=id, requiredId=requiredId)
# 6.7.18 nillable is true
nillable = modelConcept.get("nillable")
if nillable != "true":
val.modelXbrl.error(("EFM.6.07.18", "GFM.1.03.20"),
_("Taxonomy schema %(schema)s element %(concept)s nillable %(nillable)s should be 'true'"),
modelObject=modelConcept, schema=os.path.basename(modelDocument.uri),
concept=name, nillable=nillable)
# 6.7.19 not tuple
if modelConcept.isTuple:
if val.validateEFMorGFM:
val.modelXbrl.error(("EFM.6.07.19", "GFM.1.03.21"),
_("Concept %(concept)s is a tuple"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.20 no typed domain ref
if modelConcept.isTypedDimension:
val.modelXbrl.error(("EFM.6.07.20", "GFM.1.03.22"),
_("Concept %(concept)s has typedDomainRef %(typedDomainRef)s"),
modelObject=modelConcept, concept=modelConcept.qname,
typedDomainRef=modelConcept.typedDomainElement.qname if modelConcept.typedDomainElement is not None else modelConcept.typedDomainRef)
# 6.7.21 abstract must be duration
isDuration = modelConcept.periodType == "duration"
if modelConcept.abstract == "true" and not isDuration:
val.modelXbrl.error(("EFM.6.07.21", "GFM.1.03.23"),
_("Taxonomy schema %(schema)s element %(concept)s is abstract but period type is not duration"),
modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=name)
# 6.7.22 abstract must be stringItemType
''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08
if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType:
val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"),
_("Concept %(concept)s is abstract but type is not xbrli:stringItemType"),
modelObject=modelConcept, concept=modelConcept.qname)
'''
substititutionGroupQname = modelConcept.substitutionGroupQname
# 6.7.23 Axis must be subs group dimension
if name.endswith("Axis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem):
val.modelXbrl.error(("EFM.6.07.23", "GFM.1.03.25"),
_("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.24 Table must be subs group hypercube
if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem):
val.modelXbrl.error(("EFM.6.07.24", "GFM.1.03.26"),
_("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"),
modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname)
# 6.7.25 if neither hypercube or dimension, substitution group must be item
if substititutionGroupQname not in (None,
XbrlConst.qnXbrldtDimensionItem,
XbrlConst.qnXbrldtHypercubeItem,
XbrlConst.qnXbrliItem):
val.modelXbrl.error(("EFM.6.07.25", "GFM.1.03.27"),
_("Concept %(concept)s has disallowed substitution group %(substitutionGroup)s"),
modelObject=modelConcept, concept=modelConcept.qname,
substitutionGroup=modelConcept.substitutionGroupQname)
# 6.7.26 Table must be subs group hypercube
if name.endswith("LineItems") and modelConcept.abstract != "true":
val.modelXbrl.error(("EFM.6.07.26", "GFM.1.03.28"),
_("Concept %(concept)s is a LineItems but not abstract"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.27 type domainMember must end with Domain or Member
conceptType = modelConcept.type
isDomainItemType = conceptType is not None and conceptType.isDomainItemType
endsWithDomainOrMember = name.endswith("Domain") or name.endswith("Member")
if isDomainItemType != endsWithDomainOrMember:
val.modelXbrl.error(("EFM.6.07.27", "GFM.1.03.29"),
_("Concept %(concept)s must end with Domain or Member for type of domainItemType"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.28 domainItemType must be duration
if isDomainItemType and not isDuration:
val.modelXbrl.error(("EFM.6.07.28", "GFM.1.03.30"),
_("Concept %(concept)s is a domainItemType and must be periodType duration"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.8.5 semantic check, check LC3 name
if not name[0].isupper():
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.firstLetter", "GFM.2.03.05.firstLetter"),
_("Concept %(concept)s name must start with a capital letter"),
modelObject=modelConcept, concept=modelConcept.qname)
if namePattern.search(name):
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.disallowedCharacter", "GFM.2.03.05.disallowedCharacter"),
_("Concept %(concept)s has disallowed name character"),
modelObject=modelConcept, concept=modelConcept.qname)
if len(name) > 200:
val.modelXbrl.log("ERROR-SEMANTIC", "EFM.6.08.05.nameLength",
_("Concept %(concept)s name length %(namelength)s exceeds 200 characters"),
modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name))
if val.validateEFM:
label = modelConcept.label(lang="en-US", fallbackToQname=False)
if label:
# allow Joe's Bar, N.A. to be JoesBarNA -- remove ', allow A. as not article "a"
lc3name = ''.join(re.sub(r"['.-]", "", (w[0] or w[2] or w[3] or w[4])).title()
for w in re.findall(r"((\w+')+\w+)|(A[.-])|([.-]A(?=\W|$))|(\w+)", label) # EFM implies this should allow - and . re.findall(r"[\w\-\.]+", label)
if w[4].lower() not in ("the", "a", "an"))
if not(name == lc3name or
(name and lc3name and lc3name[0].isdigit() and name[1:] == lc3name and (name[0].isalpha() or name[0] == '_'))):
val.modelXbrl.log("WARNING-SEMANTIC", "EFM.6.08.05.LC3",
_("Concept %(concept)s should match expected LC3 composition %(lc3name)s"),
modelObject=modelConcept, concept=modelConcept.qname, lc3name=lc3name)
if conceptType is not None:
# 6.8.6 semantic check
if not isDomainItemType and conceptType.qname != XbrlConst.qnXbrliDurationItemType:
nameProblems = nonDomainItemNameProblemPattern.findall(name)
if any(any(t) for t in nameProblems): # list of tuples with possibly nonempty strings
val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.08.06", "GFM.2.03.06"),
_("Concept %(concept)s should not contain company or period information, found: %(matches)s"),
modelObject=modelConcept, concept=modelConcept.qname,
matches=", ".join(''.join(t) for t in nameProblems))
if conceptType.qname == XbrlConst.qnXbrliMonetaryItemType:
if not modelConcept.balance:
# 6.8.11 may not appear on a income or balance statement
if any(linkroleDefinitionBalanceIncomeSheet.match(roleType.definition)
for rel in val.modelXbrl.relationshipSet(XbrlConst.parentChild).toModelObject(modelConcept)
for roleType in val.modelXbrl.roleTypes.get(rel.linkrole,())):
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.11", "GFM.2.03.11"),
_("Concept %(concept)s must have a balance because it appears in a statement of income or balance sheet"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.11.5 semantic check, must have a documentation label
stdLabel = modelConcept.label(lang="en-US", fallbackToQname=False)
defLabel = modelConcept.label(preferredLabel=XbrlConst.documentationLabel, lang="en-US", fallbackToQname=False)
if not defLabel or ( # want different words than std label
stdLabel and re.findall(r"\w+", stdLabel) == re.findall(r"\w+", defLabel)):
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.11.05", "GFM.2.04.04"),
_("Concept %(concept)s is monetary without a balance and must have a documentation label that disambiguates its sign"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.8.16 semantic check
if conceptType.qname == XbrlConst.qnXbrliDateItemType and modelConcept.periodType != "duration":
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.16", "GFM.2.03.16"),
_("Concept %(concept)s of type xbrli:dateItemType must have periodType duration"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.8.17 semantic check
if conceptType.qname == XbrlConst.qnXbrliStringItemType and modelConcept.periodType != "duration":
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.17", "GFM.2.03.17"),
_("Concept %(concept)s of type xbrli:stringItemType must have periodType duration"),
modelObject=modelConcept, concept=modelConcept.qname)
if val.validateSBRNL:
if modelConcept.isTuple:
if modelConcept.substitutionGroupQname.localName == "presentationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year
definesPresentationTuples = True
elif modelConcept.substitutionGroupQname.localName == "specificationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year
definesSpecificationTuples = True
else:
definesTuples = True
definesConcepts = True
if modelConcept.isAbstract:
val.modelXbrl.error("SBR.NL.2.2.2.03",
_("Concept %(concept)s is an abstract tuple"),
modelObject=modelConcept, concept=modelConcept.qname)
if tupleCycle(val,modelConcept):
val.modelXbrl.error("SBR.NL.2.2.2.07",
_("Tuple %(concept)s has a tuple cycle"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.get("nillable") != "false" and modelConcept.isRoot:
val.modelXbrl.error("SBR.NL.2.2.2.17", #don't want default, just what was really there
_("Tuple %(concept)s must have nillable='false'"),
modelObject=modelConcept, concept=modelConcept.qname)
elif modelConcept.isItem:
definesConcepts = True
if modelConcept.abstract == "true":
if modelConcept.isRoot:
if modelConcept.get("nillable") != "false": #don't want default, just what was really there
val.modelXbrl.error("SBR.NL.2.2.2.16",
_("Abstract root concept %(concept)s must have nillable='false'"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.typeQname != XbrlConst.qnXbrliStringItemType:
val.modelXbrl.error("SBR.NL.2.2.2.21",
_("Abstract root concept %(concept)s must have type='xbrli:stringItemType'"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.balance:
val.modelXbrl.error("SBR.NL.2.2.2.22",
_("Abstract concept %(concept)s must not have a balance attribute"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.isHypercubeItem:
definesHypercubes = True
elif modelConcept.isDimensionItem:
definesDimensions = True
elif substititutionGroupQname and substititutionGroupQname.localName in ("domainItem","domainMemberItem"):
definesDomains = True
elif modelConcept.isItem:
definesAbstractItems = True
else: # not abstract
if modelConcept.isItem:
definesNonabstractItems = True
if not (modelConcept.label(preferredLabel=XbrlConst.documentationLabel,fallbackToQname=False,lang="nl") or
val.modelXbrl.relationshipSet(XbrlConst.conceptReference).fromModelObject(c) or
modelConcept.genLabel(role=XbrlConst.genDocumentationLabel,lang="nl") or
val.modelXbrl.relationshipSet(XbrlConst.elementReference).fromModelObject(c)):
val.modelXbrl.error("SBR.NL.2.2.2.28",
_("Concept %(concept)s must have a documentation label or reference"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.balance and not modelConcept.instanceOfType(XbrlConst.qnXbrliMonetaryItemType):
val.modelXbrl.error("SBR.NL.2.2.2.24",
_("Non-monetary concept %(concept)s must not have a balance attribute"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.isLinkPart:
definesLinkParts = True
val.modelXbrl.error("SBR.NL.2.2.5.01",
_("Link:part concept %(concept)s is not allowed"),
modelObject=modelConcept, concept=modelConcept.qname)
if not modelConcept.genLabel(fallbackToQname=False,lang="nl"):
val.modelXbrl.error("SBR.NL.2.2.5.02",
_("Link part definition %(concept)s must have a generic label in language 'nl'"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.8 check for embedded linkbase
for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
if isinstance(e,ModelObject):
val.modelXbrl.error(("EFM.6.07.08", "GFM.1.03.08"),
_("Taxonomy schema %(schema)s contains an embedded linkbase"),
modelObject=e, schema=os.path.basename(modelDocument.uri))
break
requiredUsedOns = {XbrlConst.qnLinkPresentationLink,
XbrlConst.qnLinkCalculationLink,
XbrlConst.qnLinkDefinitionLink}
standardUsedOns = {XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference, XbrlConst.qnLinkFootnote,
XbrlConst.qnLinkDefinitionArc, XbrlConst.qnLinkCalculationArc, XbrlConst.qnLinkPresentationArc,
XbrlConst.qnLinkLabelArc, XbrlConst.qnLinkReferenceArc
# XbrlConst.qnLinkFootnoteArc note: not in EFM table for 6.8.3 and causes tests 6.5.30 to fail
}
# 6.7.9 role types authority
for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}roleType"):
if isinstance(e,ModelObject):
roleURI = e.get("roleURI")
if targetNamespaceAuthority != UrlUtil.authority(roleURI):
val.modelXbrl.error(("EFM.6.07.09", "GFM.1.03.09"),
_("RoleType %(roleType)s does not match authority %(targetNamespaceAuthority)s"),
modelObject=e, roleType=roleURI, targetNamespaceAuthority=targetNamespaceAuthority)
# 6.7.9 end with .../role/lc3 name
if not roleTypePattern.match(roleURI):
val.modelXbrl.warning(("EFM.6.07.09.roleEnding", "GFM.1.03.09"),
"RoleType %(roleType)s should end with /role/{LC3name}",
modelObject=e, roleType=roleURI)
# 6.7.10 only one role type declaration in DTS
modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI)
if modelRoleTypes is not None:
modelRoleType = modelRoleTypes[0]
definition = modelRoleType.definitionNotStripped
usedOns = modelRoleType.usedOns
if len(modelRoleTypes) > 1:
val.modelXbrl.error(("EFM.6.07.10", "GFM.1.03.10"),
_("RoleType %(roleType)s is defined in multiple taxonomies"),
modelObject=modelRoleTypes, roleType=roleURI, numberOfDeclarations=len(modelRoleTypes))
elif len(modelRoleTypes) == 1:
# 6.7.11 used on's for pre, cal, def if any has a used on
if not usedOns.isdisjoint(requiredUsedOns) and len(requiredUsedOns - usedOns) > 0:
val.modelXbrl.error(("EFM.6.07.11", "GFM.1.03.11"),
_("RoleType %(roleType)s missing used on %(usedOn)s"),
modelObject=e, roleType=roleURI, usedOn=requiredUsedOns - usedOns)
# 6.7.12 definition match pattern
if (val.disclosureSystem.roleDefinitionPattern is not None and
(definition is None or not val.disclosureSystem.roleDefinitionPattern.match(definition))):
val.modelXbrl.error(("EFM.6.07.12", "GFM.1.03.12-14"),
_("RoleType %(roleType)s definition \"%(definition)s\" must match {Sortcode} - {Type} - {Title}"),
modelObject=e, roleType=roleURI, definition=definition)
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("RoleType %(roleuri)s is defined using role types already defined by standard roles for: %(qnames)s"),
modelObject=e, roleuri=roleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns))
if val.validateSBRNL:
if usedOns & XbrlConst.standardExtLinkQnames or XbrlConst.qnGenLink in usedOns:
definesLinkroles = True
if not e.genLabel():
val.modelXbrl.error("SBR.NL.2.2.3.03",
_("Link RoleType %(roleType)s missing a generic standard label"),
modelObject=e, roleType=roleURI)
nlLabel = e.genLabel(lang="nl")
if definition != nlLabel:
val.modelXbrl.error("SBR.NL.2.2.3.04",
_("Link RoleType %(roleType)s definition does not match NL standard generic label, \ndefinition: %(definition)s \nNL label: %(label)s"),
modelObject=e, roleType=roleURI, definition=definition, label=nlLabel)
if definition and (definition[0].isspace() or definition[-1].isspace()):
val.modelXbrl.error("SBR.NL.2.2.3.07",
_('Link RoleType %(roleType)s definition has leading or trailing spaces: "%(definition)s"'),
modelObject=e, roleType=roleURI, definition=definition)
# 6.7.13 arcrole types authority
for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}arcroleType"):
if isinstance(e,ModelObject):
arcroleURI = e.get("arcroleURI")
if targetNamespaceAuthority != UrlUtil.authority(arcroleURI):
val.modelXbrl.error(("EFM.6.07.13", "GFM.1.03.15"),
_("ArcroleType %(arcroleType)s does not match authority %(targetNamespaceAuthority)s"),
modelObject=e, arcroleType=arcroleURI, targetNamespaceAuthority=targetNamespaceAuthority)
# 6.7.13 end with .../arcrole/lc3 name
if not arcroleTypePattern.match(arcroleURI):
val.modelXbrl.warning(("EFM.6.07.13.arcroleEnding", "GFM.1.03.15"),
_("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"),
modelObject=e, arcroleType=arcroleURI)
# 6.7.14 only one arcrole type declaration in DTS
modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI]
if len(modelRoleTypes) > 1:
val.modelXbrl.error(("EFM.6.07.14", "GFM.1.03.16"),
_("ArcroleType %(arcroleType)s is defined in multiple taxonomies"),
modelObject=e, arcroleType=arcroleURI, numberOfDeclarations=len(modelRoleTypes) )
# 6.7.15 definition match pattern
definition = modelRoleTypes[0].definition
if definition is None or not arcroleDefinitionPattern.match(definition):
val.modelXbrl.error(("EFM.6.07.15", "GFM.1.03.17"),
_("ArcroleType %(arcroleType)s definition must be non-empty"),
modelObject=e, arcroleType=arcroleURI)
# semantic checks
usedOns = modelRoleTypes[0].usedOns
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("ArcroleType %(arcroleuri)s is defined using role types already defined by standard arcroles for: %(qnames)s"),
modelObject=e, arcroleuri=arcroleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns))
if val.validateSBRNL:
definesArcroles = True
val.modelXbrl.error("SBR.NL.2.2.4.01",
_("Arcrole type definition is not allowed: %(arcroleURI)s"),
modelObject=e, arcroleURI=arcroleURI)
if val.validateSBRNL:
for appinfoElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}appinfo"):
for nonLinkElt in appinfoElt.iterdescendants():
if isinstance(nonLinkElt, ModelObject) and nonLinkElt.namespaceURI != XbrlConst.link:
val.modelXbrl.error("SBR.NL.2.2.11.05",
_("Appinfo contains disallowed non-link element %(element)s"),
modelObject=nonLinkElt, element=nonLinkElt.qname)
for cplxTypeElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexType"):
choiceElt = cplxTypeElt.find("{http://www.w3.org/2001/XMLSchema}choice")
if choiceElt is not None:
val.modelXbrl.error("SBR.NL.2.2.11.09",
_("ComplexType contains disallowed xs:choice element"),
modelObject=choiceElt)
for cplxContentElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexContent"):
if XmlUtil.descendantAttr(cplxContentElt, "http://www.w3.org/2001/XMLSchema", ("extension","restriction"), "base") != "sbr:placeholder":
val.modelXbrl.error("SBR.NL.2.2.11.10",
_("ComplexContent is disallowed"),
modelObject=cplxContentElt)
definesTypes = (modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}complexType") is not None or
modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}simpleType") is not None)
if (definesLinkroles + definesArcroles + definesLinkParts +
definesAbstractItems + definesNonabstractItems +
definesTuples + definesPresentationTuples + definesSpecificationTuples + definesTypes +
definesEnumerations + definesDimensions + definesDomains +
definesHypercubes) != 1:
schemaContents = []
if definesLinkroles: schemaContents.append(_("linkroles"))
if definesArcroles: schemaContents.append(_("arcroles"))
if definesLinkParts: schemaContents.append(_("link parts"))
if definesAbstractItems: schemaContents.append(_("abstract items"))
if definesNonabstractItems: schemaContents.append(_("nonabstract items"))
if definesTuples: schemaContents.append(_("tuples"))
if definesPresentationTuples: schemaContents.append(_("sbrPresentationTuples"))
if definesSpecificationTuples: schemaContents.append(_("sbrSpecificationTuples"))
if definesTypes: schemaContents.append(_("types"))
if definesEnumerations: schemaContents.append(_("enumerations"))
if definesDimensions: schemaContents.append(_("dimensions"))
if definesDomains: schemaContents.append(_("domains"))
if definesHypercubes: schemaContents.append(_("hypercubes"))
if schemaContents:
val.modelXbrl.error("SBR.NL.2.2.1.01",
_("Taxonomy schema may only define one of these: %(contents)s"),
modelObject=modelDocument, contents=', '.join(schemaContents))
elif not any(refDoc.inDTS and refDoc.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces
for refDoc in modelDocument.referencesDocument.keys()): # no linkbase ref or includes
val.modelXbrl.error("SBR.NL.2.2.1.01",
_("Taxonomy schema must be a DTS entrypoint OR define linkroles OR arcroles OR link:parts OR context fragments OR abstract items OR tuples OR non-abstract elements OR types OR enumerations OR dimensions OR domains OR hypercubes"),
modelObject=modelDocument)
if definesConcepts ^ any( # xor so either concepts and no label LB or no concepts and has label LB
(refDoc.type == ModelDocument.Type.LINKBASE and
XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "labelLink") is not None)
for refDoc in modelDocument.referencesDocument.keys()): # no label linkbase
val.modelXbrl.error("SBR.NL.2.2.1.02",
_("A schema that defines concepts MUST have a linked 2.1 label linkbase"),
modelObject=modelDocument)
if (definesNonabstractItems or definesTuples) and not any( # was xor but changed to and not per RH 1/11/12
(refDoc.type == ModelDocument.Type.LINKBASE and
(XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "referenceLink") is not None or
XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/documentation" ) is not None))
for refDoc in modelDocument.referencesDocument.keys()):
val.modelXbrl.error("SBR.NL.2.2.1.03",
_("A schema that defines non-abstract items MUST have a linked (2.1) reference linkbase AND/OR a label linkbase with @xlink:role=documentation"),
modelObject=modelDocument)
if val.fileNameBasePart:
#6.3.3 filename check
expectedFilename = "{0}-{1}.xsd".format(val.fileNameBasePart, val.fileNameDatePart)
if modelDocument.basename != expectedFilename and not ( # skip if an edgar testcase
re.match("e[0-9]{8}(gd|ng)", val.fileNameBasePart) and re.match("e.*-[0-9]{8}.*", modelDocument.basename)):
val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"),
_('Invalid schema file name: %(filename)s, expected %(expectedFilename)s'),
modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename)
elif modelDocument.type == ModelDocument.Type.LINKBASE:
# if it is part of the submission (in same directory) check name
if modelDocument.filepath.startswith(val.modelXbrl.modelDocument.filepathdir):
#6.3.3 filename check
extLinkElt = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended")
if val.fileNameBasePart:
if extLinkElt is not None:
if extLinkElt.localName in extLinkEltFileNameEnding:
expectedFilename = "{0}-{1}_{2}.xml".format(val.fileNameBasePart, val.fileNameDatePart,
extLinkEltFileNameEnding[extLinkElt.localName])
if modelDocument.basename != expectedFilename and not ( # skip if an edgar testcase
re.match("e[0-9]{8}(gd|ng)", val.fileNameBasePart) and re.match("e.*-[0-9]{8}.*", modelDocument.basename)):
val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"),
_('Invalid linkbase file name: %(filename)s, expected %(expectedFilename)s'),
modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename)
else: # no ext link element
val.modelXbrl.error(("EFM.6.03.03.noLinkElement", "GFM.1.01.01.noLinkElement"),
_('Invalid linkbase file name: %(filename)s, has no extended link element, cannot determine link type.'),
modelObject=modelDocument, filename=modelDocument.basename)
visited.remove(modelDocument)
def loadStandardTaxonomiesDict(self):
if self.selection:
self.standardTaxonomiesDict = defaultdict(set)
self.familyHrefs = defaultdict(set)
self.standardLocalHrefs = defaultdict(set)
self.standardAuthorities = set()
self.standardPrefixes = {}
if not self.standardTaxonomiesUrl:
return
basename = os.path.basename(self.standardTaxonomiesUrl)
self.modelManager.cntlr.showStatus(
_("parsing {0}").format(basename))
try:
from arelle.FileSource import openXmlFileStream
for filepath in (self.standardTaxonomiesUrl,
os.path.join(
self.modelManager.cntlr.configDir,
"xbrlschemafiles.xml")):
xmldoc = etree.parse(
filepath
) # must open with file path for xinclude to know base of file
xmldoc.xinclude(
) # to include elements below root use xpointer(/*/*)
for erxlElt in xmldoc.iter(tag="Erxl"):
v = erxlElt.get("version")
if v and re.match(r"[0-9]+([.][0-9]+)*$", v):
vSplit = v.split('.') # at least 3 digits always!
self.version = tuple(
int(n) for n in vSplit) + tuple(
0 for n in range(3 - len(vSplit)))
break
for locElt in xmldoc.iter(tag="Loc"):
href = None
localHref = None
namespaceUri = None
prefix = None
attType = None
family = None
elements = None
version = None
for childElt in locElt.iterchildren():
ln = childElt.tag
value = childElt.text.strip()
if ln == "Href":
href = value
elif ln == "LocalHref":
localHref = value
elif ln == "Namespace":
namespaceUri = value
elif ln == "Prefix":
prefix = value
elif ln == "AttType":
attType = value
elif ln == "Family":
family = value
elif ln == "Elements":
elements = value
elif ln == "Version":
version = value
if href:
if namespaceUri and (attType == "SCH"
or attType == "ENT"):
self.standardTaxonomiesDict[namespaceUri].add(
href)
if localHref:
self.standardLocalHrefs[namespaceUri].add(
localHref)
authority = UrlUtil.authority(namespaceUri)
self.standardAuthorities.add(authority)
if family == "BASE":
self.baseTaxonomyNamespaces.add(
namespaceUri)
if prefix:
self.standardPrefixes[
namespaceUri] = prefix
if href not in self.standardTaxonomiesDict:
self.standardTaxonomiesDict[
href] = "Allowed" + attType
if family:
self.familyHrefs[family].add(
ErxlLoc(family, version, href, attType,
elements, namespaceUri))
elif attType == "SCH" and family == "BASE":
self.baseTaxonomyNamespaces.add(namespaceUri)
except (EnvironmentError, etree.LxmlError) as err:
self.modelManager.cntlr.addToLog(
_("Disclosure System \"%(name)s\" import %(importFile)s, error: %(error)s"
),
messageCode="arelle:disclosureSystemImportError",
messageArgs={
"error": str(err),
"name": self.name,
"importFile": basename
},
level=logging.ERROR)
etree.clear_error_log()
def checkDTS(val, modelDocument, visited):
global targetNamespaceDatePattern, roleTypePattern, arcroleTypePattern, arcroleDefinitionPattern
if targetNamespaceDatePattern is None:
targetNamespaceDatePattern = re.compile(r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|"
r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|")
roleTypePattern = re.compile(r".*/role/[^/]+")
arcroleTypePattern = re.compile(r".*/arcrole/[^/]+")
arcroleDefinitionPattern = re.compile(r"^.*[^\\s]+.*$") # at least one non-whitespace character
visited.append(modelDocument)
definesLabelLinkbase = False
for referencedDocument in modelDocument.referencesDocument.items():
#6.07.01 no includes
if referencedDocument[1] == "include":
val.modelXbrl.error(("EFM.6.07.01", "GFM.1.03.01", "SBR.NL.2.2.0.18"),
_("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"),
modelObject=modelDocument,
schema=os.path.basename(modelDocument.uri),
include=os.path.basename(referencedDocument[0].uri))
if referencedDocument[0] not in visited:
checkDTS(val, referencedDocument[0], visited)
if val.disclosureSystem.standardTaxonomiesDict is None:
pass
if (modelDocument.type == ModelDocument.Type.SCHEMA and
modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and
modelDocument.uri.startswith(val.modelXbrl.uriDir)):
# check schema contents types
if val.validateSBRNL:
definesLinkroles = False
definesArcroles = False
definesLinkParts = False
definesAbstractItems = False
definesNonabstractItems = False
definesConcepts = False
definesTuples = False
definesPresentationTuples = False
definesSpecificationTuples = False
definesTypes = False
definesEnumerations = False
definesDimensions = False
definesDomains = False
definesHypercubes = False
# 6.7.3 check namespace for standard authority
targetNamespaceAuthority = UrlUtil.authority(modelDocument.targetNamespace)
if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities:
val.modelXbrl.error(("EFM.6.07.03", "GFM.1.03.03"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s is a disallowed authority"),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
# 6.7.4 check namespace format
if modelDocument.targetNamespace is None:
match = None
elif val.validateEFMorGFM:
targetNamespaceDate = modelDocument.targetNamespace[len(targetNamespaceAuthority):]
match = targetNamespaceDatePattern.match(targetNamespaceDate)
else:
match = None
if match is not None:
try:
if match.lastindex == 3:
datetime.date(int(match.group(1)),int(match.group(2)),int(match.group(3)))
elif match.lastindex == 6:
datetime.date(int(match.group(4)),int(match.group(5)),int(match.group(6)))
else:
match = None
except ValueError:
match = None
if match is None:
val.modelXbrl.error(("EFM.6.07.04", "GFM.1.03.04"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s must have format http://{authority}/{versionDate}"),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
if modelDocument.targetNamespace is not None:
# 6.7.5 check prefix for _
prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement,modelDocument.targetNamespace)
if prefix and "_" in prefix:
val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s prefix %(prefix)s must not have an '_'"),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, prefix=prefix)
if val.validateSBRNL:
genrlSpeclRelSet = val.modelXbrl.relationshipSet(XbrlConst.generalSpecial)
for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"):
if isinstance(modelConcept,ModelConcept):
# 6.7.16 name not duplicated in standard taxonomies
name = modelConcept.get("name")
if name is None:
name = ""
if modelConcept.get("ref") is not None:
continue # don't validate ref's here
for c in val.modelXbrl.nameConcepts.get(name, []):
if c.modelDocument != modelDocument:
if (val.validateEFMorGFM and
not c.modelDocument.uri.startswith(val.modelXbrl.uriDir)):
val.modelXbrl.error(("EFM.6.07.16", "GFM.1.03.18"),
_("Concept %(concept)s is also defined in standard taxonomy schema schema %(standardSchema)s"),
modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri))
elif val.validateSBRNL:
if not (genrlSpeclRelSet.isRelated(modelConcept, "child", c) or genrlSpeclRelSet.isRelated(c, "child", modelConcept)):
val.modelXbrl.error("SBR.NL.2.2.2.02",
_("Concept %(concept)s is also defined in standard taxonomy schema %(standardSchema)s without a general-special relationship"),
modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri))
''' removed RH 2011-12-23 corresponding set up of table in ValidateFiling
if val.validateSBRNL and name in val.nameWordsTable:
if not any( any( genrlSpeclRelSet.isRelated(c, "child", modelConcept)
for c in val.modelXbrl.nameConcepts.get(partialWordName, []))
for partialWordName in val.nameWordsTable[name]):
val.modelXbrl.error("SBR.NL.2.3.2.01",
_("Concept %(specialName)s is appears to be missing a general-special relationship to %(generalNames)s"),
modelObject=c, specialName=modelConcept.qname, generalNames=', or to '.join(val.nameWordsTable[name]))
'''
# 6.7.17 id properly formed
id = modelConcept.id
requiredId = (prefix if prefix is not None else "") + "_" + name
if val.validateEFMorGFM and id != requiredId:
val.modelXbrl.error(("EFM.6.07.17", "GFM.1.03.19"),
_("Concept %(concept)s id %(id)s should be $(requiredId)s"),
modelObject=modelConcept, concept=modelConcept.qname, id=id, requiredId=requiredId)
# 6.7.18 nillable is true
nillable = modelConcept.get("nillable")
if nillable != "true":
val.modelXbrl.error(("EFM.6.07.18", "GFM.1.03.20"),
_("Taxonomy schema %(schema)s element %(concept)s nillable %(nillable)s should be 'true'"),
modelObject=modelConcept, schema=os.path.basename(modelDocument.uri),
concept=name, nillable=nillable)
# 6.7.19 not tuple
if modelConcept.isTuple:
if val.validateEFMorGFM:
val.modelXbrl.error(("EFM.6.07.19", "GFM.1.03.21"),
_("Concept %(concept)s is a tuple"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.20 no typed domain ref
if modelConcept.isTypedDimension:
val.modelXbrl.error(("EFM.6.07.20", "GFM.1.03.22"),
_("Concept %(concept)s has typedDomainRef %(typedDomainRef)s"),
modelObject=modelConcept, concept=modelConcept.qname,
typedDomainRef=modelConcept.typedDomainElement.qname if modelConcept.typedDomainElement is not None else modelConcept.typedDomainRef)
# 6.7.21 abstract must be duration
isDuration = modelConcept.periodType == "duration"
if modelConcept.abstract == "true" and not isDuration:
val.modelXbrl.error(("EFM.6.07.21", "GFM.1.03.23"),
_("Taxonomy schema %(schema)s element %(concept)s is abstract but period type is not duration"),
modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=name)
# 6.7.22 abstract must be stringItemType
''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08
if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType:
val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"),
_("Concept %(concept)s is abstract but type is not xbrli:stringItemType"),
modelObject=modelConcept, concept=modelConcept.qname)
'''
substititutionGroupQname = modelConcept.substitutionGroupQname
# 6.7.23 Axis must be subs group dimension
if name.endswith("Axis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem):
val.modelXbrl.error(("EFM.6.07.23", "GFM.1.03.25"),
_("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.24 Table must be subs group hypercube
if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem):
val.modelXbrl.error(("EFM.6.07.24", "GFM.1.03.26"),
_("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"),
modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname)
# 6.7.25 if neither hypercube or dimension, substitution group must be item
if substititutionGroupQname not in (None,
XbrlConst.qnXbrldtDimensionItem,
XbrlConst.qnXbrldtHypercubeItem,
XbrlConst.qnXbrliItem):
val.modelXbrl.error(("EFM.6.07.25", "GFM.1.03.27"),
_("Concept %(concept)s has disallowed substitution group %(substitutionGroup)s"),
modelObject=modelConcept, concept=modelConcept.qname,
substitutionGroup=modelConcept.substitutionGroupQname)
# 6.7.26 Table must be subs group hypercube
if name.endswith("LineItems") and modelConcept.abstract != "true":
val.modelXbrl.error(("EFM.6.07.26", "GFM.1.03.28"),
_("Concept %(concept)s is a LineItems but not abstract"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.27 type domainMember must end with Domain or Member
conceptType = modelConcept.type
isDomainItemType = conceptType is not None and conceptType.isDomainItemType
endsWithDomainOrMember = name.endswith("Domain") or name.endswith("Member")
if isDomainItemType != endsWithDomainOrMember:
val.modelXbrl.error(("EFM.6.07.27", "GFM.1.03.29"),
_("Concept %(concept)s must end with Domain or Member for type of domainItemType"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.28 domainItemType must be duration
if isDomainItemType and not isDuration:
val.modelXbrl.error(("EFM.6.07.28", "GFM.1.03.30"),
_("Concept %(concept)s is a domainItemType and must be periodType duration"),
modelObject=modelConcept, concept=modelConcept.qname)
if val.validateSBRNL:
if modelConcept.isTuple:
if modelConcept.substitutionGroupQname.localName == "presentationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year
definesPresentationTuples = True
elif modelConcept.substitutionGroupQname.localName == "specificationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year
definesSpecificationTuples = True
else:
definesTuples = True
definesConcepts = True
if modelConcept.isAbstract:
val.modelXbrl.error("SBR.NL.2.2.2.03",
_("Concept %(concept)s is an abstract tuple"),
modelObject=modelConcept, concept=modelConcept.qname)
if tupleCycle(val,modelConcept):
val.modelXbrl.error("SBR.NL.2.2.2.07",
_("Tuple %(concept)s has a tuple cycle"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.get("nillable") != "false" and modelConcept.isRoot:
val.modelXbrl.error("SBR.NL.2.2.2.17", #don't want default, just what was really there
_("Tuple %(concept)s must have nillable='false'"),
modelObject=modelConcept, concept=modelConcept.qname)
elif modelConcept.isItem:
definesConcepts = True
if modelConcept.abstract == "true":
if modelConcept.isRoot:
if modelConcept.get("nillable") != "false": #don't want default, just what was really there
val.modelXbrl.error("SBR.NL.2.2.2.16",
_("Abstract root concept %(concept)s must have nillable='false'"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.typeQname != XbrlConst.qnXbrliStringItemType:
val.modelXbrl.error("SBR.NL.2.2.2.21",
_("Abstract root concept %(concept)s must have type='xbrli:stringItemType'"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.balance:
val.modelXbrl.error("SBR.NL.2.2.2.22",
_("Abstract concept %(concept)s must not have a balance attribute"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.isHypercubeItem:
definesHypercubes = True
elif modelConcept.isDimensionItem:
definesDimensions = True
elif substititutionGroupQname and substititutionGroupQname.localName in ("domainItem","domainMemberItem"):
definesDomains = True
elif modelConcept.isItem:
definesAbstractItems = True
else: # not abstract
if modelConcept.isItem:
definesNonabstractItems = True
if not (modelConcept.label(preferredLabel=XbrlConst.documentationLabel,fallbackToQname=False,lang="nl") or
val.modelXbrl.relationshipSet(XbrlConst.conceptReference).fromModelObject(c) or
modelConcept.genLabel(role=XbrlConst.genDocumentationLabel,lang="nl") or
val.modelXbrl.relationshipSet(XbrlConst.elementReference).fromModelObject(c)):
val.modelXbrl.error("SBR.NL.2.2.2.28",
_("Concept %(concept)s must have a documentation label or reference"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.balance and not modelConcept.instanceOfType(XbrlConst.qnXbrliMonetaryItemType):
val.modelXbrl.error("SBR.NL.2.2.2.24",
_("Non-monetary concept %(concept)s must not have a balance attribute"),
modelObject=modelConcept, concept=modelConcept.qname)
if modelConcept.isLinkPart:
definesLinkParts = True
val.modelXbrl.error("SBR.NL.2.2.5.01",
_("Link:part concept %(concept)s is not allowed"),
modelObject=modelConcept, concept=modelConcept.qname)
if not modelConcept.genLabel(fallbackToQname=False,lang="nl"):
val.modelXbrl.error("SBR.NL.2.2.5.02",
_("Link part definition %(concept)s must have a generic label in language 'nl'"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.8 check for embedded linkbase
for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
if isinstance(e,ModelObject):
val.modelXbrl.error(("EFM.6.07.08", "GFM.1.03.08"),
_("Taxonomy schema %(schema)s contains an embedded linkbase"),
modelObject=e, schema=os.path.basename(modelDocument.uri))
break
requiredUsedOns = {XbrlConst.qnLinkPresentationLink,
XbrlConst.qnLinkCalculationLink,
XbrlConst.qnLinkDefinitionLink}
# 6.7.9 role types authority
for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}roleType"):
if isinstance(e,ModelObject):
roleURI = e.get("roleURI")
if targetNamespaceAuthority != UrlUtil.authority(roleURI):
val.modelXbrl.error(("EFM.6.07.09", "GFM.1.03.09"),
_("RoleType %(roleType)s does not match authority %(targetNamespaceAuthority)s"),
modelObject=e, roleType=roleURI, targetNamespaceAuthority=targetNamespaceAuthority)
# 6.7.9 end with .../role/lc3 name
if not roleTypePattern.match(roleURI):
val.modelXbrl.warning(("EFM.6.07.09", "GFM.1.03.09"),
"RoleType %(roleType)s should end with /role/{LC3name}",
modelObject=e, roleType=roleURI)
# 6.7.10 only one role type declaration in DTS
modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI)
if modelRoleTypes is not None:
modelRoleType = modelRoleTypes[0]
definition = modelRoleType.definitionNotStripped
usedOns = modelRoleType.usedOns
if len(modelRoleTypes) > 1:
val.modelXbrl.error(("EFM.6.07.10", "GFM.1.03.10"),
_("RoleType %(roleType)s is defined in multiple taxonomies"),
modelObject=e, roleType=roleURI)
elif len(modelRoleTypes) == 1:
# 6.7.11 used on's for pre, cal, def if any has a used on
if not usedOns.isdisjoint(requiredUsedOns) and len(requiredUsedOns - usedOns) > 0:
val.modelXbrl.error(("EFM.6.07.11", "GFM.1.03.11"),
_("RoleType %(roleType)s missing used on %(usedOn)s"),
modelObject=e, roleType=roleURI, usedOn=requiredUsedOns - usedOns)
# 6.7.12 definition match pattern
if (val.disclosureSystem.roleDefinitionPattern is not None and
(definition is None or not val.disclosureSystem.roleDefinitionPattern.match(definition))):
val.modelXbrl.error(("EFM.6.07.12", "GFM.1.03.12-14"),
_("RoleType %(roleType)s definition \"%(definition)s\" must match {Sortcode} - {Type} - {Title}"),
modelObject=e, roleType=roleURI, definition=definition)
if val.validateSBRNL:
if usedOns & XbrlConst.standardExtLinkQnames or XbrlConst.qnGenLink in usedOns:
definesLinkroles = True
if not e.genLabel():
val.modelXbrl.error("SBR.NL.2.2.3.03",
_("Link RoleType %(roleType)s missing a generic standard label"),
modelObject=e, roleType=roleURI)
nlLabel = e.genLabel(lang="nl")
if definition != nlLabel:
val.modelXbrl.error("SBR.NL.2.2.3.04",
_("Link RoleType %(roleType)s definition does not match NL standard generic label, \ndefinition: %(definition)s \nNL label: %(label)s"),
modelObject=e, roleType=roleURI, definition=definition, label=nlLabel)
if definition and (definition[0].isspace() or definition[-1].isspace()):
val.modelXbrl.error("SBR.NL.2.2.3.07",
_('Link RoleType %(roleType)s definition has leading or trailing spaces: "%(definition)s"'),
modelObject=e, roleType=roleURI, definition=definition)
# 6.7.13 arcrole types authority
for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}arcroleType"):
if isinstance(e,ModelObject):
arcroleURI = e.get("arcroleURI")
if targetNamespaceAuthority != UrlUtil.authority(arcroleURI):
val.modelXbrl.error(("EFM.6.07.13", "GFM.1.03.15"),
_("ArcroleType %(arcroleType)s does not match authority %(targetNamespaceAuthority)s"),
modelObject=e, arcroleType=arcroleURI, targetNamespaceAuthority=targetNamespaceAuthority)
# 6.7.13 end with .../arcrole/lc3 name
if not arcroleTypePattern.match(arcroleURI):
val.modelXbrl.warning(("EFM.6.07.13", "GFM.1.03.15"),
_("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"),
modelObject=e, arcroleType=arcroleURI)
# 6.7.14 only one arcrole type declaration in DTS
modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI]
if len(modelRoleTypes) > 1:
val.modelXbrl.error(("EFM.6.07.14", "GFM.1.03.16"),
_("ArcroleType %(arcroleType)s is defined in multiple taxonomies"),
modelObject=e, arcroleType=arcroleURI)
# 6.7.15 definition match pattern
definition = modelRoleTypes[0].definition
if definition is None or not arcroleDefinitionPattern.match(definition):
val.modelXbrl.error(("EFM.6.07.15", "GFM.1.03.17"),
_("ArcroleType %(arcroleType)s definition must be non-empty"),
modelObject=e, arcroleType=arcroleURI)
if val.validateSBRNL:
definesArcroles = True
val.modelXbrl.error("SBR.NL.2.2.4.01",
_("Arcrole type definition is not allowed: %(arcroleURI)s"),
modelObject=e, arcroleURI=arcroleURI)
if val.validateSBRNL:
for appinfoElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}appinfo"):
for nonLinkElt in appinfoElt.iterdescendants():
if isinstance(nonLinkElt, ModelObject) and nonLinkElt.namespaceURI != XbrlConst.link:
val.modelXbrl.error("SBR.NL.2.2.11.05",
_("Appinfo contains disallowed non-link element %(element)s"),
modelObject=nonLinkElt, element=nonLinkElt.qname)
for cplxTypeElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexType"):
choiceElt = cplxTypeElt.find("{http://www.w3.org/2001/XMLSchema}choice")
if choiceElt is not None:
val.modelXbrl.error("SBR.NL.2.2.11.09",
_("ComplexType contains disallowed xs:choice element"),
modelObject=choiceElt)
for cplxContentElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexContent"):
if XmlUtil.descendantAttr(cplxContentElt, "http://www.w3.org/2001/XMLSchema", ("extension","restriction"), "base") != "sbr:placeholder":
val.modelXbrl.error("SBR.NL.2.2.11.10",
_("ComplexContent is disallowed"),
modelObject=cplxContentElt)
definesTypes = (modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}complexType") is not None or
modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}simpleType") is not None)
if (definesLinkroles + definesArcroles + definesLinkParts +
definesAbstractItems + definesNonabstractItems +
definesTuples + definesPresentationTuples + definesSpecificationTuples + definesTypes +
definesEnumerations + definesDimensions + definesDomains +
definesHypercubes) != 1:
schemaContents = []
if definesLinkroles: schemaContents.append(_("linkroles"))
if definesArcroles: schemaContents.append(_("arcroles"))
if definesLinkParts: schemaContents.append(_("link parts"))
if definesAbstractItems: schemaContents.append(_("abstract items"))
if definesNonabstractItems: schemaContents.append(_("nonabstract items"))
if definesTuples: schemaContents.append(_("tuples"))
if definesPresentationTuples: schemaContents.append(_("sbrPresentationTuples"))
if definesSpecificationTuples: schemaContents.append(_("sbrSpecificationTuples"))
if definesTypes: schemaContents.append(_("types"))
if definesEnumerations: schemaContents.append(_("enumerations"))
if definesDimensions: schemaContents.append(_("dimensions"))
if definesDomains: schemaContents.append(_("domains"))
if definesHypercubes: schemaContents.append(_("hypercubes"))
if schemaContents:
val.modelXbrl.error("SBR.NL.2.2.1.01",
_("Taxonomy schema may only define one of these: %(contents)s"),
modelObject=modelDocument, contents=', '.join(schemaContents))
elif not any(refDoc.inDTS and refDoc.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces
for refDoc in modelDocument.referencesDocument.keys()): # no linkbase ref or includes
val.modelXbrl.error("SBR.NL.2.2.1.01",
_("Taxonomy schema must be a DTS entrypoint OR define linkroles OR arcroles OR link:parts OR context fragments OR abstract items OR tuples OR non-abstract elements OR types OR enumerations OR dimensions OR domains OR hypercubes"),
modelObject=modelDocument)
if definesConcepts ^ any( # xor so either concepts and no label LB or no concepts and has label LB
(refDoc.type == ModelDocument.Type.LINKBASE and
XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "labelLink") is not None)
for refDoc in modelDocument.referencesDocument.keys()): # no label linkbase
val.modelXbrl.error("SBR.NL.2.2.1.02",
_("A schema that defines concepts MUST have a linked 2.1 label linkbase"),
modelObject=modelDocument)
if (definesNonabstractItems or definesTuples) and not any( # was xor but changed to and not per RH 1/11/12
(refDoc.type == ModelDocument.Type.LINKBASE and
(XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "referenceLink") is not None or
XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/documentation" ) is not None))
for refDoc in modelDocument.referencesDocument.keys()):
val.modelXbrl.error("SBR.NL.2.2.1.03",
_("A schema that defines non-abstract items MUST have a linked (2.1) reference linkbase AND/OR a label linkbase with @xlink:role=documentation"),
modelObject=modelDocument)
visited.remove(modelDocument)
def checkDTSdocument(val, modelDocument, isFilingDocument):
for hrefElt, _hrefedDoc, hrefId in modelDocument.hrefObjects:
if hrefId: #check scheme regardless of whether document loaded
# check all xpointer schemes
for scheme, _path in XmlUtil.xpointerSchemes(hrefId):
if scheme == "element" and val.validateDisclosureSystem:
val.modelXbrl.error(
("EFM.6.03.06", "GFM.1.01.03"),
_("Href %(elementHref)s may only have shorthand xpointers"
),
modelObject=hrefElt,
elementHref=hrefElt.get(
"{http://www.w3.org/1999/xlink}href"))
if not isFilingDocument:
return # not a filing's extension document
if modelDocument.type == ModelDocument.Type.SCHEMA:
if modelDocument.targetNamespace is not None and len(
modelDocument.targetNamespace) > 85:
l = len(modelDocument.targetNamespace.encode("utf-8"))
if l > 255:
val.modelXbrl.error(
"EFM.6.07.30",
_("Schema targetNamespace length (%(length)s) is over 255 bytes long in utf-8 %(targetNamespace)s"
),
modelObject=modelDocument.xmlRootElement,
length=l,
targetNamespace=modelDocument.targetNamespace,
value=modelDocument.targetNamespace)
if modelDocument.type in (ModelDocument.Type.SCHEMA,
ModelDocument.Type.LINKBASE):
isSchema = modelDocument.type == ModelDocument.Type.SCHEMA
if isSchema:
for elt in modelDocument.xmlRootElement.iter(
tag="{http://www.w3.org/2001/XMLSchema}*"):
if elt.namespaceURI == XbrlConst.xsd:
localName = elt.localName
if localName in {"element", "complexType", "simpleType"}:
name = elt.get("name")
if name and len(name) > 64:
l = len(name.encode("utf-8"))
if l > 200:
val.modelXbrl.error(
"EFM.6.07.29",
_("Schema %(element)s has a name length (%(length)s) over 200 bytes long in utf-8, %(name)s."
),
modelObject=elt,
element=localName,
name=name,
length=l)
for elt in modelDocument.xmlRootElement.iter():
if isinstance(elt, ModelObject):
xlinkType = elt.get("{http://www.w3.org/1999/xlink}type")
xlinkRole = elt.get("{http://www.w3.org/1999/xlink}role")
if elt.namespaceURI == XbrlConst.link:
# check schema roleTypes
if elt.localName in ("roleType", "arcroleType"):
uriAttr = {
"roleType": "roleURI",
"arcroleType": "arcroleURI"
}[elt.localName]
roleURI = elt.get(uriAttr)
if len(roleURI) > 85:
l = len(roleURI.encode("utf-8"))
if l > 255:
val.modelXbrl.error(
"EFM.6.07.30",
_("Schema %(element)s %(attribute)s length (%(length)s) is over 255 bytes long in utf-8 %(roleURI)s"
),
modelObject=elt,
element=elt.qname,
attribute=uriAttr,
length=l,
roleURI=roleURI,
value=roleURI)
if elt.localName == "arcroleRef":
refUri = elt.get("arcroleURI")
hrefAttr = elt.get("{http://www.w3.org/1999/xlink}href")
hrefUri, hrefId = UrlUtil.splitDecodeFragment(hrefAttr)
if hrefUri not in val.disclosureSystem.standardTaxonomiesDict:
val.modelXbrl.error(
("EFM.6.09.06", "GFM.1.04.06"),
_("Arcrole %(refURI)s arcroleRef %(xlinkHref)s must be a standard taxonomy"
),
modelObject=elt,
refURI=refUri,
xlinkHref=hrefUri)
if modelDocument.type == ModelDocument.Type.INLINEXBRL and elt.namespaceURI in XbrlConst.ixbrlAll:
if elt.localName == "footnote":
if val.validateGFM:
if elt.get("{http://www.w3.org/1999/xlink}arcrole"
) != XbrlConst.factFootnote:
# must be in a nonDisplay div
if not any(
inlineDisplayNonePattern.search(
e.get("style") or "")
for e in XmlUtil.ancestors(
elt, XbrlConst.xhtml, "div")):
val.modelXbrl.error(
("EFM.N/A", "GFM:1.10.16"),
_("Inline XBRL footnote %(footnoteID)s must be in non-displayable div due to arcrole %(arcrole)s"
),
modelObject=elt,
footnoteID=elt.get("footnoteID"),
arcrole=elt.get(
"{http://www.w3.org/1999/xlink}arcrole"
))
if not elt.get(
"{http://www.w3.org/XML/1998/namespace}lang"):
val.modelXbrl.error(
("EFM.N/A", "GFM:1.10.13"),
_("Inline XBRL footnote %(footnoteID)s is missing an xml:lang attribute"
),
modelObject=elt,
footnoteID=id)
if xlinkType == "extended":
if not xlinkRole or xlinkRole == "":
val.modelXbrl.error(("EFM.6.09.04", "GFM.1.04.04"),
"%(element)s is missing an xlink:role",
modelObject=elt,
element=elt.qname)
if not val.extendedElementName:
val.extendedElementName = elt.qname
elif val.extendedElementName != elt.qname:
val.modelXbrl.error(
("EFM.6.09.07", "GFM:1.04.07"),
_("Extended element %(element)s must be the same as %(element2)s"
),
modelObject=elt,
element=elt.qname,
element2=val.extendedElementName)
elif xlinkType == "resource":
if not xlinkRole:
val.modelXbrl.error(
("EFM.6.09.04", "GFM.1.04.04"),
_("%(element)s is missing an xlink:role"),
modelObject=elt,
element=elt.qname)
elif not XbrlConst.isStandardRole(xlinkRole):
modelsRole = val.modelXbrl.roleTypes.get(xlinkRole)
if (modelsRole is None or len(modelsRole) == 0
or modelsRole[0].modelDocument.targetNamespace
not in val.disclosureSystem.standardTaxonomiesDict
):
val.modelXbrl.error(
("EFM.6.09.05", "GFM.1.04.05"),
_("Resource %(xlinkLabel)s role %(role)s is not a standard taxonomy role"
),
modelObject=elt,
xlinkLabel=elt.get(
"{http://www.w3.org/1999/xlink}label"),
role=xlinkRole,
element=elt.qname,
roleDefinition=val.modelXbrl.roleTypeDefinition(
xlinkRole))
elif xlinkType == "arc":
if elt.get("priority") is not None:
priority = elt.get("priority")
try:
if int(priority) >= 10:
val.modelXbrl.error(
("EFM.6.09.09", "GFM.1.04.08"),
_("Arc from %(xlinkFrom)s to %(xlinkTo)s priority %(priority)s must be less than 10"
),
modelObject=elt,
arcElement=elt.qname,
xlinkFrom=elt.get(
"{http://www.w3.org/1999/xlink}from"),
xlinkTo=elt.get(
"{http://www.w3.org/1999/xlink}to"),
priority=priority)
except (ValueError):
val.modelXbrl.error(
("EFM.6.09.09", "GFM.1.04.08"),
_("Arc from %(xlinkFrom)s to %(xlinkTo)s priority %(priority)s is not an integer"
),
modelObject=elt,
arcElement=elt.qname,
xlinkFrom=elt.get(
"{http://www.w3.org/1999/xlink}from"),
xlinkTo=elt.get(
"{http://www.w3.org/1999/xlink}to"),
priority=priority)
if elt.namespaceURI == XbrlConst.link:
if elt.localName == "presentationArc" and not elt.get(
"order"):
val.modelXbrl.error(
("EFM.6.12.01", "GFM.1.06.01"),
_("PresentationArc from %(xlinkFrom)s to %(xlinkTo)s must have an order"
),
modelObject=elt,
xlinkFrom=elt.get(
"{http://www.w3.org/1999/xlink}from"),
xlinkTo=elt.get(
"{http://www.w3.org/1999/xlink}to"),
conceptFrom=arcFromConceptQname(elt),
conceptTo=arcToConceptQname(elt))
elif elt.localName == "calculationArc":
if not elt.get("order"):
val.modelXbrl.error(
("EFM.6.14.01", "GFM.1.07.01"),
_("CalculationArc from %(xlinkFrom)s to %(xlinkTo)s must have an order"
),
modelObject=elt,
xlinkFrom=elt.get(
"{http://www.w3.org/1999/xlink}from"),
xlinkTo=elt.get(
"{http://www.w3.org/1999/xlink}to"),
conceptFrom=arcFromConceptQname(elt),
conceptTo=arcToConceptQname(elt))
try:
weightAttr = elt.get("weight")
weight = float(weightAttr)
if not weight in (1, -1):
val.modelXbrl.error(
("EFM.6.14.02", "GFM.1.07.02"),
_("CalculationArc from %(xlinkFrom)s to %(xlinkTo)s weight %(weight)s must be 1 or -1"
),
modelObject=elt,
xlinkFrom=elt.get(
"{http://www.w3.org/1999/xlink}from"),
xlinkTo=elt.get(
"{http://www.w3.org/1999/xlink}to"),
conceptFrom=arcFromConceptQname(elt),
conceptTo=arcToConceptQname(elt),
weight=weightAttr)
except ValueError:
val.modelXbrl.error(
("EFM.6.14.02", "GFM.1.07.02"),
_("CalculationArc from %(xlinkFrom)s to %(xlinkTo)s must have an weight (value error in \"%(weight)s\")"
),
modelObject=elt,
xlinkFrom=elt.get(
"{http://www.w3.org/1999/xlink}from"),
xlinkTo=elt.get(
"{http://www.w3.org/1999/xlink}to"),
conceptFrom=arcFromConceptQname(elt),
conceptTo=arcToConceptQname(elt),
weight=weightAttr)
elif elt.localName == "definitionArc":
if not elt.get("order"):
val.modelXbrl.error(
("EFM.6.16.01", "GFM.1.08.01"),
_("DefinitionArc from %(xlinkFrom)s to %(xlinkTo)s must have an order"
),
modelObject=elt,
xlinkFrom=elt.get(
"{http://www.w3.org/1999/xlink}from"),
xlinkTo=elt.get(
"{http://www.w3.org/1999/xlink}to"),
conceptFrom=arcFromConceptQname(elt),
conceptTo=arcToConceptQname(elt))
def validateValue(modelXbrl, elt, attrTag, baseXsdType, value, isNillable=False, facets=None):
if baseXsdType:
try:
if (len(value) == 0 and
not attrTag is None and
not isNillable and
baseXsdType not in ("anyType", "string", "normalizedString", "token", "NMTOKEN", "anyURI", "noContent")):
raise ValueError("missing value for not nillable element")
xValid = VALID
whitespaceReplace = (baseXsdType == "normalizedString")
whitespaceCollapse = (not whitespaceReplace and baseXsdType != "string")
pattern = baseXsdTypePatterns.get(baseXsdType)
if facets:
if "pattern" in facets:
pattern = facets["pattern"]
# note multiple patterns are or'ed togetner, which isn't yet implemented!
if "whiteSpace" in facets:
whitespaceReplace, whitespaceCollapse = {"preserve":(False,False), "replace":(True,False), "collapse":(False,True)}
if whitespaceReplace:
value = normalizeWhitespacePattern.sub(' ', value)
elif whitespaceCollapse:
value = collapseWhitespacePattern.sub(' ', value.strip())
if pattern is not None and pattern.match(value) is None:
raise ValueError("pattern facet " + facets["pattern"].pattern if facets and "pattern" in facets else "pattern mismatch")
if facets:
if "enumeration" in facets and value not in facets["enumeration"]:
raise ValueError("{0} is not in {1}".format(value, facets["enumeration"]))
if "length" in facets and len(value) != facets["length"]:
raise ValueError("length {0}, expected {1}".format(len(value), facets["length"]))
if "minLength" in facets and len(value) < facets["minLength"]:
raise ValueError("length {0}, minLength {1}".format(len(value), facets["minLength"]))
if "maxLength" in facets and len(value) > facets["maxLength"]:
raise ValueError("length {0}, maxLength {1}".format(len(value), facets["maxLength"]))
if baseXsdType == "noContent":
if len(value) > 0 and not value.isspace():
raise ValueError("value content not permitted")
xValue = sValue = None
elif baseXsdType in {"string", "normalizedString", "language", "token", "NMTOKEN","Name","NCName","IDREF","ENTITY"}:
xValue = sValue = value
elif baseXsdType == "ID":
xValue = sValue = value
xValid = VALID_ID
elif baseXsdType == "anyURI":
xValue = anyURI(value)
sValue = value
if xValue and not UrlUtil.isValid(xValue): # allow empty strings to be valid anyURIs
raise ValueError("invalid anyURI value")
elif not value: # rest of types get None if nil/empty value
xValue = sValue = None
elif baseXsdType in ("decimal", "float", "double"):
xValue = sValue = float(value)
if facets:
if "totalDigits" in facets and len(value.replace(".","")) > facets["totalDigits"]:
raise ValueError("totalDigits facet {0}".format(facets["totalDigits"]))
if "fractionDigits" in facets and ( '.' in value and
len(value[value.index('.') + 1:]) > facets["fractionDigits"]):
raise ValueError("fraction digits facet {0}".format(facets["fractionDigits"]))
elif baseXsdType in {"integer",
"nonPositiveInteger","negativeInteger","nonNegativeInteger","positiveInteger",
"long","unsignedLong",
"int","unsignedInt",
"short","unsignedShort",
"byte","unsignedByte"}:
xValue = sValue = int(value)
if ((baseXsdType in {"nonNegativeInteger","unsignedLong","unsignedInt"}
and xValue < 0) or
(baseXsdType == "nonPositiveInteger" and xValue > 0) or
(baseXsdType == "positiveInteger" and xValue <= 0) or
(baseXsdType == "byte" and not -128 <= xValue < 127) or
(baseXsdType == "unsignedByte" and not 0 <= xValue < 255) or
(baseXsdType == "short" and not -32768 <= xValue < 32767) or
(baseXsdType == "unsignedShort" and not 0 <= xValue < 65535) or
(baseXsdType == "positiveInteger" and xValue <= 0)):
raise ValueError("{0} is not {1}".format(value, baseXsdType))
elif baseXsdType == "boolean":
if value in ("true", "1"):
xValue = sValue = True
elif value in ("false", "0"):
xValue = sValue = False
else: raise ValueError
elif baseXsdType == "QName":
xValue = qname(elt, value, castException=ValueError)
sValue = value
''' not sure here, how are explicitDimensions validated, but bad units not?
if xValue.namespaceURI in modelXbrl.namespaceDocs:
if (xValue not in modelXbrl.qnameConcepts and
xValue not in modelXbrl.qnameTypes and
xValue not in modelXbrl.qnameAttributes and
xValue not in modelXbrl.qnameAttributeGroups):
raise ValueError("qname not defined " + str(xValue))
'''
elif baseXsdType in ("XBRLI_DECIMALSUNION", "XBRLI_PRECISIONUNION"):
xValue = sValue = value if value == "INF" else int(value)
elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
xValue = sValue = int(value)
if xValue == 0:
raise ValueError("invalid value")
elif baseXsdType == "XBRLI_DATEUNION":
xValue = dateTime(value, type=DATEUNION, castException=ValueError)
sValue = value
elif baseXsdType == "dateTime":
xValue = dateTime(value, type=DATETIME, castException=ValueError)
sValue = value
elif baseXsdType == "date":
xValue = dateTime(value, type=DATE, castException=ValueError)
sValue = value
elif baseXsdType == "regex-pattern":
# for facet compiling
try:
xValue = re.compile(value + "$") # must match whole string
sValue = value
except Exception as err:
raise ValueError(err)
else:
if baseXsdType in lexicalPatterns:
match = lexicalPatterns[baseXsdType].match(value)
if match is None:
raise ValueError("lexical pattern mismatch")
if baseXsdType == "gMonthDay":
month, day, zSign, zHrMin, zHr, zMin = match.groups()
if int(day) > {2:29, 4:30, 6:30, 9:30, 11:30, 1:31, 3:31, 5:31, 7:31, 8:31, 10:31, 12:31}[int(month)]:
raise ValueError("invalid day {0} for month {1}".format(day, month))
xValue = value
sValue = value
except ValueError as err:
if ModelInlineFact is not None and isinstance(elt, ModelInlineFact):
errElt = "{0} fact {1}".format(elt.elementQname, elt.qname)
else:
errElt = elt.elementQname
if attrTag:
modelXbrl.error("xmlSchema:valueError",
_("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"),
modelObject=elt,
element=errElt,
attribute=XmlUtil.clarkNotationToPrefixedName(elt,attrTag,isAttribute=True),
typeName=baseXsdType,
value=value,
error=err)
else:
modelXbrl.error("xmlSchema:valueError",
_("Element %(element)s type %(typeName)s value error: %(value)s, %(error)s"),
modelObject=elt,
element=errElt,
typeName=baseXsdType,
value=value,
error=err)
xValue = None
sValue = value
xValid = INVALID
else:
xValue = sValue = None
xValid = UNKNOWN
if attrTag:
try: # dynamically allocate attributes (otherwise given shared empty set)
xAttributes = elt.xAttributes
except AttributeError:
elt.xAttributes = xAttributes = {}
xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue, sValue, value)
else:
elt.xValid = xValid
elt.xValue = xValue
elt.sValue = sValue
def validateValue(modelXbrl,
elt,
attrTag,
baseXsdType,
value,
isNillable=False,
isNil=False,
facets=None):
if baseXsdType:
try:
'''
if (len(value) == 0 and attrTag is None and not isNillable and
baseXsdType not in ("anyType", "string", "normalizedString", "token", "NMTOKEN", "anyURI", "noContent")):
raise ValueError("missing value for not nillable element")
'''
xValid = VALID
whitespaceReplace = (baseXsdType == "normalizedString")
whitespaceCollapse = (not whitespaceReplace
and baseXsdType != "string")
isList = baseXsdType in {"IDREFS", "ENTITIES", "NMTOKENS"}
if isList:
baseXsdType = baseXsdType[:-1] # remove plural
if facets:
if "minLength" not in facets:
facets = facets.copy()
facets["minLength"] = 1
else:
facets = {"minLength": 1}
pattern = baseXsdTypePatterns.get(baseXsdType)
if facets:
if "pattern" in facets:
pattern = facets["pattern"]
# note multiple patterns are or'ed togetner, which isn't yet implemented!
if "whiteSpace" in facets:
whitespaceReplace, whitespaceCollapse = {
"preserve": (False, False),
"replace": (True, False),
"collapse": (False, True)
}[facets["whiteSpace"]]
if whitespaceReplace:
value = normalizeWhitespacePattern.sub(
' ', value) # replace tab, line feed, return with space
elif whitespaceCollapse:
value = collapseWhitespacePattern.sub(' ', value).strip(
' '
) # collapse multiple spaces, tabs, line feeds and returns to single space
if baseXsdType == "noContent":
if len(value) > 0 and not entirelyWhitespacePattern.match(
value): # only xml schema pattern whitespaces removed
raise ValueError("value content not permitted")
# note that sValue and xValue are not innerText but only text elements on specific element (or attribute)
xValue = sValue = None
xValid = VALID_NO_CONTENT # notify others that element may contain subelements (for stringValue needs)
elif not value and isNil and isNillable: # rest of types get None if nil/empty value
xValue = sValue = None
else:
if pattern is not None:
if ((isList and any(
pattern.match(v) is None for v in value.split()))
or (not isList and pattern.match(value) is None)):
raise ValueError(
"pattern facet " +
facets["pattern"].pattern if facets
and "pattern" in facets else "pattern mismatch")
if facets:
if "enumeration" in facets and value not in facets[
"enumeration"]:
raise ValueError("{0} is not in {1}".format(
value, facets["enumeration"].keys()))
if "length" in facets and len(value) != facets["length"]:
raise ValueError("length {0}, expected {1}".format(
len(value), facets["length"]))
if "minLength" in facets and len(
value) < facets["minLength"]:
raise ValueError("length {0}, minLength {1}".format(
len(value), facets["minLength"]))
if "maxLength" in facets and len(
value) > facets["maxLength"]:
raise ValueError("length {0}, maxLength {1}".format(
len(value), facets["maxLength"]))
if baseXsdType in {
"string", "normalizedString", "language",
"languageOrEmpty", "token", "NMTOKEN", "Name",
"NCName", "IDREF", "ENTITY"
}:
xValue = sValue = value
elif baseXsdType == "ID":
xValue = sValue = value
xValid = VALID_ID
elif baseXsdType == "anyURI":
if value: # allow empty strings to be valid anyURIs
if UrlUtil.relativeUrlPattern.match(value) is None:
raise ValueError("IETF RFC 2396 4.3 syntax")
# encode PSVI xValue similarly to Xerces and other implementations
xValue = anyURI(UrlUtil.anyUriQuoteForPSVI(value))
sValue = value
elif baseXsdType in ("decimal", "float", "double",
"XBRLI_NONZERODECIMAL"):
if baseXsdType in ("decimal", "XBRLI_NONZERODECIMAL"):
if decimalPattern.match(value) is None:
raise ValueError("lexical pattern mismatch")
xValue = Decimal(value)
sValue = float(
value
) # s-value uses Number (float) representation
if sValue == 0 and baseXsdType == "XBRLI_NONZERODECIMAL":
raise ValueError("zero is not allowed")
else:
if floatPattern.match(value) is None:
raise ValueError("lexical pattern mismatch")
xValue = sValue = float(value)
if facets:
if "totalDigits" in facets and len(
value.replace(".",
"")) > facets["totalDigits"]:
raise ValueError("totalDigits facet {0}".format(
facets["totalDigits"]))
if "fractionDigits" in facets and (
'.' in value
and len(value[value.index('.') + 1:]) >
facets["fractionDigits"]):
raise ValueError(
"fraction digits facet {0}".format(
facets["fractionDigits"]))
if "maxInclusive" in facets and xValue > facets[
"maxInclusive"]:
raise ValueError(" > maxInclusive {0}".format(
facets["maxInclusive"]))
if "maxExclusive" in facets and xValue >= facets[
"maxExclusive"]:
raise ValueError(" >= maxInclusive {0}".format(
facets["maxExclusive"]))
if "minInclusive" in facets and xValue < facets[
"minInclusive"]:
raise ValueError(" < minInclusive {0}".format(
facets["minInclusive"]))
if "minExclusive" in facets and xValue <= facets[
"minExclusive"]:
raise ValueError(" <= minExclusive {0}".format(
facets["minExclusive"]))
elif baseXsdType in {
"integer", "nonPositiveInteger", "negativeInteger",
"nonNegativeInteger", "positiveInteger", "long",
"unsignedLong", "int", "unsignedInt", "short",
"unsignedShort", "byte", "unsignedByte"
}:
xValue = sValue = _INT(value)
if ((baseXsdType in {
"nonNegativeInteger", "unsignedLong", "unsignedInt"
} and xValue < 0) or
(baseXsdType == "nonPositiveInteger" and xValue > 0) or
(baseXsdType == "positiveInteger" and xValue <= 0) or
(baseXsdType == "byte" and not -128 <= xValue < 127)
or (baseXsdType == "unsignedByte"
and not 0 <= xValue < 255)
or (baseXsdType == "short"
and not -32768 <= xValue < 32767)
or (baseXsdType == "unsignedShort"
and not 0 <= xValue < 65535) or
(baseXsdType == "positiveInteger" and xValue <= 0)):
raise ValueError("{0} is not {1}".format(
value, baseXsdType))
if facets:
if "totalDigits" in facets and len(
value.replace(".",
"")) > facets["totalDigits"]:
raise ValueError("totalDigits facet {0}".format(
facets["totalDigits"]))
if "fractionDigits" in facets and (
'.' in value
and len(value[value.index('.') + 1:]) >
facets["fractionDigits"]):
raise ValueError(
"fraction digits facet {0}".format(
facets["fractionDigits"]))
if "maxInclusive" in facets and xValue > facets[
"maxInclusive"]:
raise ValueError(" > maxInclusive {0}".format(
facets["maxInclusive"]))
if "maxExclusive" in facets and xValue >= facets[
"maxExclusive"]:
raise ValueError(" >= maxInclusive {0}".format(
facets["maxExclusive"]))
if "minInclusive" in facets and xValue < facets[
"minInclusive"]:
raise ValueError(" < minInclusive {0}".format(
facets["minInclusive"]))
if "minExclusive" in facets and xValue <= facets[
"minExclusive"]:
raise ValueError(" <= minExclusive {0}".format(
facets["minExclusive"]))
elif baseXsdType == "boolean":
if value in ("true", "1"):
xValue = sValue = True
elif value in ("false", "0"):
xValue = sValue = False
else:
raise ValueError
elif baseXsdType == "QName":
xValue = qnameEltPfxName(elt,
value,
prefixException=ValueError)
#xValue = qname(elt, value, castException=ValueError, prefixException=ValueError)
sValue = value
''' not sure here, how are explicitDimensions validated, but bad units not?
if xValue.namespaceURI in modelXbrl.namespaceDocs:
if (xValue not in modelXbrl.qnameConcepts and
xValue not in modelXbrl.qnameTypes and
xValue not in modelXbrl.qnameAttributes and
xValue not in modelXbrl.qnameAttributeGroups):
raise ValueError("qname not defined " + str(xValue))
'''
elif baseXsdType == "enumerationHrefs":
xValue = [qnameHref(href) for href in value.split()]
sValue = value
elif baseXsdType == "enumerationQNames":
xValue = [
qnameEltPfxName(elt, qn, prefixException=ValueError)
for qn in value.split()
]
sValue = value
elif baseXsdType in ("XBRLI_DECIMALSUNION",
"XBRLI_PRECISIONUNION"):
xValue = sValue = value if value == "INF" else _INT(value)
elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
xValue = sValue = _INT(value)
if xValue == 0:
raise ValueError("invalid value")
elif baseXsdType == "regex-pattern":
# for facet compiling
try:
sValue = value
if value in xmlSchemaPatterns:
xValue = xmlSchemaPatterns[value]
else:
if r"\i" in value or r"\c" in value:
value = value.replace(r"[\i-[:]]", iNameChar).replace(r"\i", iNameChar) \
.replace(r"[\c-[:]]", cMinusCNameChar).replace(r"\c", cNameChar)
xValue = re_compile(value +
"$") # must match whole string
except Exception as err:
raise ValueError(err)
elif baseXsdType == "fraction":
numeratorStr, denominatorStr = elt.fractionValue
if numeratorStr == INVALIDixVALUE or denominatorStr == INVALIDixVALUE:
sValue = xValue = INVALIDixVALUE
xValid = INVALID
else:
sValue = value
numeratorNum = float(numeratorStr)
denominatorNum = float(denominatorStr)
if numeratorNum.is_integer(
) and denominatorNum.is_integer():
xValue = Fraction(int(numeratorNum),
int(denominatorNum))
else:
xValue = Fraction(numeratorNum / denominatorNum)
else:
if baseXsdType in lexicalPatterns:
match = lexicalPatterns[baseXsdType].match(value)
if match is None:
raise ValueError("lexical pattern mismatch")
if baseXsdType == "XBRLI_DATEUNION":
xValue = dateTime(value,
type=DATEUNION,
castException=ValueError)
sValue = value
elif baseXsdType == "dateTime":
xValue = dateTime(value,
type=DATETIME,
castException=ValueError)
sValue = value
elif baseXsdType == "date":
xValue = dateTime(value,
type=DATE,
castException=ValueError)
sValue = value
elif baseXsdType == "gMonthDay":
month, day, zSign, zHrMin, zHr, zMin = match.groups(
)
if int(day) > {
2: 29,
4: 30,
6: 30,
9: 30,
11: 30,
1: 31,
3: 31,
5: 31,
7: 31,
8: 31,
10: 31,
12: 31
}[int(month)]:
raise ValueError(
"invalid day {0} for month {1}".format(
day, month))
xValue = gMonthDay(month, day)
elif baseXsdType == "gYearMonth":
year, month, zSign, zHrMin, zHr, zMin = match.groups(
)
xValue = gYearMonth(year, month)
elif baseXsdType == "gYear":
year, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gYear(year)
elif baseXsdType == "gMonth":
month, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gMonth(month)
elif baseXsdType == "gDay":
day, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gDay(day)
elif baseXsdType == "duration":
xValue = isoDuration(value)
else:
xValue = value
else: # no lexical pattern, forget compiling value
xValue = value
sValue = value
except (ValueError, InvalidOperation) as err:
if ModelInlineValueObject is not None and isinstance(
elt, ModelInlineValueObject):
errElt = "{0} fact {1}".format(elt.elementQname, elt.qname)
else:
errElt = elt.elementQname
if attrTag:
modelXbrl.error(
"xmlSchema:valueError",
_("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"
),
modelObject=elt,
element=errElt,
attribute=XmlUtil.clarkNotationToPrefixedName(
elt, attrTag, isAttribute=True),
typeName=baseXsdType,
value=strTruncate(value, 30),
error=err)
else:
modelXbrl.error(
"xmlSchema:valueError",
_("Element %(element)s type %(typeName)s value error: %(value)s, %(error)s"
),
modelObject=elt,
element=errElt,
typeName=baseXsdType,
value=strTruncate(value, 30),
error=err)
xValue = None
sValue = value
xValid = INVALID
else:
xValue = sValue = None
xValid = UNKNOWN
if attrTag:
try: # dynamically allocate attributes (otherwise given shared empty set)
xAttributes = elt.xAttributes
except AttributeError:
elt.xAttributes = xAttributes = {}
xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue,
sValue, value)
else:
elt.xValid = xValid
elt.xValue = xValue
elt.sValue = sValue
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY):
for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri):
self.validateTestcase(doc) # testcases doc's are sorted by their uri (file names), e.g., for formula
elif hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcaseVariationsByTarget(testcase.testcaseVariations):
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s%(name)s%(target)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=(" {}".format(modelTestcaseVariation.name) if modelTestcaseVariation.name else ""),
target=(" target {}".format(modelTestcaseVariation.ixdsTarget) if modelTestcaseVariation.ixdsTarget else ""),
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings:
errorCaptureLevel = logging._checkLevel("WARNING")
else:
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
elif resultIsTaxonomyPackage:
from arelle import PackageManager, PrototypeInstanceObject
dtsName = readMeFirstUri
modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel,
ixdsTarget=modelTestcaseVariation.ixdsTarget)
else: # need own file source, may need instance discovery
filesource = FileSource.openFileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr, base=baseForElement)
if filesource and not filesource.selection and filesource.isArchive:
try:
if filesource.isTaxonomyPackage:
_rptPkgIxdsOptions = {}
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxdsOptions"):
pluginXbrlMethod(self, _rptPkgIxdsOptions)
filesource.loadTaxonomyPackageMappings()
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxds"):
filesource.select(pluginXbrlMethod(filesource, **_rptPkgIxdsOptions))
else:
from arelle.CntlrCmdLine import filesourceEntrypointFiles
entrypoints = filesourceEntrypointFiles(filesource)
if entrypoints:
# resolve an IXDS in entrypoints
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ArchiveIxds"):
pluginXbrlMethod(self, filesource,entrypoints)
filesource.select(entrypoints[0].get("file", None) )
except Exception as err:
self.modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, entry URL: %(instance)s"),
modelXbrl=self.modelXbrl, instance=readMeFirstUri, error=err)
continue # don't try to load this entry URL
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel,
ixdsTarget=modelTestcaseVariation.ixdsTarget)
modelXbrl.isTestcaseVariation = True
if modelXbrl.modelDocument is None:
modelXbrl.info("arelle:notLoaded",
_("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif resultIsVersioningReport or resultIsTaxonomyPackage:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError))
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif resultIsTaxonomyPackage:
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement)
for d in modelTestcaseVariation.dataUris.values() for uri in d
if not UrlUtil.isAbsolute(uri))
foundDataFiles = set()
variationBase = os.path.dirname(baseForElement)
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
for _inputDTS in inputDTS:
for docUrl, doc in _inputDTS.urlDocs.items():
if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET:
if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file
if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles:
docUrl = docUrl.replace("-formula.xml", ".xf")
foundDataFiles.add(docUrl)
if expectedDataFiles - foundDataFiles:
modelXbrl.info("arelle:testcaseDataNotUsed",
_("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"),
modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles)))
if foundDataFiles - expectedDataFiles:
modelXbrl.info("arelle:testcaseDataUnexpected",
_("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"),
modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles)))
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
try:
RenderingEvaluator.init(modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrlHasFormulae = modelXbrl.hasFormulae
if modelXbrlHasFormulae and self.modelXbrl.modelManager.formulaOptions.formulaAction != "none":
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError))
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
try:
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and
modelTestcaseVariation.resultXbrlInstanceUri is not None)
if compareIxResultInstance:
formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
errMsgPrefix = "ix"
else: # delete input instances before formula output comparision
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
errMsgPrefix = "formula"
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
_matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
_("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.ExpectedInstance.Loaded"):
pluginXbrlMethod(expectedInstance, formulaOutputInstance)
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts),
messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
else:
formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
def factFootnotes(fact, footnotesRelSet):
footnotes = {}
footnoteRels = footnotesRelSet.fromModelObject(fact)
if footnoteRels:
# most process rels in same order between two instances, use labels to sort
for i, footnoteRel in enumerate(sorted(footnoteRels,
key=lambda r: (r.fromLabel,r.toLabel))):
modelObject = footnoteRel.toModelObject
if isinstance(modelObject, ModelResource):
xml = collapseWhitespace(modelObject.viewText().strip())
footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))
elif isinstance(modelObject, ModelFact):
footnotes["Footnoted fact {}".format(i+1)] = \
"{} context: {} value: {}".format(
modelObject.qname,
modelObject.contextID,
collapseWhitespace(modelObject.value))
return footnotes
for expectedInstanceFact in expectedInstance.facts:
unmatchedFactsStack = []
formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False)
#formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True)
if formulaOutputFact is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = expectedInstanceFact
# is it possible to show value mismatches?
expectedFacts = formulaOutputInstance.factsByQname.get(missingFact.qname)
if len(expectedFacts) == 1:
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s, extracted value \"%(value1)s\", expected value \"%(value2)s\""),
modelXbrl=missingFact, fact=missingFact.qname, value1=missingFact.xValue, value2=next(iter(expectedFacts)).xValue,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else:
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else: # compare footnotes
expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or
set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())):
formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
_("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()),
messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
if compareIxResultInstance:
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
_statusCounts = OrderedDict((("pass",0),("fail",0)))
for tv in getattr(testcase, "testcaseVariations", ()):
_statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1
self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k))
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def checkDTSdocument(val, modelDocument, isFilingDocument):
for hrefElt, _hrefedDoc, hrefId in modelDocument.hrefObjects:
if hrefId: #check scheme regardless of whether document loaded
# check all xpointer schemes
for scheme, _path in XmlUtil.xpointerSchemes(hrefId):
if scheme == "element" and val.validateDisclosureSystem:
val.modelXbrl.error(
("EFM.6.03.06", "GFM.1.01.03"),
_("Href %(elementHref)s may only have shorthand xpointers"
),
edgarCode="cp-0306-Href-Xpointers",
modelObject=hrefElt,
elementHref=hrefElt.get(
"{http://www.w3.org/1999/xlink}href"))
if not isFilingDocument:
return # not a filing's extension document
if modelDocument.type == ModelDocument.Type.SCHEMA:
if modelDocument.targetNamespace is not None and len(
modelDocument.targetNamespace) > 85:
l = len(modelDocument.targetNamespace.encode("utf-8"))
if l > 255:
val.modelXbrl.error(
"EFM.6.07.30",
_("Shorten this declaration URI from %(length)s to 255 bytes or fewer in UTF-8: %(value)s."
),
edgarCode="du-0730-uri-length-limit",
modelObject=modelDocument.xmlRootElement,
length=l,
targetNamespace=modelDocument.targetNamespace,
value=modelDocument.targetNamespace)
if modelDocument.type in (ModelDocument.Type.SCHEMA,
ModelDocument.Type.LINKBASE):
isSchema = modelDocument.type == ModelDocument.Type.SCHEMA
if isSchema:
for elt in modelDocument.xmlRootElement.iter(
tag="{http://www.w3.org/2001/XMLSchema}*"):
if elt.namespaceURI == XbrlConst.xsd:
localName = elt.localName
if localName in {"element", "complexType", "simpleType"}:
name = elt.get("name")
if name and len(name) > 64:
l = len(name.encode("utf-8"))
if l > 200:
val.modelXbrl.error(
"EFM.6.07.29",
_("Shorten this declaration name from %(length)s to 200 bytes or fewer in UTF-8: %(name)s"
),
edgarCode="du-0729-name-length-limit",
modelObject=elt,
element=localName,
name=name,
length=l)
for elt in modelDocument.xmlRootElement.iter():
if isinstance(elt, ModelObject):
xlinkType = elt.get("{http://www.w3.org/1999/xlink}type")
xlinkRole = elt.get("{http://www.w3.org/1999/xlink}role")
if elt.namespaceURI == XbrlConst.link:
# check schema roleTypes
if elt.localName in ("roleType", "arcroleType"):
uriAttr = {
"roleType": "roleURI",
"arcroleType": "arcroleURI"
}[elt.localName]
roleURI = elt.get(uriAttr)
if len(roleURI) > 85:
l = len(roleURI.encode("utf-8"))
if l > 255:
val.modelXbrl.error(
"EFM.6.07.30",
_("Shorten this declaration URI from %(length)s to 255 bytes or fewer in UTF-8: %(value)s."
),
edgarCode="du-0730-uri-length-limit",
modelObject=elt,
element=elt.qname,
attribute=uriAttr,
length=l,
roleURI=roleURI,
value=roleURI)
if elt.localName == "arcroleRef":
refUri = elt.get("arcroleURI")
hrefAttr = elt.get("{http://www.w3.org/1999/xlink}href")
hrefUri, hrefId = UrlUtil.splitDecodeFragment(hrefAttr)
if hrefUri not in val.disclosureSystem.standardTaxonomiesDict:
val.modelXbrl.error(
("EFM.6.09.06", "GFM.1.04.06"),
_("An arcroleRef element refers to %(xlinkHref)s, an arcrole, %(refURI)s, that is not defined in a standard taxonomy. "
"Please recheck submission."),
edgarCode="fs-0906-Custom-Arcrole-Referenced",
modelObject=elt,
refURI=refUri,
xlinkHref=hrefUri)
if modelDocument.type == ModelDocument.Type.INLINEXBRL and elt.namespaceURI in XbrlConst.ixbrlAll:
if elt.localName == "footnote":
if val.validateGFM:
if elt.get("{http://www.w3.org/1999/xlink}arcrole"
) != XbrlConst.factFootnote:
# must be in a nonDisplay div
if not any(
inlineDisplayNonePattern.search(
e.get("style") or "")
for e in XmlUtil.ancestors(
elt, XbrlConst.xhtml, "div")):
val.modelXbrl.error(
("EFM.N/A", "GFM:1.10.16"),
_("Inline XBRL footnote %(footnoteID)s must be in non-displayable div due to arcrole %(arcrole)s"
),
modelObject=elt,
footnoteID=elt.get("footnoteID"),
arcrole=elt.get(
"{http://www.w3.org/1999/xlink}arcrole"
))
if not elt.get(
"{http://www.w3.org/XML/1998/namespace}lang"):
val.modelXbrl.error(
("EFM.N/A", "GFM:1.10.13"),
_("Inline XBRL footnote %(footnoteID)s is missing an xml:lang attribute"
),
modelObject=elt,
footnoteID=id)
if xlinkType == "extended":
if not xlinkRole or xlinkRole == "":
val.modelXbrl.error(
("EFM.6.09.04", "GFM.1.04.04"),
"The %(element)s element requires an xlink:role such as the default, 'http://www.xbrl.org/2003/role/label'. "
"Please recheck submission.",
edgarCode="fs-0904-Resource-Role-Missing",
modelObject=elt,
element=elt.qname)
if not val.extendedElementName:
val.extendedElementName = elt.qname
elif val.extendedElementName != elt.qname:
val.modelXbrl.error(
("EFM.6.09.07", "GFM:1.04.07"),
_("Your filing contained extended type links with roles of different namesapces and local names, %(element)s "
"and %(element2)s in the same linkbase, %(refUrl)s. Please recheck your submission and ensure that all "
"extended-type links in a single linkbase have the same namespace and local name."
),
edgarCode="cp-0907-Linkbases-Distinct",
refUrl=elt.modelDocument.basename,
modelObject=elt,
element=elt.qname,
element2=val.extendedElementName)
elif xlinkType == "resource":
if not xlinkRole:
val.modelXbrl.error(
("EFM.6.09.04", "GFM.1.04.04"),
_("The %(element)s element requires an xlink:role such as the default, 'http://www.xbrl.org/2003/role/label'. "
"Please recheck submission."),
edgarCode="fs-0904-Resource-Role-Missing",
modelObject=elt,
element=elt.qname)
elif not XbrlConst.isStandardRole(xlinkRole):
modelsRole = val.modelXbrl.roleTypes.get(xlinkRole)
if (modelsRole is None or len(modelsRole) == 0
or modelsRole[0].modelDocument.targetNamespace
not in val.disclosureSystem.standardTaxonomiesDict
):
val.modelXbrl.error(
("EFM.6.09.05", "GFM.1.04.05"),
_("The %(element)s element has a role, %(roleDefinition)s, that is not defined in a standard taxonomy, "
"resource labeled %(xlinkLabel)s. Please recheck submission."
),
edgarCode="fs-0905-Custom-Resource-Role-Used",
modelObject=elt,
xlinkLabel=elt.get(
"{http://www.w3.org/1999/xlink}label"),
role=xlinkRole,
element=elt.qname,
roleDefinition=val.modelXbrl.roleTypeDefinition(
xlinkRole))
elif xlinkType == "arc":
if elt.get("priority") is not None:
priority = elt.get("priority")
try:
if int(priority) >= 10:
val.modelXbrl.error(
("EFM.6.09.09", "GFM.1.04.08"),
_("The priority attribute %(priority)s has a value of ten or greater, on arc %(arcElement)s from %(xlinkFrom)s to %(xlinkTo)s. "
"Please change the priority to value less than 10 and resubmit."
),
edgarCode=
"du-0909-Relationship-Priority-Not-Less-Than-Ten",
modelObject=elt,
arcElement=elt.qname,
xlinkFrom=elt.get(
"{http://www.w3.org/1999/xlink}from"),
xlinkTo=elt.get(
"{http://www.w3.org/1999/xlink}to"),
priority=priority)
except (ValueError):
val.modelXbrl.error(
("EFM.6.09.09", "GFM.1.04.08"),
_("The priority attribute %(priority)s has a value of ten or greater, on arc %(arcElement)s from %(xlinkFrom)s to %(xlinkTo)s. "
"Please change the priority to value less than 10 and resubmit."
),
edgarCode=
"du-0909-Relationship-Priority-Not-Less-Than-Ten",
modelObject=elt,
arcElement=elt.qname,
xlinkFrom=elt.get(
"{http://www.w3.org/1999/xlink}from"),
xlinkTo=elt.get(
"{http://www.w3.org/1999/xlink}to"),
priority=priority)
if elt.namespaceURI == XbrlConst.link:
if elt.localName == "presentationArc" and not elt.get(
"order"):
val.modelXbrl.error(
("EFM.6.12.01", "GFM.1.06.01"),
_("The presentation relationship from %(conceptFrom)s to %(conceptTo)s does not have an order attribute. "
"Please provide a value."),
edgarCode="rq-1201-Presentation-Order-Missing",
modelObject=elt,
xlinkFrom=elt.get(
"{http://www.w3.org/1999/xlink}from"),
xlinkTo=elt.get(
"{http://www.w3.org/1999/xlink}to"),
conceptFrom=arcFromConceptQname(elt),
conceptTo=arcToConceptQname(elt))
elif elt.localName == "calculationArc":
if not elt.get("order"):
val.modelXbrl.error(
("EFM.6.14.01", "GFM.1.07.01"),
_("The calculation relationship from %(conceptFrom)s to %(conceptTo)s does not have an order attribute."
),
edgarCode=
"du-1401-Calculation-Relationship-Order-Missing",
modelObject=elt,
xlinkFrom=elt.get(
"{http://www.w3.org/1999/xlink}from"),
xlinkTo=elt.get(
"{http://www.w3.org/1999/xlink}to"),
conceptFrom=arcFromConceptQname(elt),
conceptTo=arcToConceptQname(elt))
try:
weightAttr = elt.get("weight")
weight = float(weightAttr)
if not weight in (1, -1):
val.modelXbrl.error(
("EFM.6.14.02", "GFM.1.07.02"),
_("Weight value %(weight)s on the calculation relationship from %(conceptFrom)s to %(conceptTo)s "
"must be equal to 1 or to -1. Please recheck submission."
),
edgarCode=
"fs-1402-Calculation-Relationship-Weight-Not-Unitary",
modelObject=elt,
xlinkFrom=elt.get(
"{http://www.w3.org/1999/xlink}from"),
xlinkTo=elt.get(
"{http://www.w3.org/1999/xlink}to"),
conceptFrom=arcFromConceptQname(elt),
conceptTo=arcToConceptQname(elt),
weight=weightAttr)
except ValueError:
val.modelXbrl.error(
("EFM.6.14.02", "GFM.1.07.02"),
_("Weight value %(weight)s on the calculation relationship from %(conceptFrom)s to %(conceptTo)s "
"must be equal to 1 or to -1. Please recheck submission."
),
edgarCode=
"fs-1402-Calculation-Relationship-Weight-Not-Unitary",
modelObject=elt,
xlinkFrom=elt.get(
"{http://www.w3.org/1999/xlink}from"),
xlinkTo=elt.get(
"{http://www.w3.org/1999/xlink}to"),
conceptFrom=arcFromConceptQname(elt),
conceptTo=arcToConceptQname(elt),
weight=weightAttr)
elif elt.localName == "definitionArc":
if not elt.get("order"):
val.modelXbrl.error(
("EFM.6.16.01", "GFM.1.08.01"),
_("The Definition relationship from %(conceptFrom)s to %(conceptTo)s does not have an order attribute."
),
edgarCode=
"du-1601-Definition-Relationship-Order-Missing",
modelObject=elt,
xlinkFrom=elt.get(
"{http://www.w3.org/1999/xlink}from"),
xlinkTo=elt.get(
"{http://www.w3.org/1999/xlink}to"),
conceptFrom=arcFromConceptQname(elt),
conceptTo=arcToConceptQname(elt))
def validateValue(modelXbrl, elt, attrTag, baseXsdType, value, isNillable=False, facets=None):
if baseXsdType:
try:
if (len(value) == 0 and
not attrTag is None and
not isNillable and
baseXsdType not in ("anyType", "string", "normalizedString", "token", "NMTOKEN", "anyURI", "noContent")):
raise ValueError("missing value for not nillable element")
xValid = VALID
whitespaceReplace = (baseXsdType == "normalizedString")
whitespaceCollapse = (not whitespaceReplace and baseXsdType != "string")
pattern = baseXsdTypePatterns.get(baseXsdType)
if facets:
if "pattern" in facets:
pattern = facets["pattern"]
# note multiple patterns are or'ed togetner, which isn't yet implemented!
if "whiteSpace" in facets:
whitespaceReplace, whitespaceCollapse = {"preserve":(False,False), "replace":(True,False), "collapse":(False,True)}
if whitespaceReplace:
value = normalizeWhitespacePattern.sub(' ', value)
elif whitespaceCollapse:
value = collapseWhitespacePattern.sub(' ', value.strip())
if pattern is not None and pattern.match(value) is None:
raise ValueError("pattern facet " + facets["pattern"].pattern if facets and "pattern" in facets else "pattern mismatch")
if facets:
if "enumeration" in facets and value not in facets["enumeration"]:
raise ValueError("is not in {1}".format(value, facets["enumeration"]))
if "length" in facets and len(value) != facets["length"]:
raise ValueError("length facet")
if "minLength" in facets and len(value) < facets["minLength"]:
raise ValueError("minLength facet")
if "maxLength" in facets and len(value) > facets["maxLength"]:
raise ValueError("maxLength facet")
if baseXsdType == "noContent":
if len(value) > 0 and not value.isspace():
raise ValueError("value content not permitted")
xValue = sValue = None
elif baseXsdType in {"string", "normalizedString", "language", "token", "NMTOKEN","Name","NCName","IDREF","ENTITY"}:
xValue = sValue = value
elif baseXsdType == "ID":
xValue = sValue = value
xValid = VALID_ID
elif baseXsdType == "anyURI":
xValue = anyURI(value)
sValue = value
if xValue and not UrlUtil.isValid(xValue): # allow empty strings to be valid anyURIs
raise ValueError("invalid anyURI value")
elif not value: # rest of types get None if nil/empty value
xValue = sValue = None
elif baseXsdType in ("decimal", "float", "double"):
xValue = sValue = float(value)
if facets:
if "totalDigits" in facets and len(value.replace(".","")) != facets["totalDigits"]:
raise ValueError("total digits facet")
if "fractionDigits" in facets and ( '.' not in value or
len(value[value.index('.') + 1:]) != facets["fractionDigits"]):
raise ValueError("fraction digits facet")
elif baseXsdType in ("integer",):
xValue = sValue = int(value)
elif baseXsdType == "boolean":
if value in ("true", "1"):
xValue = sValue = True
elif value in ("false", "0"):
xValue = sValue = False
else: raise ValueError
elif baseXsdType == "QName":
xValue = qname(elt, value, castException=ValueError)
sValue = value
''' not sure here, how are explicitDimensions validated, but bad units not?
if xValue.namespaceURI in modelXbrl.namespaceDocs:
if (xValue not in modelXbrl.qnameConcepts and
xValue not in modelXbrl.qnameTypes and
xValue not in modelXbrl.qnameAttributes and
xValue not in modelXbrl.qnameAttributeGroups):
raise ValueError("qname not defined " + str(xValue))
'''
elif baseXsdType in ("XBRLI_DECIMALSUNION", "XBRLI_PRECISIONUNION"):
xValue = sValue = value if value == "INF" else int(value)
elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
xValue = sValue = int(value)
if xValue == 0:
raise ValueError("invalid value")
elif baseXsdType == "XBRLI_DATEUNION":
xValue = dateTime(value, type=DATEUNION, castException=ValueError)
sValue = value
elif baseXsdType == "dateTime":
xValue = dateTime(value, type=DATETIME, castException=ValueError)
sValue = value
elif baseXsdType == "date":
xValue = dateTime(value, type=DATE, castException=ValueError)
sValue = value
elif baseXsdType == "regex-pattern":
# for facet compiling
try:
xValue = re.compile(value + "$") # must match whole string
sValue = value
except Exception as err:
raise ValueError(err)
else:
xValue = value
sValue = value
except ValueError as err:
if attrTag:
modelXbrl.error("xmlSchema:valueError",
_("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"),
modelObject=elt,
element=elt.elementQname,
attribute=XmlUtil.clarkNotationToPrefixedName(elt,attrTag,isAttribute=True),
typeName=baseXsdType,
value=value,
error=err)
else:
modelXbrl.error("xmlSchema:valueError",
_("Element %(element)s type %(typeName)s value error: %(value)s"),
modelObject=elt,
element=elt.elementQname,
typeName=baseXsdType,
value=value)
xValue = None
sValue = value
xValid = INVALID
else:
xValue = sValue = None
xValid = UNKNOWN
if attrTag:
elt.xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue, sValue, value)
else:
elt.xValid = xValid
elt.xValue = xValue
elt.sValue = sValue
def logArguments(self, codes, msg, codedArgs):
# determine logCode
messageCode = None
for argCode in codes if isinstance(codes,tuple) else (codes,):
if (isinstance(argCode, ModelValue.QName) or
(self.modelManager.disclosureSystem.EFM and argCode.startswith("EFM")) or
(self.modelManager.disclosureSystem.GFM and argCode.startswith("GFM")) or
(self.modelManager.disclosureSystem.HMRC and argCode.startswith("HMRC")) or
(self.modelManager.disclosureSystem.SBRNL and argCode.startswith("SBR.NL")) or
argCode[0:3] not in ("EFM", "GFM", "HMR", "SBR")):
messageCode = argCode
break
# determine message and extra arguments
fmtArgs = {}
extras = {"messageCode":messageCode}
for argName, argValue in codedArgs.items():
if argName in ("modelObject", "modelXbrl", "modelDocument"):
try:
entryUrl = self.modelDocument.uri
except AttributeError:
entryUrl = self.entryLoadingUrl
try:
objectUrl = argValue.modelDocument.uri
except AttributeError:
try:
objectUrl = self.modelDocument.uri
except AttributeError:
objectUrl = self.entryLoadingUrl
refs = []
for arg in (argValue if isinstance(argValue, (tuple,list)) else (argValue,)):
if arg is not None:
file = UrlUtil.relativeUri(entryUrl, objectUrl)
ref = {}
if isinstance(arg,ModelObject):
ref["href"] = file + "#" + XmlUtil.elementFragmentIdentifier(arg)
ref["sourceLine"] = arg.sourceline
ref["objectId"] = arg.objectId()
else:
ref["href"] = file
refs.append(ref)
extras["refs"] = refs
elif argName == "sourceLine":
if isinstance(argValue, _INT_TYPES): # must be sortable with int's in logger
extras["sourceLine"] = argValue
elif argName != "exc_info":
if isinstance(argValue, (ModelValue.QName, ModelObject, bool, FileNamedStringIO)):
fmtArgs[argName] = str(argValue)
elif isinstance(argValue, _INT_TYPES):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale, '%i', argValue)
elif isinstance(argValue,float):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale, '%f', argValue)
else:
fmtArgs[argName] = argValue
if "refs" not in extras:
try:
file = os.path.basename(self.modelDocument.uri)
except AttributeError:
try:
file = os.path.basename(self.entryLoadingUrl)
except:
file = ""
extras["refs"] = [{"href": file}]
return (messageCode,
(msg, fmtArgs) if fmtArgs else (msg,),
extras)
def final(val):
if not (val.validateEBA or val.validateEIOPA):
return
modelXbrl = val.modelXbrl
modelDocument = modelXbrl.modelDocument
_statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name)
modelXbrl.profileActivity()
modelXbrl.modelManager.showStatus(_statusMsg)
if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA):
if not modelDocument.uri.endswith(".xbrl"):
modelXbrl.warning("EBA.1.1",
_('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'),
modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1])
modelXbrl.error("EIOPA.S.1.1.a",
_('XBRL instance documents MUST use the extension ".xbrl" but it is "%(extension)s"'),
modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1])
if val.isEIOPA_2_0_1: _encodings = ("UTF-8", "utf-8-sig")
else: _encodings = ("utf-8", "UTF-8", "utf-8-sig")
if modelDocument.documentEncoding not in _encodings:
modelXbrl.error(("EBA.1.4", "EIOPA.1.4"),
_('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'),
modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding)
schemaRefElts = []
schemaRefFileNames = []
for doc, docRef in modelDocument.referencesDocument.items():
if docRef.referenceType == "href":
if docRef.referringModelObject.localName == "schemaRef":
schemaRefElts.append(docRef.referringModelObject)
schemaRefFileNames.append(doc.basename)
if not UrlUtil.isAbsolute(doc.uri):
modelXbrl.error(("EBA.2.2", "EIOPA.S.1.5.a" if val.isEIOPAfullVersion else "EIOPA.S.1.5.b"),
_('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'),
modelObject=docRef.referringModelObject, url=doc.uri,
messageCodes=("EBA.2.2", "EIOPA.S.1.5.a","EIOPA.S.1.5.b"))
elif docRef.referringModelObject.localName == "linkbaseRef":
modelXbrl.error(("EBA.2.3","EIOPA.S.1.5.a"),
_('The link:linkbaseRef element is not allowed: %(fileName)s.'),
modelObject=docRef.referringModelObject, fileName=doc.basename)
_numSchemaRefs = len(XmlUtil.children(modelDocument.xmlRootElement, XbrlConst.link, "schemaRef"))
if _numSchemaRefs > 1:
modelXbrl.error(("EIOPA.S.1.5.a", "EBA.1.5"),
_('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'),
modelObject=modelDocument, numEntryPoints=_numSchemaRefs, entryPointNames=', '.join(sorted(schemaRefFileNames)))
### check entry point names appropriate for filing indicator (DPM DB?)
if len(schemaRefElts) != 1:
modelXbrl.error("EBA.2.3",
_('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'),
modelObject=schemaRefElts, entryPointCount=len(schemaRefElts))
# non-streaming EBA checks
if not getattr(modelXbrl, "isStreamingMode", False):
val.qnReportedCurrency = None
if val.isEIOPA_2_0_1 and qnMetReportingCurrency in modelXbrl.factsByQname:
for _multiCurrencyFact in modelXbrl.factsByQname[qnMetReportingCurrency]:
# multi-currency fact
val.qnReportedCurrency = _multiCurrencyFact.xValue
break
validateFacts(val, modelXbrl.facts)
# check sum of fact md5s (otherwise checked in streaming process)
xbrlFactsCheckVersion = None
expectedSumOfFactMd5s = None
for pi in modelDocument.xmlRootElement.getchildren():
if isinstance(pi, etree._ProcessingInstruction) and pi.target == "xbrl-facts-check":
_match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text)
if _match:
_matchGroups = _match.groups()
if len(_matchGroups) == 2:
if _matchGroups[0] == "version":
xbrlFactsCheckVersion = _matchGroups[1]
elif _matchGroups[0] == "sum-of-fact-md5s":
try:
expectedSumOfFactMd5s = Md5Sum(_matchGroups[1])
except ValueError:
modelXbrl.error("EIOPA:xbrlFactsCheckError",
_("Invalid sum-of-md5s %(sumOfMd5)s"),
modelObject=modelXbrl, sumOfMd5=_matchGroups[1])
if xbrlFactsCheckVersion and expectedSumOfFactMd5s:
sumOfFactMd5s = Md5Sum()
for f in modelXbrl.factsInInstance:
sumOfFactMd5s += f.md5sum
if sumOfFactMd5s != expectedSumOfFactMd5s:
modelXbrl.warning("EIOPA:xbrlFactsCheckWarning",
_("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"),
modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s)
else:
modelXbrl.info("info",
_("Successful XBRL facts sum of md5s."),
modelObject=modelXbrl)
if any(badError in modelXbrl.errors
for badError in ("EBA.2.1", "EIOPA.2.1", "EIOPA.S.1.5.a/EIOPA.S.1.5.b")):
pass # skip checking filingIndicators if bad errors
elif not val.filingIndicators:
modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"),
_('Missing filing indicators. Reported XBRL instances MUST include appropriate (positive) filing indicator elements'),
modelObject=modelDocument)
elif all(filed == False for filed in val.filingIndicators.values()):
modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"),
_('All filing indicators are filed="false". Reported XBRL instances MUST include appropriate (positive) filing indicator elements'),
modelObject=modelDocument)
if val.numFilingIndicatorTuples > 1:
modelXbrl.warning(("EBA.1.6.2", "EIOPA.1.6.2"),
_('Multiple filing indicators tuples when not in streaming mode (info).'),
modelObject=modelXbrl.factsByQname[qnFIndicators])
if len(val.cntxDates) > 1:
modelXbrl.error(("EBA.2.13","EIOPA.2.13"),
_('Contexts must have the same date: %(dates)s.'),
# when streaming values are no longer available, but without streaming they can be logged
modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs),
dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True)
for _dt in val.cntxDates.keys()))
if val.unusedCntxIDs:
if val.isEIOPA_2_0_1:
modelXbrl.error("EIOPA.2.7",
_('Unused xbrli:context nodes MUST NOT be present in the instance: %(unusedContextIDs)s.'),
modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts],
unusedContextIDs=", ".join(sorted(val.unusedCntxIDs)))
else:
modelXbrl.warning(("EBA.2.7", "EIOPA.2.7"),
_('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'),
modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts],
unusedContextIDs=", ".join(sorted(val.unusedCntxIDs)))
if len(val.cntxEntities) > 1:
modelXbrl.error(("EBA.2.9", "EIOPA.2.9"),
_('All entity identifiers and schemes MUST be the same, %(count)s found: %(entities)s.'),
modelObject=modelDocument, count=len(val.cntxEntities),
entities=", ".join(sorted(str(cntxEntity) for cntxEntity in val.cntxEntities)))
for _scheme, _LEI in val.cntxEntities:
if (_scheme in ("http://standards.iso.org/iso/17442", "http://standard.iso.org/iso/17442", "LEI") or
(not val.isEIOPAfullVersion and _scheme == "PRE-LEI")):
if _scheme == "http://standard.iso.org/iso/17442":
modelXbrl.warning(("EBA.3.6", "EIOPA.S.2.8.c"),
_("Warning, context has entity scheme %(scheme)s should be plural: http://standards.iso.org/iso/17442."),
modelObject=modelDocument, scheme=_scheme)
result = LeiUtil.checkLei(_LEI)
if result == LeiUtil.LEI_INVALID_LEXICAL:
modelXbrl.error("EIOPA.S.2.8.c",
_("Context has lexically invalid LEI %(lei)s."),
modelObject=modelDocument, lei=_LEI)
elif result == LeiUtil.LEI_INVALID_CHECKSUM:
modelXbrl.error("EIOPA.S.2.8.c",
_("Context has LEI checksum error in %(lei)s."),
modelObject=modelDocument, lei=_LEI)
elif _scheme == "SC":
pass # anything is ok for Specific Code
else:
modelXbrl.error("EIOPA.S.2.8.c",
_("Context has unrecognized entity scheme %(scheme)s."),
modelObject=modelDocument, scheme=_scheme)
if val.unusedUnitIDs:
if val.isEIOPA_2_0_1:
modelXbrl.error("EIOPA.2.22",
_('Unused xbrli:unit nodes MUST NOT be present in the instance: %(unusedUnitIDs)s.'),
modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units],
unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs)))
else:
modelXbrl.warning(("EBA.2.22", "EIOPA.2.22"),
_('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'),
modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units],
unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs)))
if len(val.currenciesUsed) > 1:
modelXbrl.error(("EBA.3.1","EIOPA.3.1"),
_("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"),
modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join(str(c) for c in val.currenciesUsed.keys()))
elif val.isEIOPA_2_0_1 and any(_measure.localName != val.reportingCurrency for _measure in val.currenciesUsed.keys()):
modelXbrl.error("EIOPA.3.1",
_("There MUST be only one currency but reporting currency %(reportingCurrency)s differs from unit currencies: %(currencies)s.'"),
modelObject=val.currenciesUsed.values(), reportingCurrency=val.reportingCurrency, currencies=", ".join(str(c) for c in val.currenciesUsed.keys()))
if val.prefixesUnused:
modelXbrl.warning(("EBA.3.4", "EIOPA.3.4"),
_("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"),
modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused)))
for ns, prefixes in val.namespacePrefixesUsed.items():
nsDocs = modelXbrl.namespaceDocs.get(ns)
if nsDocs:
for nsDoc in nsDocs:
nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns)
if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None):
modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"),
_("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"),
modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None})))
elif ns in CANONICAL_PREFIXES and any(prefix != CANONICAL_PREFIXES[ns] for prefix in prefixes if prefix is not None):
modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"),
_("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"),
modelObject=modelDocument, namespace=ns, declaredPrefix=CANONICAL_PREFIXES[ns], foundPrefixes=', '.join(sorted(prefixes - {None})))
modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0)
modelXbrl.modelManager.showStatus(None)
del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements
del val.utrValidator, val.firstFact, val.footnotesRelationshipSet
def uriAuthorityValid(self, uri):
if self.standardTaxonomiesUrl:
return UrlUtil.authority(uri) in self.standardAuthorities
return True # no standard authorities to test
def checkDTS(val, modelDocument, visited):
visited.append(modelDocument)
definesLabelLinkbase = False
for referencedDocument in modelDocument.referencesDocument.items():
#6.07.01 no includes
if referencedDocument[1] == "include":
val.modelXbrl.error(
_("Taxonomy schema {0} includes {1}, only import is allowed").format(
os.path.basename(modelDocument.uri),
os.path.basename(referencedDocument[0].uri)),
"err", "EFM.6.07.01", "GFM.1.03.01", "SBR.NL.2.2.0.18")
if referencedDocument[0] not in visited:
checkDTS(val, referencedDocument[0], visited)
if val.disclosureSystem.standardTaxonomiesDict is None:
pass
if (modelDocument.type == ModelDocument.Type.SCHEMA and
modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and
modelDocument.uri.startswith(val.modelXbrl.uriDir)):
# check schema contents types
if val.validateSBRNL:
definesConcepts = False
definesLinkroles = False
definesArcroles = False
definesLinkParts = False
definesAbstractItems = False
definesNonabstractItems = False
definesTuples = False
definesEnumerations = False
definesDimensions = False
definesDomains = False
definesHypercubes = False
# 6.7.3 check namespace for standard authority
targetNamespaceAuthority = UrlUtil.authority(modelDocument.targetNamespace)
if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities:
val.modelXbrl.error(
_("Taxonomy schema {0} namespace {1} is a disallowed authority").format(
os.path.basename(modelDocument.uri),
modelDocument.targetNamespace),
"err", "EFM.6.07.03", "GFM.1.03.03")
# 6.7.4 check namespace format
if modelDocument.targetNamespace is None:
match = None
elif val.validateEFMorGFM:
targetNamespaceDate = modelDocument.targetNamespace[len(targetNamespaceAuthority):]
match = val.targetNamespaceDatePattern.match(targetNamespaceDate)
else:
match = None
if match is not None:
try:
if match.lastindex == 3:
datetime.date(int(match.group(1)),int(match.group(2)),int(match.group(3)))
elif match.lastindex == 6:
datetime.date(int(match.group(4)),int(match.group(5)),int(match.group(6)))
else:
match = None
except ValueError:
match = None
if match is None:
val.modelXbrl.error(
_("Taxonomy schema {0} namespace {1} must have format http://{2}authority{3}/{2}versionDate{3}").format(
os.path.basename(modelDocument.uri),
modelDocument.targetNamespace, "{", "}"),
"err", "EFM.6.07.04", "GFM.1.03.04")
if modelDocument.targetNamespace is not None:
# 6.7.5 check prefix for _
prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement,modelDocument.targetNamespace)
if prefix and "_" in prefix:
val.modelXbrl.error(
_("Taxonomy schema {0} namespace {1} prefix {2} must not have an '_'").format(
os.path.basename(modelDocument.uri),
modelDocument.targetNamespace,
prefix),
"err", "EFM.6.07.07", "GFM.1.03.07")
for eltName in modelDocument.xmlRootElement.getElementsByTagNameNS(XbrlConst.xsd, "element"):
# 6.7.16 name not duplicated in standard taxonomies
name = eltName.getAttribute("name")
concepts = val.modelXbrl.nameConcepts.get(name)
modelConcept = None
if concepts is not None:
for c in concepts:
if c.modelDocument == modelDocument:
modelConcept = c
elif (val.validateEFMorGFM and
not c.modelDocument.uri.startswith(val.modelXbrl.uriDir)):
val.modelXbrl.error(
_("Taxonomy schema {0} element {1} is also defined in standard taxonomy schema {2}").format(
os.path.basename(modelDocument.uri),
name,
os.path.basename(c.modelDocument.uri)),
"err", "EFM.6.07.16", "GFM.1.03.18")
if val.validateSBRNL:
for c in concepts:
if c.modelDocument != modelDocument:
relSet = val.modelXbrl.relationshipSet(XbrlConst.generalSpecial)
if not (relSet.isRelated(modelConcept, "child", c) or relSet.isRelated(modelConcept, "child", c)):
val.modelXbrl.error(
_("Taxonomy schema {0} element {1} is also defined in standard taxonomy schema {2} without a general-special relationship").format(
os.path.basename(modelDocument.uri),
name,
os.path.basename(c.modelDocument.uri)),
"err", "SBR.NL.2.2.2.02")
# 6.7.17 id properly formed
id = eltName.getAttribute("id")
requiredId = str(prefix) + "_" + name
if val.validateEFMorGFM and id != requiredId:
val.modelXbrl.error(
_("Taxonomy schema {0} element {1} id {2} should be {3}").format(
os.path.basename(modelDocument.uri),
name, id, requiredId),
"err", "EFM.6.07.17", "GFM.1.03.19")
# 6.7.18 nillable is true
nillable = eltName.getAttribute("nillable")
if nillable != "true":
val.modelXbrl.error(
_("Taxonomy schema {0} element {1} nillable {2} should be 'true'").format(
os.path.basename(modelDocument.uri),
name, nillable),
"err", "EFM.6.07.18", "GFM.1.03.20")
if modelConcept is not None:
# 6.7.19 not tuple
if modelConcept.isTuple:
if val.validateEFMorGFM:
val.modelXbrl.error(
_("Taxonomy schema {0} element {1} is a tuple").format(
os.path.basename(modelDocument.uri),
name),
"err", "EFM.6.07.19", "GFM.1.03.21")
# 6.7.20 no typed domain ref
if modelConcept.typedDomainRefQname is not None:
val.modelXbrl.error(
_("Taxonomy schema {0} element {1} has typedDomainRef {2}").format(
os.path.basename(modelDocument.uri),
name,
modelConcept.typedDomainRefQname),
"err", "EFM.6.07.20", "GFM.1.03.22")
# 6.7.21 abstract must be duration
isDuration = modelConcept.periodType == "duration"
if modelConcept.abstract == "true" and not isDuration:
val.modelXbrl.error(
_("Taxonomy schema {0} element {1} is abstract but period type is not duration").format(
os.path.basename(modelDocument.uri),
name),
"err", "EFM.6.07.21", "GFM.1.03.23")
# 6.7.22 abstract must be stringItemType
''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08
if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType:
val.modelXbrl.error(
_("Taxonomy schema {0} element {1} is abstract but type is not xbrli:stringItemType").format(
os.path.basename(modelDocument.uri),
name),
"err", "EFM.6.07.22", "GFM.1.03.24")
'''
substititutionGroupQname = modelConcept.substitutionGroupQname
# 6.7.23 Axis must be subs group dimension
if name.endswith("Axis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem):
val.modelXbrl.error(
_("Taxonomy schema {0} element {1} must end in Axis to be in dimensionItem substitution group").format(
os.path.basename(modelDocument.uri),
name),
"err", "EFM.6.07.23", "GFM.1.03.25")
# 6.7.24 Table must be subs group hypercube
if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem):
val.modelXbrl.error(
_("Taxonomy schema {0} element {1} is an Axis but not in hypercubeItem substitution group").format(
os.path.basename(modelDocument.uri),
name),
"err", "EFM.6.07.24", "GFM.1.03.26")
# 6.7.25 if neither hypercube or dimension, substitution group must be item
if substititutionGroupQname not in (None,
XbrlConst.qnXbrldtDimensionItem,
XbrlConst.qnXbrldtHypercubeItem,
XbrlConst.qnXbrliItem):
val.modelXbrl.error(
_("Taxonomy schema {0} element {1} has disallowed substitution group {2}").format(
os.path.basename(modelDocument.uri),
name, modelConcept.substitutionGroupQname),
"err", "EFM.6.07.25", "GFM.1.03.27")
# 6.7.26 Table must be subs group hypercube
if name.endswith("LineItems") and modelConcept.abstract != "true":
val.modelXbrl.error(
_("Taxonomy schema {0} element {1} is a LineItems but not abstract").format(
os.path.basename(modelDocument.uri),
name),
"err", "EFM.6.07.26", "GFM.1.03.28")
# 6.7.27 type domainMember must end with Domain or Member
conceptType = modelConcept.type
isDomainItemType = conceptType is not None and conceptType.isDomainItemType
endsWithDomainOrMember = name.endswith("Domain") or name.endswith("Member")
if isDomainItemType != endsWithDomainOrMember:
val.modelXbrl.error(
_("Taxonomy schema {0} element {1} must end with Domain or Member for type of domainItemType").format(
os.path.basename(modelDocument.uri),
name),
"err", "EFM.6.07.27", "GFM.1.03.29")
# 6.7.28 domainItemType must be duration
if isDomainItemType and not isDuration:
val.modelXbrl.error(
_("Taxonomy schema {0} element {1} is a domainItemType and must be periodType duration").format(
os.path.basename(modelDocument.uri),
name),
"err", "EFM.6.07.28", "GFM.1.03.30")
if val.validateSBRNL:
definesConcepts = True
if modelConcept.isTuple:
definesTuples = True
if modelConcept.abstract == "true":
val.modelXbrl.error(
_("Taxonomy schema {0} element {1} is an abstract tuple").format(
os.path.basename(modelDocument.uri), modelConcept.qname),
"err", "SBR.NL.2.2.2.03")
if tupleCycle(val,modelConcept):
val.modelXbrl.error(
_("Taxonomy schema {0} tuple {1} has a tuple cycle").format(
os.path.basename(modelDocument.uri), modelConcept.qname),
"err", "SBR.NL.2.2.2.07")
if modelConcept.nillable != "false" and modelConcept.isRoot:
val.modelXbrl.error(
_("Taxonomy schema {0} tuple {1} must have nillable='false'").format(
os.path.basename(modelDocument.uri), modelConcept.qname),
"err", "SBR.NL.2.2.2.17")
if modelConcept.abstract == "true":
if modelConcept.isRoot:
if modelConcept.nillable != "false":
val.modelXbrl.error(
_("Taxonomy schema {0} abstract root concept {1} must have nillable='false'").format(
os.path.basename(modelDocument.uri), modelConcept.qname),
"err", "SBR.NL.2.2.2.16")
if modelConcept.typeQname != XbrlConst.qnXbrliStringItemType:
val.modelXbrl.error(
_("Taxonomy schema {0} abstract root concept {1} must have type='xbrli:stringItemType'").format(
os.path.basename(modelDocument.uri), modelConcept.qname),
"err", "SBR.NL.2.2.2.21")
else: # not root
if modelConcept.isItem:
val.modelXbrl.error(
_("Taxonomy schema {0} abstract item {1} must not be a child of a tuple").format(
os.path.basename(modelDocument.uri), modelConcept.qname),
"err", "SBR.NL.2.2.2.31")
if modelConcept.balance:
val.modelXbrl.error(
_("Taxonomy schema {0} abstract concept {1} must not have a balance attribute").format(
os.path.basename(modelDocument.uri), modelConcept.qname),
"err", "SBR.NL.2.2.2.22")
if modelConcept.isTuple:
val.modelXbrl.error(
_("Taxonomy schema {0} tuple {1} must not be abstract").format(
os.path.basename(modelDocument.uri), modelConcept.qname),
"err", "SBR.NL.2.2.2.31")
if modelConcept.isHypercubeItem:
definesHypercubes = True
elif modelConcept.isDimensionItem:
definesDimensions = True
elif substititutionGroupQname.localName == "domainItem":
definesDomains = True
elif modelConcept.isItem:
definesAbstractItems = True
else: # not abstract
if not (modelConcept.label(preferredLabel=XbrlConst.documentationLabel,fallbackToQname=False,lang="nl") or
val.modelXbrl.relationshipSet(XbrlConst.conceptReference).fromModelObject(c)):
val.modelXbrl.error(
_("Taxonomy schema {0} {1} must have a documentation label or reference").format(
os.path.basename(modelDocument.uri), modelConcept.qname),
"err", "SBR.NL.2.2.2.28")
if modelConcept.balance and not modelConcept.instanceOfType(XbrlConst.qnXbrliMonetaryItemType):
val.modelXbrl.error(
_("Taxonomy schema {0} non-monetary concept {1} must not have a balance attribute").format(
os.path.basename(modelDocument.uri), modelConcept.qname),
"err", "SBR.NL.2.2.2.24")
if not modelConcept.label(fallbackToQname=False,lang="nl"):
val.modelXbrl.error(
_("Taxonomy schema {0} {1} must have a standard label in language 'nl'").format(
os.path.basename(modelDocument.uri), modelConcept.qname),
"err", "SBR.NL.2.2.2.26")
if not modelConcept.isRoot: # tuple child
if modelConcept.element.hasAttribute("maxOccurs") and modelConcept.element.getAttribute("maxOccurs") != "1":
val.modelXbrl.error(
_("Taxonomy schema {0} tuple concept {1} must have maxOccurs='1'").format(
os.path.basename(modelDocument.uri), modelConcept.qname),
"err", "SBR.NL.2.2.2.30")
if modelConcept.isLinkPart:
definesLinkParts = True
val.modelXbrl.error(
_("Taxonomy schema {0} link:part concept {1} is not allowed").format(
os.path.basename(modelDocument.uri), modelConcept.qname),
"err", "SBR.NL.2.2.5.01")
if modelConcept.isTypedDimension:
domainElt = modelConcept.typedDomainElement
if domainElt and domainElt.element and domainElt.element.localName == "complexType":
val.modelXbrl.error(
_("Taxonomy schema {0} typed dimension {1} domain element {2} has disallowed complex content").format(
os.path.basename(modelDocument.uri), modelConcept.qname, domainElt.qname),
"err", "SBR.NL.2.2.8.02")
# 6.7.8 check for embedded linkbase
for e in modelDocument.xmlRootElement.getElementsByTagNameNS(XbrlConst.link, "linkbase"):
val.modelXbrl.error(
_("Taxonomy schema {0} contains an embedded linkbase").format(
os.path.basename(modelDocument.uri)),
"err", "EFM.6.07.08", "GFM.1.03.08")
break
requiredUsedOns = {XbrlConst.qnLinkPresentationLink,
XbrlConst.qnLinkCalculationLink,
XbrlConst.qnLinkDefinitionLink}
# 6.7.9 role types authority
for e in modelDocument.xmlRootElement.getElementsByTagNameNS(XbrlConst.link, "roleType"):
roleURI = e.getAttribute("roleURI")
if targetNamespaceAuthority != UrlUtil.authority(roleURI):
val.modelXbrl.error(
_("Taxonomy schema {0} roleType {1} does not match authority {2}").format(
os.path.basename(modelDocument.uri),
roleURI,
targetNamespaceAuthority),
"err", "EFM.6.07.09", "GFM.1.03.09")
# 6.7.9 end with .../role/lc3 name
if not val.roleTypePattern.match(roleURI):
val.modelXbrl.error(
_("Taxonomy schema {0} roleType {1} should end with /role/{2}LC3name{3}").format(
os.path.basename(modelDocument.uri),
roleURI, '{', '}'),
"wrn", "EFM.6.07.09", "GFM.1.03.09")
# 6.7.10 only one role type declaration in DTS
modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI)
if modelRoleTypes is not None:
if len(modelRoleTypes) > 1:
val.modelXbrl.error(
_("Taxonomy schema {0} roleType {1} is defined in multiple taxonomies").format(
os.path.basename(modelDocument.uri),
roleURI),
"err", "EFM.6.07.10", "GFM.1.03.10")
elif len(modelRoleTypes) == 1:
# 6.7.11 used on's for pre, cal, def if any has a used on
usedOns = modelRoleTypes[0].usedOns
if not usedOns.isdisjoint(requiredUsedOns) and len(requiredUsedOns - usedOns) > 0:
val.modelXbrl.error(
_("Taxonomy schema {0} roleType {1} missing used on {2}").format(
os.path.basename(modelDocument.uri),
roleURI,
requiredUsedOns - usedOns),
"err", "EFM.6.07.11", "GFM.1.03.11")
# 6.7.12 definition match pattern
definition = modelRoleTypes[0].definitionNotStripped
if definition is None or not val.disclosureSystem.roleDefinitionPattern.match(definition):
val.modelXbrl.error(
_("Taxonomy schema {0} roleType {1} definition \"{2}\" must match {3}Sortcode{4} - {3}Type{4} - {3}Title{4}").format(
os.path.basename(modelDocument.uri),
roleURI, definition, '{', '}'),
"err", "EFM.6.07.12", "GFM.1.03.12-14")
if val.validateSBRNL and (usedOns & XbrlConst.standardExtLinkQnames):
definesLinkroles = True
# 6.7.13 arcrole types authority
for e in modelDocument.xmlRootElement.getElementsByTagNameNS(XbrlConst.link, "arcroleType"):
arcroleURI = e.getAttribute("arcroleURI")
if targetNamespaceAuthority != UrlUtil.authority(arcroleURI):
val.modelXbrl.error(
_("Taxonomy schema {0} arcroleType {1} does not match authority {2}").format(
os.path.basename(modelDocument.uri),
arcroleURI,
targetNamespaceAuthority),
"err", "EFM.6.07.13", "GFM.1.03.15")
# 6.7.13 end with .../arcrole/lc3 name
if not val.arcroleTypePattern.match(arcroleURI):
val.modelXbrl.error(
_("Taxonomy schema {0} arcroleType {1} should end with /arcrole/{2}LC3name{3}").format(
os.path.basename(modelDocument.uri),
arcroleURI, '{', '}'),
"wrn", "EFM.6.07.13", "GFM.1.03.15")
# 6.7.14 only one arcrole type declaration in DTS
modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI]
if len(modelRoleTypes) > 1:
val.modelXbrl.error(
_("Taxonomy schema {0} arcroleType {1} is defined in multiple taxonomies").format(
os.path.basename(modelDocument.uri),
arcroleURI),
"err", "EFM.6.07.14", "GFM.1.03.16")
# 6.7.15 definition match pattern
definition = modelRoleTypes[0].definition
if definition is None or not val.arcroleDefinitionPattern.match(definition):
val.modelXbrl.error(
_("Taxonomy schema {0} arcroleType {1} definition must be non-empty").format(
os.path.basename(modelDocument.uri),
arcroleURI, '{', '}'),
"err", "EFM.6.07.15", "GFM.1.03.17")
if val.validateSBRNL:
definesArcroles = True
if val.validateSBRNL and (definesLinkroles + definesArcroles + definesLinkParts +
definesAbstractItems + definesNonabstractItems + definesTuples +
definesEnumerations + definesDimensions + definesDomains +
definesHypercubes) > 1:
schemaContents = []
if definesLinkroles: schemaContents.append(_("linkroles"))
if definesArcroles: schemaContents.append(_("arcroles"))
if definesLinkParts: schemaContents.append(_("link parts"))
if definesAbstractItems: schemaContents.append(_("abstract items"))
if definesNonabstractItems: schemaContents.append(_("nonabstract items"))
if definesTuples: schemaContents.append(_("tuples"))
if definesEnumerations: schemaContents.append(_("enumerations"))
if definesDimensions: schemaContents.append(_("dimensions"))
if definesDomains: schemaContents.append(_("domains"))
if definesHypercubes: schemaContents.append(_("hypercubes"))
val.modelXbrl.error(
_("Taxonomy schema {0} may only define one of these: {1}").format(
os.path.basename(modelDocument.uri), ', '.join(schemaContents)),
"err", "SBR.NL.2.2.1.01")
visited.remove(modelDocument)
def relativeUri(self, uri): # return uri relative to this modelDocument uri
return UrlUtil.relativeUri(self.uri, uri)
def checkFilingDTS(val, modelDocument, isEFM, isGFM, visited):
global targetNamespaceDatePattern, efmFilenamePattern, htmlFileNamePattern, roleTypePattern, arcroleTypePattern, \
arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet
if targetNamespaceDatePattern is None:
targetNamespaceDatePattern = re.compile(r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|"
r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|")
efmFilenamePattern = re.compile(r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml|\.htm)$")
htmlFileNamePattern = re.compile(r"^[a-zA-Z0-9][._a-zA-Z0-9-]*(\.htm)$")
roleTypePattern = re.compile(r"^.*/role/[^/\s]+$")
arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$")
arcroleDefinitionPattern = re.compile(r"^.*[^\\s]+.*$") # at least one non-whitespace character
namePattern = re.compile("[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]") # u20ac=Euro, u00a3=pound sterling
linkroleDefinitionBalanceIncomeSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)",
re.IGNORECASE)
nonDomainItemNameProblemPattern = re.compile(
r"({0})|(FirstQuarter|SecondQuarter|ThirdQuarter|FourthQuarter|[1-4]Qtr|Qtr[1-4]|ytd|YTD|HalfYear)(?:$|[A-Z\W])"
.format(re.sub(r"\W", "", (val.entityRegistrantName or "").title())))
visited.append(modelDocument)
for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items():
#6.07.01 no includes
if modelDocumentReference.referenceType == "include":
val.modelXbrl.error(("EFM.6.07.01", "GFM.1.03.01"),
_("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"),
modelObject=modelDocumentReference.referringModelObject,
schema=modelDocument.basename,
include=referencedDocument.basename)
if referencedDocument not in visited and (referencedDocument.inDTS or referencedDocument.type == ModelDocument.Type.INLINEXBRLDOCUMENTSET): # ignore EdgarRenderer added non-DTS documents
checkFilingDTS(val, referencedDocument, isEFM, isGFM, visited)
if modelDocument.type == ModelDocument.Type.INLINEXBRLDOCUMENTSET:
return # nothing to check in inline document set surrogate parent
if val.disclosureSystem.standardTaxonomiesDict is None:
pass
if isEFM:
if modelDocument.uri in val.disclosureSystem.standardTaxonomiesDict:
if modelDocument.targetNamespace:
# check for duplicates of us-types, dei, and rr taxonomies
match = standardNamespacesPattern.match(modelDocument.targetNamespace)
if match is not None:
conflictClass = match.group(2) or match.group(5)
if (conflictClass == 'us-gaap' and match.group(3) < '2018') or conflictClass == 'srt':
val.standardNamespaceConflicts['srt+us-gaap'].add(modelDocument) # ignore non-srt multi-usgaap in Filing.py
if conflictClass == 'us-gaap' or conflictClass == 'ifrs-full':
val.standardNamespaceConflicts['ifrs+us-gaap'].add(modelDocument)
if conflictClass not in ('us-gaap', 'srt'):
val.standardNamespaceConflicts[conflictClass].add(modelDocument)
else:
if len(modelDocument.basename) > 32:
val.modelXbrl.error("EFM.5.01.01.tooManyCharacters",
_("Document file name %(filename)s must not exceed 32 characters."),
edgarCode="cp-0501-File-Name-Length",
modelObject=modelDocument, filename=modelDocument.basename)
if modelDocument.type != ModelDocument.Type.INLINEXBRLDOCUMENTSET:
if modelDocument.type == ModelDocument.Type.INLINEXBRL:
_pattern = htmlFileNamePattern
_suffix = ".htm"
else:
_pattern = efmFilenamePattern
_suffix = ".xsd or .xml"
if not _pattern.match(modelDocument.basename):
val.modelXbrl.error("EFM.5.01.01",
_("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with %(suffix)s."),
edgarCode="cp-0501-File-Name",
modelObject=modelDocument, filename=modelDocument.basename, suffix=_suffix)
if (modelDocument.type == ModelDocument.Type.SCHEMA and
modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and
modelDocument.uri.startswith(val.modelXbrl.uriDir)):
val.hasExtensionSchema = True
# check schema contents types
# 6.7.3 check namespace for standard authority
targetNamespaceAuthority = UrlUtil.authority(modelDocument.targetNamespace)
if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities:
val.modelXbrl.error(("EFM.6.07.03", "GFM.1.03.03"),
_("The target namespace, %(targetNamespace)s cannot have the same authority (%(targetNamespaceAuthority)s) as a standard "
"taxonomy, in %(schema)s. Please change your target namespace."),
edgarCode="fs-0703-Extension-Has-Standard-Namespace-Authority",
modelObject=modelDocument, schema=modelDocument.basename, targetNamespace=modelDocument.targetNamespace,
targetNamespaceAuthority=UrlUtil.authority(modelDocument.targetNamespace, includeScheme=False))
# 6.7.4 check namespace format
if modelDocument.targetNamespace is None or not modelDocument.targetNamespace.startswith("http://"):
match = None
else:
targetNamespaceDate = modelDocument.targetNamespace[len(targetNamespaceAuthority):]
match = targetNamespaceDatePattern.match(targetNamespaceDate)
if match is not None:
try:
if match.lastindex == 3:
date = datetime.date(int(match.group(1)),int(match.group(2)),int(match.group(3)))
elif match.lastindex == 6:
date = datetime.date(int(match.group(4)),int(match.group(5)),int(match.group(6)))
else:
match = None
except ValueError:
match = None
if match is None:
val.modelXbrl.error(("EFM.6.07.04", "GFM.1.03.04"),
_("You did not adhere to the requirements for target namespace, for %(targetNamespace)s in %(schema)s. "
"Please recheck your submission and adhere to the target namespace requirements."),
edgarCode="cp-0704-Taxonomy-Valid-Target-Namespace",
modelObject=modelDocument, schema=modelDocument.basename, targetNamespace=modelDocument.targetNamespace)
elif val.fileNameDate and date > val.fileNameDate:
val.modelXbrl.info(("EFM.6.07.06", "GFM.1.03.06"),
_("Warning: Taxonomy schema %(schema)s namespace %(targetNamespace)s has date later than document name date %(docNameDate)s"),
modelObject=modelDocument, schema=modelDocument.basename, targetNamespace=modelDocument.targetNamespace,
docNameDate=val.fileNameDate)
if modelDocument.targetNamespace is not None:
# 6.7.5 check prefix for _
authority = UrlUtil.authority(modelDocument.targetNamespace)
if not re.match(r"(http://|https://|ftp://|urn:)\w+",authority):
val.modelXbrl.error(("EFM.6.07.05", "GFM.1.03.05"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s must be a valid URI with a valid authority for the namespace."),
edgarCode="du-0705-Namespace-Authority",
modelObject=modelDocument, schema=modelDocument.basename, targetNamespace=modelDocument.targetNamespace)
# may be multiple prefixes for namespace
prefixes = [(prefix if prefix is not None else "")
for prefix, NS in modelDocument.xmlRootElement.nsmap.items()
if NS == modelDocument.targetNamespace]
if not prefixes:
prefix = None
val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"),
_("The schema does not supply a prefix for %(targetNamespace)s without an underscore character, in file %(schema)s. "
"Please provide or change a prefix"),
edgarCode="du-0707-Recommended-Prefix-Disallowed",
modelObject=modelDocument, schema=modelDocument.basename, targetNamespace=modelDocument.targetNamespace)
else:
prefix = prefixes[0]
if len(prefixes) > 1:
val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"),
_("The schema does not supply a prefix for %(targetNamespace)s without an underscore character, in file %(schema)s. "
"Please provide or change a prefix"),
edgarCode="du-0707-Recommended-Prefix-Disallowed",
modelObject=modelDocument, schema=modelDocument.basename, targetNamespace=modelDocument.targetNamespace, prefix=", ".join(prefixes))
elif "_" in prefix:
val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"),
_("The schema does not supply a prefix for %(targetNamespace)s without an underscore character, in file %(schema)s. "
"Please provide or change a prefix"),
edgarCode="du-0707-Recommended-Prefix-Disallowed",
modelObject=modelDocument, schema=modelDocument.basename, targetNamespace=modelDocument.targetNamespace, prefix=prefix)
for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"):
if isinstance(modelConcept,ModelConcept):
# 6.7.16 name not duplicated in standard taxonomies
name = modelConcept.get("name")
if name is None:
name = ""
if modelConcept.get("ref") is not None:
continue # don't validate ref's here
for c in val.modelXbrl.nameConcepts.get(name, []):
if c.modelDocument != modelDocument:
if not c.modelDocument.uri.startswith(val.modelXbrl.uriDir):
val.modelXbrl.error(("EFM.6.07.16", "GFM.1.03.18"),
_("Your extension taxonomy contains an element, %(concept)s, which has the same name as an element in the base taxonomy, "
"%(standardConcept)s. Please ensure that this extension is appropriate and if so, please change the extension concept name."),
edgarCode="cp-0716-Element-Name-Same-As-Base",
modelObject=(modelConcept,c), concept=modelConcept.qname, standardSchema=c.modelDocument.basename, standardConcept=c.qname)
# 6.7.17 id properly formed
_id = modelConcept.id
requiredId = (prefix if prefix is not None else "") + "_" + name
if _id != requiredId:
val.modelXbrl.error(("EFM.6.07.17", "GFM.1.03.19"),
_("You did not adhere to the declarations for concepts by containing an 'id' attribute whose value begins with the recommended "
"namespace prefix of the taxonomy, followed by an underscore, followed by an element name, for the concept %(concept)s. "
"Please recheck your submission."),
edgarCode="cp-0717-Element-Id",
modelObject=modelConcept, concept=modelConcept.qname, id=_id, requiredId=requiredId)
# 6.7.18 nillable is true
nillable = modelConcept.get("nillable")
if nillable != "true" and modelConcept.isItem:
val.modelXbrl.error(("EFM.6.07.18", "GFM.1.03.20"),
_("Element %(concept)s is declared without a 'true' value for the nillable attribute. Please set the value to 'true'."),
edgarCode="du-0718-Nillable-Not-True",
modelObject=modelConcept, schema=modelDocument.basename,
concept=name, nillable=nillable)
# 6.7.19 not tuple
if modelConcept.isTuple:
val.modelXbrl.error(("EFM.6.07.19", "GFM.1.03.21"),
_("You provided an extension concept which is a tuple, %(concept)s. Please remove tuples and check your submission."),
edgarCode="cp-0719-No-Tuple-Element",
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.20 no typed domain ref
if modelConcept.isTypedDimension:
val.modelXbrl.error(("EFM.6.07.20", "GFM.1.03.22"),
_("There is an xbrldt:typedDomainRef attribute on %(concept)s (%(typedDomainRef)s). Please remove it."),
edgarCode="du-0720-Typed-Domain-Ref-Disallowed",
modelObject=modelConcept, concept=modelConcept.qname,
typedDomainRef=modelConcept.typedDomainElement.qname if modelConcept.typedDomainElement is not None else modelConcept.typedDomainRef)
# 6.7.21 abstract must be duration
isDuration = modelConcept.periodType == "duration"
if modelConcept.isAbstract and not isDuration:
val.modelXbrl.error(("EFM.6.07.21", "GFM.1.03.23"),
_("Element %(concept)s is declared as an abstract item with period type 'instant'. Please change it to 'duration' or "
"make the element not abstract."),
edgarCode="du-0721-Abstract-Is-Instant",
modelObject=modelConcept, schema=modelDocument.basename, concept=modelConcept.qname)
# 6.7.22 abstract must be stringItemType
''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08
if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType:
val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"),
_("Concept %(concept)s is abstract but type is not xbrli:stringItemType"),
modelObject=modelConcept, concept=modelConcept.qname)
'''
substitutionGroupQname = modelConcept.substitutionGroupQname
# 6.7.23 Axis must be subs group dimension
if name.endswith("Axis") ^ (substitutionGroupQname == XbrlConst.qnXbrldtDimensionItem):
val.modelXbrl.error(("EFM.6.07.23", "GFM.1.03.25"),
_("The substitution group 'xbrldt:dimensionItem' is only consistent with an element name that ends with 'Axis'. "
"Please change %(conceptLocalName)s or change the substitutionGroup."),
edgarCode="du-0723-Axis-Dimension-Name-Mismatch",
modelObject=modelConcept, concept=modelConcept.qname, conceptLocalName=modelConcept.qname.localName)
# 6.7.24 Table must be subs group hypercube
if name.endswith("Table") ^ (substitutionGroupQname == XbrlConst.qnXbrldtHypercubeItem):
val.modelXbrl.error(("EFM.6.07.24", "GFM.1.03.26"),
_("The substitution group 'xbrldt:hypercubeItem' is only allowed with an element name that ends with 'Table'. "
"Please change %(conceptLocalName)s or change the substitutionGroup."),
edgarCode="du-0724-Table-Hypercube-Name-Mismatch",
modelObject=modelConcept, schema=modelDocument.basename, concept=modelConcept.qname, conceptLocalName=modelConcept.qname.localName)
# 6.7.25 if neither hypercube or dimension, substitution group must be item
if substitutionGroupQname not in (None,
XbrlConst.qnXbrldtDimensionItem,
XbrlConst.qnXbrldtHypercubeItem,
XbrlConst.qnXbrliItem):
val.modelXbrl.error(("EFM.6.07.25", "GFM.1.03.27"),
_("The substitution group attribute value %(substitutionGroup)s of element %(conceptLocalName)s is not allowed. "
"Please change it to one of 'xbrli:item', 'xbrldt:dimensionItem' or 'xbrldt:hypercubeItem'."),
edgarCode="du-0725-Substitution-Group-Custom",
modelObject=modelConcept, concept=modelConcept.qname, conceptLocalName=modelConcept.qname.localName,
substitutionGroup=modelConcept.substitutionGroupQname)
# 6.7.26 Table must be subs group hypercube
if name.endswith("LineItems") and modelConcept.abstract != "true":
val.modelXbrl.error(("EFM.6.07.26", "GFM.1.03.28"),
_("The element %(conceptLocalName)s ends with 'LineItems' but is not abstract. Please change %(conceptLocalName)s or "
"the value of the 'abstract' attribute."),
edgarCode="du-0726-LineItems-Abstract-Name-Mismatch",
modelObject=modelConcept, concept=modelConcept.qname, conceptLocalName=modelConcept.qname.localName)
# 6.7.27 type domainMember must end with Domain or Member
conceptType = modelConcept.type
isDomainItemType = conceptType is not None and conceptType.isDomainItemType
endsWithDomainOrMember = name.endswith("Domain") or name.endswith("Member")
if isDomainItemType != endsWithDomainOrMember:
val.modelXbrl.error(("EFM.6.07.27", "GFM.1.03.29"),
_("The type 'us-types:domainItemType' is only allowed with an element name that ends with 'Domain' or 'Member'. "
"Please change %(conceptLocalName)s or change the type."),
edgarCode="du-0727-Domain-Type-Name-Mismatch",
modelObject=modelConcept, concept=modelConcept.qname, conceptLocalName=modelConcept.qname.localName)
# 6.7.28 domainItemType must be duration
if isDomainItemType and not isDuration:
val.modelXbrl.error(("EFM.6.07.28", "GFM.1.03.30"),
_("Element %(conceptLocalName)s is declared as a us-types:domainItemType with period type 'instant'. "
"Please change it to 'duration' or change the item type."),
edgarCode="du-0728-Domain-Member-Is-Instant",
modelObject=modelConcept, concept=modelConcept.qname, conceptLocalName=modelConcept.qname.localName)
#6.7.31 (version 27) fractions
if modelConcept.isFraction:
val.modelXbrl.error("EFM.6.07.31",
_("Element %(concept)s is declared as a fraction item type. Change or remove the declaration."),
edgarCode="du-0731-Fraction-Item-Type",
modelObject=modelConcept, concept=modelConcept.qname)
#6.7.32 (version 27) instant non numeric
if modelConcept.isItem and (not modelConcept.isNumeric and not isDuration and not modelConcept.isAbstract and not isDomainItemType):
val.modelXbrl.error("EFM.6.07.32",
_("Declaration of element %(concept)s in %(schema)s must have xbrli:periodType of 'duration' because its base type is not numeric."),
edgarCode="rq-0732-Nonnnumeric-Must-Be-Duration",
modelObject=modelConcept, schema=modelDocument.basename, concept=modelConcept.qname)
# 6.8.5 semantic check, check LC3 name
if name:
if not name[0].isupper():
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.firstLetter", "GFM.2.03.05.firstLetter"),
_("Concept %(concept)s name must start with a capital letter"),
modelObject=modelConcept, concept=modelConcept.qname)
if namePattern.search(name):
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.disallowedCharacter", "GFM.2.03.05.disallowedCharacter"),
_("Concept %(concept)s has disallowed name character"),
modelObject=modelConcept, concept=modelConcept.qname)
if len(name) > 200:
val.modelXbrl.log("ERROR-SEMANTIC", "EFM.6.08.05.nameLength",
_("Concept %(concept)s name length %(namelength)s exceeds 200 characters"),
modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name))
if isEFM:
label = modelConcept.label(lang="en-US", fallbackToQname=False)
if label:
# allow Joe's Bar, N.A. to be JoesBarNA -- remove ', allow A. as not article "a"
lc3name = ''.join(re.sub(r"['.-]", "", (w[0] or w[2] or w[3] or w[4])).title()
for w in re.findall(r"((\w+')+\w+)|(A[.-])|([.-]A(?=\W|$))|(\w+)", label) # EFM implies this should allow - and . re.findall(r"[\w\-\.]+", label)
if w[4].lower() not in ("the", "a", "an"))
if not(name == lc3name or
(name and lc3name and lc3name[0].isdigit() and name[1:] == lc3name and (name[0].isalpha() or name[0] == '_'))):
val.modelXbrl.log("WARNING-SEMANTIC", "EFM.6.08.05.LC3",
_("Concept %(concept)s should match expected LC3 composition %(lc3name)s"),
modelObject=modelConcept, concept=modelConcept.qname, lc3name=lc3name)
if conceptType is not None:
# 6.8.6 semantic check
if not isDomainItemType and conceptType.qname != XbrlConst.qnXbrliDurationItemType:
nameProblems = nonDomainItemNameProblemPattern.findall(name)
if any(any(t) for t in nameProblems): # list of tuples with possibly nonempty strings
val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.08.06", "GFM.2.03.06"),
_("Concept %(concept)s should not contain company or period information, found: %(matches)s"),
modelObject=modelConcept, concept=modelConcept.qname,
matches=", ".join(''.join(t) for t in nameProblems))
if conceptType.qname == XbrlConst.qnXbrliMonetaryItemType:
if not modelConcept.balance:
# 6.8.11 may not appear on a income or balance statement
if any(linkroleDefinitionBalanceIncomeSheet.match(roleType.definition)
for rel in val.modelXbrl.relationshipSet(XbrlConst.parentChild).toModelObject(modelConcept)
for roleType in val.modelXbrl.roleTypes.get(rel.linkrole,())):
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.11", "GFM.2.03.11"),
_("Concept %(concept)s must have a balance because it appears in a statement of income or balance sheet"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.11.5 semantic check, must have a documentation label
stdLabel = modelConcept.label(lang="en-US", fallbackToQname=False)
defLabel = modelConcept.label(preferredLabel=XbrlConst.documentationLabel, lang="en-US", fallbackToQname=False)
if not defLabel or ( # want different words than std label
stdLabel and re.findall(r"\w+", stdLabel) == re.findall(r"\w+", defLabel)):
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.11.05", "GFM.2.04.04"),
_("Concept %(concept)s is monetary without a balance and must have a documentation label that disambiguates its sign"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.8.16 semantic check
if conceptType.qname == XbrlConst.qnXbrliDateItemType and modelConcept.periodType != "duration":
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.16", "GFM.2.03.16"),
_("Concept %(concept)s of type xbrli:dateItemType must have periodType duration"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.8.17 semantic check
if conceptType.qname == XbrlConst.qnXbrliStringItemType and modelConcept.periodType != "duration":
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.17", "GFM.2.03.17"),
_("Concept %(concept)s of type xbrli:stringItemType must have periodType duration"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.8 check for embedded linkbase
for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
if isinstance(e,ModelObject):
val.modelXbrl.error(("EFM.6.07.08", "GFM.1.03.08"),
_("Your filing contained embedded linkbases in %(schema)s. Please recheck your submission and remove all embedded linkbases."),
edgarCode="cp-0708-No-Embedded-Linkbases",
modelObject=e, schema=modelDocument.basename)
break
requiredUsedOns = {XbrlConst.qnLinkPresentationLink,
XbrlConst.qnLinkCalculationLink,
XbrlConst.qnLinkDefinitionLink}
standardUsedOns = {XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference,
XbrlConst.qnLinkDefinitionArc, XbrlConst.qnLinkCalculationArc, XbrlConst.qnLinkPresentationArc,
XbrlConst.qnLinkLabelArc, XbrlConst.qnLinkReferenceArc,
# per WH, private footnote arc and footnore resource roles are not allowed
XbrlConst.qnLinkFootnoteArc, XbrlConst.qnLinkFootnote,
}
# 6.7.9 role types authority
for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}roleType"):
if isinstance(e,ModelObject):
roleURI = e.get("roleURI")
if targetNamespaceAuthority != UrlUtil.authority(roleURI):
val.modelXbrl.error(("EFM.6.07.09", "GFM.1.03.09"),
_("Role %(roleType)s does not begin with %(targetNamespace)s's scheme and authority. "
"Please change the role URI or target namespace URI."),
edgarCode="du-0709-Role-Namespace-Mismatch",
modelObject=e, roleType=roleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace)
# 6.7.9 end with .../role/lc3 name
if not roleTypePattern.match(roleURI):
val.modelXbrl.warning(("EFM.6.07.09.roleEnding", "GFM.1.03.09"),
"RoleType %(roleType)s should end with /role/{LC3name}",
modelObject=e, roleType=roleURI)
# 6.7.10 only one role type declaration in DTS
modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI)
if modelRoleTypes is not None:
modelRoleType = modelRoleTypes[0]
definition = modelRoleType.definitionNotStripped
usedOns = modelRoleType.usedOns
if len(modelRoleTypes) == 1:
# 6.7.11 used on's for pre, cal, def if any has a used on
if not usedOns.isdisjoint(requiredUsedOns) and len(requiredUsedOns - usedOns) > 0:
val.modelXbrl.error(("EFM.6.07.11", "GFM.1.03.11"),
_("The role %(roleType)s did not provide a usedOn element for all three link types (presentation, "
"calculation and definition), missing %(usedOn)s. Change the declaration to be for all three types of link, and resubmit."),
edgarCode="du-0711-Role-Type-Declaration-Incomplete",
modelObject=e, roleType=roleURI, usedOn=requiredUsedOns - usedOns)
# 6.7.12 definition match pattern
if (val.disclosureSystem.roleDefinitionPattern is not None and
(definition is None or not val.disclosureSystem.roleDefinitionPattern.match(definition))):
val.modelXbrl.error(("EFM.6.07.12", "GFM.1.03.12-14"),
_("The definition '%(definition)s' of role %(roleType)s does not match the expected format. "
"Please check that the definition matches {number} - {type} - {text}."),
edgarCode="rq-0712-Role-Definition-Mismatch",
modelObject=e, roleType=roleURI, definition=(definition or ""))
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("RoleType %(roleuri)s is defined using role types already defined by standard roles for: %(qnames)s"),
modelObject=e, roleuri=roleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns))
# 6.7.13 arcrole types authority
for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}arcroleType"):
if isinstance(e,ModelObject):
arcroleURI = e.get("arcroleURI")
if targetNamespaceAuthority != UrlUtil.authority(arcroleURI):
val.modelXbrl.error(("EFM.6.07.13", "GFM.1.03.15"),
_("Relationship role %(arcroleType)s does not begin with %(targetNamespace)s's scheme and authority. "
"Please change the relationship role URI or target namespace URI."),
edgarCode="du-0713-Arcrole-Namespace-Mismatch",
modelObject=e, arcroleType=arcroleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace)
# 6.7.13 end with .../arcrole/lc3 name
if not arcroleTypePattern.match(arcroleURI):
val.modelXbrl.warning(("EFM.6.07.13.arcroleEnding", "GFM.1.03.15"),
_("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"),
modelObject=e, arcroleType=arcroleURI)
# 6.7.15 definition match pattern
modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI]
definition = modelRoleTypes[0].definition
if definition is None or not arcroleDefinitionPattern.match(definition):
val.modelXbrl.error(("EFM.6.07.15", "GFM.1.03.17"),
_("Relationship role declaration %(arcroleType)s is missing a definition. Please provide a definition."),
edgarCode="du-0715-Arcrole-Definition-Missing",
modelObject=e, arcroleType=arcroleURI)
# semantic checks
usedOns = modelRoleTypes[0].usedOns
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("ArcroleType %(arcroleuri)s is defined using role types already defined by standard arcroles for: %(qnames)s"),
modelObject=e, arcroleuri=arcroleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns))
#6.3.3 filename check
m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9]).xsd$", modelDocument.basename)
if m:
try: # check date value
datetime.datetime.strptime(m.group(1),"%Y%m%d").date()
# date and format are ok, check "should" part of 6.3.3
if val.fileNameBasePart:
expectedFilename = "{0}-{1}.xsd".format(val.fileNameBasePart, val.fileNameDatePart)
if modelDocument.basename != expectedFilename:
val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"),
_('Schema file name warning: %(filename)s, should match %(expectedFilename)s'),
modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename)
except ValueError:
val.modelXbrl.error((val.EFM60303, "GFM.1.01.01"),
_('Invalid schema file base name part (date) in "{base}-{yyyymmdd}.xsd": %(filename)s'),
modelObject=modelDocument, filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
else:
val.modelXbrl.error((val.EFM60303, "GFM.1.01.01"),
_('Invalid schema file name, must match "{base}-{yyyymmdd}.xsd": %(filename)s'),
modelObject=modelDocument, filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
elif modelDocument.type == ModelDocument.Type.LINKBASE:
# if it is part of the submission (in same directory) check name
labelRels = None
if modelDocument.filepath.startswith(val.modelXbrl.modelDocument.filepathdir):
#6.3.3 filename check
extLinkElt = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended")
if extLinkElt is None:# no ext link element
val.modelXbrl.error((val.EFM60303 + ".noLinkElement", "GFM.1.01.01.noLinkElement"),
_('Invalid linkbase file name: %(filename)s, has no extended link element, cannot determine link type.'),
modelObject=modelDocument, filename=modelDocument.basename,
messageCodes=("EFM.6.03.03.noLinkElement", "EFM.6.23.01.noLinkElement", "GFM.1.01.01.noLinkElement"))
elif extLinkElt.localName not in extLinkEltFileNameEnding:
val.modelXbrl.error("EFM.6.03.02",
_('Invalid linkbase link element %(linkElement)s in %(filename)s'),
modelObject=modelDocument, linkElement=extLinkElt.localName, filename=modelDocument.basename)
else:
m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9])(_[a-z]{3}).xml$", modelDocument.basename)
expectedSuffix = extLinkEltFileNameEnding[extLinkElt.localName]
if m and m.group(2) == expectedSuffix:
try: # check date value
datetime.datetime.strptime(m.group(1),"%Y%m%d").date()
# date and format are ok, check "should" part of 6.3.3
if val.fileNameBasePart:
expectedFilename = "{0}-{1}{2}.xml".format(val.fileNameBasePart, val.fileNameDatePart, expectedSuffix)
if modelDocument.basename != expectedFilename:
val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"),
_('Linkbase name warning: %(filename)s should match %(expectedFilename)s'),
modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename)
except ValueError:
val.modelXbrl.error((val.EFM60303, "GFM.1.01.01"),
_('Invalid linkbase base file name part (date) in "{base}-{yyyymmdd}_{suffix}.xml": %(filename)s'),
modelObject=modelDocument, filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
else:
val.modelXbrl.error((val.EFM60303, "GFM.1.01.01"),
_('Invalid linkbase name, must match "{base}-{yyyymmdd}%(expectedSuffix)s.xml": %(filename)s'),
modelObject=modelDocument, filename=modelDocument.basename, expectedSuffix=expectedSuffix,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
if extLinkElt.localName == "labelLink":
if labelRels is None:
labelRels = val.modelXbrl.relationshipSet(XbrlConst.conceptLabel)
for labelElt in XmlUtil.children(extLinkElt, XbrlConst.link, "label"):
# 6.10.9
if XbrlConst.isNumericRole(labelElt.role):
for rel in labelRels.toModelObject(labelElt):
if rel.fromModelObject is not None and not rel.fromModelObject.isNumeric:
val.modelXbrl.error("EFM.6.10.09",
_("Non-numeric element %(concept)s has a label role for numeric elements: %(role)s. "
"Please change the role attribute."),
edgarCode="du-1009-Numeric-Label-Role",
modelObject=(labelElt, rel.fromModelObject), concept=rel.fromModelObject.qname, role=labelElt.role)
def checkFilingDTS(val, modelDocument, isEFM, isGFM, visited):
global targetNamespaceDatePattern, efmFilenamePattern, htmlFileNamePattern, roleTypePattern, arcroleTypePattern, \
arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet, \
usNamespacesConflictPattern, ifrsNamespacesConflictPattern
if targetNamespaceDatePattern is None:
targetNamespaceDatePattern = re.compile(r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|"
r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|")
efmFilenamePattern = re.compile(r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml|\.htm)$")
htmlFileNamePattern = re.compile(r"^[a-zA-Z0-9][._a-zA-Z0-9-]*(\.htm)$")
roleTypePattern = re.compile(r"^.*/role/[^/\s]+$")
arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$")
arcroleDefinitionPattern = re.compile(r"^.*[^\\s]+.*$") # at least one non-whitespace character
namePattern = re.compile("[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]") # u20ac=Euro, u00a3=pound sterling
linkroleDefinitionBalanceIncomeSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)",
re.IGNORECASE)
usNamespacesConflictPattern = re.compile(r"http://(xbrl\.us|fasb\.org|xbrl\.sec\.gov)/(dei|us-types|us-roles|rr)/([0-9]{4}-[0-9]{2}-[0-9]{2})$")
ifrsNamespacesConflictPattern = re.compile(r"http://xbrl.ifrs.org/taxonomy/([0-9]{4}-[0-9]{2}-[0-9]{2})/(ifrs[\w-]*)$")
nonDomainItemNameProblemPattern = re.compile(
r"({0})|(FirstQuarter|SecondQuarter|ThirdQuarter|FourthQuarter|[1-4]Qtr|Qtr[1-4]|ytd|YTD|HalfYear)(?:$|[A-Z\W])"
.format(re.sub(r"\W", "", (val.entityRegistrantName or "").title())))
visited.append(modelDocument)
for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items():
#6.07.01 no includes
if modelDocumentReference.referenceType == "include":
val.modelXbrl.error(("EFM.6.07.01", "GFM.1.03.01"),
_("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"),
modelObject=modelDocumentReference.referringModelObject,
schema=os.path.basename(modelDocument.uri),
include=os.path.basename(referencedDocument.uri))
if referencedDocument not in visited and referencedDocument.inDTS: # ignore EdgarRenderer added non-DTS documents
checkFilingDTS(val, referencedDocument, isEFM, isGFM, visited)
if val.disclosureSystem.standardTaxonomiesDict is None:
pass
if isEFM:
if modelDocument.uri in val.disclosureSystem.standardTaxonomiesDict:
if modelDocument.targetNamespace:
# check for duplicates of us-types, dei, and rr taxonomies
for pattern, indexGroup in ((usNamespacesConflictPattern,2),(ifrsNamespacesConflictPattern,2)):
match = pattern.match(modelDocument.targetNamespace)
if match is not None:
val.standardNamespaceConflicts[match.group(indexGroup)].add(modelDocument)
else:
if len(modelDocument.basename) > 32:
val.modelXbrl.error("EFM.5.01.01.tooManyCharacters",
_("Document file name %(filename)s must not exceed 32 characters."),
modelObject=modelDocument, filename=modelDocument.basename)
if modelDocument.type == ModelDocument.Type.INLINEXBRL:
if not htmlFileNamePattern.match(modelDocument.basename):
val.modelXbrl.error("EFM.5.01.01",
_("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with .htm."),
modelObject=modelDocument, filename=modelDocument.basename)
elif not efmFilenamePattern.match(modelDocument.basename):
val.modelXbrl.error("EFM.5.01.01",
_("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with .xsd or .xml."),
modelObject=modelDocument, filename=modelDocument.basename)
if (modelDocument.type == ModelDocument.Type.SCHEMA and
modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and
modelDocument.uri.startswith(val.modelXbrl.uriDir)):
val.hasExtensionSchema = True
# check schema contents types
# 6.7.3 check namespace for standard authority
targetNamespaceAuthority = UrlUtil.authority(modelDocument.targetNamespace)
if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities:
val.modelXbrl.error(("EFM.6.07.03", "GFM.1.03.03"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s is a disallowed authority"),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace,
targetNamespaceAuthority=UrlUtil.authority(modelDocument.targetNamespace, includeScheme=False))
# 6.7.4 check namespace format
if modelDocument.targetNamespace is None or not modelDocument.targetNamespace.startswith("http://"):
match = None
else:
targetNamespaceDate = modelDocument.targetNamespace[len(targetNamespaceAuthority):]
match = targetNamespaceDatePattern.match(targetNamespaceDate)
if match is not None:
try:
if match.lastindex == 3:
date = datetime.date(int(match.group(1)),int(match.group(2)),int(match.group(3)))
elif match.lastindex == 6:
date = datetime.date(int(match.group(4)),int(match.group(5)),int(match.group(6)))
else:
match = None
except ValueError:
match = None
if match is None:
val.modelXbrl.error(("EFM.6.07.04", "GFM.1.03.04"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s must have format http://{authority}/{versionDate}"),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
elif val.fileNameDate and date > val.fileNameDate:
val.modelXbrl.info(("EFM.6.07.06", "GFM.1.03.06"),
_("Warning: Taxonomy schema %(schema)s namespace %(targetNamespace)s has date later than document name date %(docNameDate)s"),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace,
docNameDate=val.fileNameDate)
if modelDocument.targetNamespace is not None:
# 6.7.5 check prefix for _
authority = UrlUtil.authority(modelDocument.targetNamespace)
if not re.match(r"(http://|https://|ftp://|urn:)\w+",authority):
val.modelXbrl.error(("EFM.6.07.05", "GFM.1.03.05"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s must be a valid URL with a valid authority for the namespace."),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement,modelDocument.targetNamespace)
if not prefix:
val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s missing prefix for the namespace."),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
elif "_" in prefix:
val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s prefix %(prefix)s must not have an '_'"),
modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, prefix=prefix)
for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"):
if isinstance(modelConcept,ModelConcept):
# 6.7.16 name not duplicated in standard taxonomies
name = modelConcept.get("name")
if name is None:
name = ""
if modelConcept.get("ref") is not None:
continue # don't validate ref's here
for c in val.modelXbrl.nameConcepts.get(name, []):
if c.modelDocument != modelDocument:
if not c.modelDocument.uri.startswith(val.modelXbrl.uriDir):
val.modelXbrl.error(("EFM.6.07.16", "GFM.1.03.18"),
_("Concept %(concept)s is also defined in standard taxonomy schema schema %(standardSchema)s"),
modelObject=(modelConcept,c), concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri), standardConcept=c.qname)
# 6.7.17 id properly formed
_id = modelConcept.id
requiredId = (prefix if prefix is not None else "") + "_" + name
if _id != requiredId:
val.modelXbrl.error(("EFM.6.07.17", "GFM.1.03.19"),
_("Concept %(concept)s id %(id)s should be %(requiredId)s"),
modelObject=modelConcept, concept=modelConcept.qname, id=_id, requiredId=requiredId)
# 6.7.18 nillable is true
nillable = modelConcept.get("nillable")
if nillable != "true" and modelConcept.isItem:
val.modelXbrl.error(("EFM.6.07.18", "GFM.1.03.20"),
_("Taxonomy schema %(schema)s element %(concept)s nillable %(nillable)s should be 'true'"),
modelObject=modelConcept, schema=os.path.basename(modelDocument.uri),
concept=name, nillable=nillable)
# 6.7.19 not tuple
if modelConcept.isTuple:
val.modelXbrl.error(("EFM.6.07.19", "GFM.1.03.21"),
_("Concept %(concept)s is a tuple"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.20 no typed domain ref
if modelConcept.isTypedDimension:
val.modelXbrl.error(("EFM.6.07.20", "GFM.1.03.22"),
_("Concept %(concept)s has typedDomainRef %(typedDomainRef)s"),
modelObject=modelConcept, concept=modelConcept.qname,
typedDomainRef=modelConcept.typedDomainElement.qname if modelConcept.typedDomainElement is not None else modelConcept.typedDomainRef)
# 6.7.21 abstract must be duration
isDuration = modelConcept.periodType == "duration"
if modelConcept.isAbstract and not isDuration:
val.modelXbrl.error(("EFM.6.07.21", "GFM.1.03.23"),
_("Taxonomy schema %(schema)s element %(concept)s is abstract but period type is not duration"),
modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname)
# 6.7.22 abstract must be stringItemType
''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08
if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType:
val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"),
_("Concept %(concept)s is abstract but type is not xbrli:stringItemType"),
modelObject=modelConcept, concept=modelConcept.qname)
'''
substitutionGroupQname = modelConcept.substitutionGroupQname
# 6.7.23 Axis must be subs group dimension
if name.endswith("Axis") ^ (substitutionGroupQname == XbrlConst.qnXbrldtDimensionItem):
val.modelXbrl.error(("EFM.6.07.23", "GFM.1.03.25"),
_("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.24 Table must be subs group hypercube
if name.endswith("Table") ^ (substitutionGroupQname == XbrlConst.qnXbrldtHypercubeItem):
val.modelXbrl.error(("EFM.6.07.24", "GFM.1.03.26"),
_("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"),
modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname)
# 6.7.25 if neither hypercube or dimension, substitution group must be item
if substitutionGroupQname not in (None,
XbrlConst.qnXbrldtDimensionItem,
XbrlConst.qnXbrldtHypercubeItem,
XbrlConst.qnXbrliItem):
val.modelXbrl.error(("EFM.6.07.25", "GFM.1.03.27"),
_("Concept %(concept)s has disallowed substitution group %(substitutionGroup)s"),
modelObject=modelConcept, concept=modelConcept.qname,
substitutionGroup=modelConcept.substitutionGroupQname)
# 6.7.26 Table must be subs group hypercube
if name.endswith("LineItems") and modelConcept.abstract != "true":
val.modelXbrl.error(("EFM.6.07.26", "GFM.1.03.28"),
_("Concept %(concept)s is a LineItems but not abstract"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.27 type domainMember must end with Domain or Member
conceptType = modelConcept.type
isDomainItemType = conceptType is not None and conceptType.isDomainItemType
endsWithDomainOrMember = name.endswith("Domain") or name.endswith("Member")
if isDomainItemType != endsWithDomainOrMember:
val.modelXbrl.error(("EFM.6.07.27", "GFM.1.03.29"),
_("Concept %(concept)s must end with Domain or Member for type of domainItemType"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.28 domainItemType must be duration
if isDomainItemType and not isDuration:
val.modelXbrl.error(("EFM.6.07.28", "GFM.1.03.30"),
_("Concept %(concept)s is a domainItemType and must be periodType duration"),
modelObject=modelConcept, concept=modelConcept.qname)
#6.7.31 (version 27) fractions
if modelConcept.isFraction:
val.modelXbrl.error("EFM.6.07.31",
_("Concept %(concept)s is a fraction"),
modelObject=modelConcept, concept=modelConcept.qname)
#6.7.32 (version 27) instant non numeric
if modelConcept.isItem and (not modelConcept.isNumeric and not isDuration and not modelConcept.isAbstract and not isDomainItemType):
val.modelXbrl.error("EFM.6.07.32",
_("Taxonomy schema %(schema)s element %(concept)s is non-numeric but period type is not duration"),
modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname)
# 6.8.5 semantic check, check LC3 name
if name:
if not name[0].isupper():
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.firstLetter", "GFM.2.03.05.firstLetter"),
_("Concept %(concept)s name must start with a capital letter"),
modelObject=modelConcept, concept=modelConcept.qname)
if namePattern.search(name):
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.disallowedCharacter", "GFM.2.03.05.disallowedCharacter"),
_("Concept %(concept)s has disallowed name character"),
modelObject=modelConcept, concept=modelConcept.qname)
if len(name) > 200:
val.modelXbrl.log("ERROR-SEMANTIC", "EFM.6.08.05.nameLength",
_("Concept %(concept)s name length %(namelength)s exceeds 200 characters"),
modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name))
if isEFM:
label = modelConcept.label(lang="en-US", fallbackToQname=False)
if label:
# allow Joe's Bar, N.A. to be JoesBarNA -- remove ', allow A. as not article "a"
lc3name = ''.join(re.sub(r"['.-]", "", (w[0] or w[2] or w[3] or w[4])).title()
for w in re.findall(r"((\w+')+\w+)|(A[.-])|([.-]A(?=\W|$))|(\w+)", label) # EFM implies this should allow - and . re.findall(r"[\w\-\.]+", label)
if w[4].lower() not in ("the", "a", "an"))
if not(name == lc3name or
(name and lc3name and lc3name[0].isdigit() and name[1:] == lc3name and (name[0].isalpha() or name[0] == '_'))):
val.modelXbrl.log("WARNING-SEMANTIC", "EFM.6.08.05.LC3",
_("Concept %(concept)s should match expected LC3 composition %(lc3name)s"),
modelObject=modelConcept, concept=modelConcept.qname, lc3name=lc3name)
if conceptType is not None:
# 6.8.6 semantic check
if not isDomainItemType and conceptType.qname != XbrlConst.qnXbrliDurationItemType:
nameProblems = nonDomainItemNameProblemPattern.findall(name)
if any(any(t) for t in nameProblems): # list of tuples with possibly nonempty strings
val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.08.06", "GFM.2.03.06"),
_("Concept %(concept)s should not contain company or period information, found: %(matches)s"),
modelObject=modelConcept, concept=modelConcept.qname,
matches=", ".join(''.join(t) for t in nameProblems))
if conceptType.qname == XbrlConst.qnXbrliMonetaryItemType:
if not modelConcept.balance:
# 6.8.11 may not appear on a income or balance statement
if any(linkroleDefinitionBalanceIncomeSheet.match(roleType.definition)
for rel in val.modelXbrl.relationshipSet(XbrlConst.parentChild).toModelObject(modelConcept)
for roleType in val.modelXbrl.roleTypes.get(rel.linkrole,())):
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.11", "GFM.2.03.11"),
_("Concept %(concept)s must have a balance because it appears in a statement of income or balance sheet"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.11.5 semantic check, must have a documentation label
stdLabel = modelConcept.label(lang="en-US", fallbackToQname=False)
defLabel = modelConcept.label(preferredLabel=XbrlConst.documentationLabel, lang="en-US", fallbackToQname=False)
if not defLabel or ( # want different words than std label
stdLabel and re.findall(r"\w+", stdLabel) == re.findall(r"\w+", defLabel)):
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.11.05", "GFM.2.04.04"),
_("Concept %(concept)s is monetary without a balance and must have a documentation label that disambiguates its sign"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.8.16 semantic check
if conceptType.qname == XbrlConst.qnXbrliDateItemType and modelConcept.periodType != "duration":
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.16", "GFM.2.03.16"),
_("Concept %(concept)s of type xbrli:dateItemType must have periodType duration"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.8.17 semantic check
if conceptType.qname == XbrlConst.qnXbrliStringItemType and modelConcept.periodType != "duration":
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.17", "GFM.2.03.17"),
_("Concept %(concept)s of type xbrli:stringItemType must have periodType duration"),
modelObject=modelConcept, concept=modelConcept.qname)
# 6.7.8 check for embedded linkbase
for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
if isinstance(e,ModelObject):
val.modelXbrl.error(("EFM.6.07.08", "GFM.1.03.08"),
_("Taxonomy schema %(schema)s contains an embedded linkbase"),
modelObject=e, schema=modelDocument.basename)
break
requiredUsedOns = {XbrlConst.qnLinkPresentationLink,
XbrlConst.qnLinkCalculationLink,
XbrlConst.qnLinkDefinitionLink}
standardUsedOns = {XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference,
XbrlConst.qnLinkDefinitionArc, XbrlConst.qnLinkCalculationArc, XbrlConst.qnLinkPresentationArc,
XbrlConst.qnLinkLabelArc, XbrlConst.qnLinkReferenceArc,
# per WH, private footnote arc and footnore resource roles are not allowed
XbrlConst.qnLinkFootnoteArc, XbrlConst.qnLinkFootnote,
}
# 6.7.9 role types authority
for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}roleType"):
if isinstance(e,ModelObject):
roleURI = e.get("roleURI")
if targetNamespaceAuthority != UrlUtil.authority(roleURI):
val.modelXbrl.error(("EFM.6.07.09", "GFM.1.03.09"),
_("RoleType %(roleType)s does not match authority %(targetNamespaceAuthority)s"),
modelObject=e, roleType=roleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace)
# 6.7.9 end with .../role/lc3 name
if not roleTypePattern.match(roleURI):
val.modelXbrl.warning(("EFM.6.07.09.roleEnding", "GFM.1.03.09"),
"RoleType %(roleType)s should end with /role/{LC3name}",
modelObject=e, roleType=roleURI)
# 6.7.10 only one role type declaration in DTS
modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI)
if modelRoleTypes is not None:
modelRoleType = modelRoleTypes[0]
definition = modelRoleType.definitionNotStripped
usedOns = modelRoleType.usedOns
if len(modelRoleTypes) == 1:
# 6.7.11 used on's for pre, cal, def if any has a used on
if not usedOns.isdisjoint(requiredUsedOns) and len(requiredUsedOns - usedOns) > 0:
val.modelXbrl.error(("EFM.6.07.11", "GFM.1.03.11"),
_("RoleType %(roleType)s missing used on %(usedOn)s"),
modelObject=e, roleType=roleURI, usedOn=requiredUsedOns - usedOns)
# 6.7.12 definition match pattern
if (val.disclosureSystem.roleDefinitionPattern is not None and
(definition is None or not val.disclosureSystem.roleDefinitionPattern.match(definition))):
val.modelXbrl.error(("EFM.6.07.12", "GFM.1.03.12-14"),
_("RoleType %(roleType)s definition \"%(definition)s\" must match {Sortcode} - {Type} - {Title}"),
modelObject=e, roleType=roleURI, definition=(definition or ""))
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("RoleType %(roleuri)s is defined using role types already defined by standard roles for: %(qnames)s"),
modelObject=e, roleuri=roleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns))
# 6.7.13 arcrole types authority
for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}arcroleType"):
if isinstance(e,ModelObject):
arcroleURI = e.get("arcroleURI")
if targetNamespaceAuthority != UrlUtil.authority(arcroleURI):
val.modelXbrl.error(("EFM.6.07.13", "GFM.1.03.15"),
_("ArcroleType %(arcroleType)s does not match authority %(targetNamespaceAuthority)s"),
modelObject=e, arcroleType=arcroleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace)
# 6.7.13 end with .../arcrole/lc3 name
if not arcroleTypePattern.match(arcroleURI):
val.modelXbrl.warning(("EFM.6.07.13.arcroleEnding", "GFM.1.03.15"),
_("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"),
modelObject=e, arcroleType=arcroleURI)
# 6.7.15 definition match pattern
modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI]
definition = modelRoleTypes[0].definition
if definition is None or not arcroleDefinitionPattern.match(definition):
val.modelXbrl.error(("EFM.6.07.15", "GFM.1.03.17"),
_("ArcroleType %(arcroleType)s definition must be non-empty"),
modelObject=e, arcroleType=arcroleURI)
# semantic checks
usedOns = modelRoleTypes[0].usedOns
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("ArcroleType %(arcroleuri)s is defined using role types already defined by standard arcroles for: %(qnames)s"),
modelObject=e, arcroleuri=arcroleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns))
#6.3.3 filename check
m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9]).xsd$", modelDocument.basename)
if m:
try: # check date value
datetime.datetime.strptime(m.group(1),"%Y%m%d").date()
# date and format are ok, check "should" part of 6.3.3
if val.fileNameBasePart:
expectedFilename = "{0}-{1}.xsd".format(val.fileNameBasePart, val.fileNameDatePart)
if modelDocument.basename != expectedFilename:
val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"),
_('Schema file name warning: %(filename)s, should match %(expectedFilename)s'),
modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename)
except ValueError:
val.modelXbrl.error((val.EFM60303, "GFM.1.01.01"),
_('Invalid schema file base name part (date) in "{base}-{yyyymmdd}.xsd": %(filename)s'),
modelObject=modelDocument, filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
else:
val.modelXbrl.error((val.EFM60303, "GFM.1.01.01"),
_('Invalid schema file name, must match "{base}-{yyyymmdd}.xsd": %(filename)s'),
modelObject=modelDocument, filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
elif modelDocument.type == ModelDocument.Type.LINKBASE:
# if it is part of the submission (in same directory) check name
labelRels = None
if modelDocument.filepath.startswith(val.modelXbrl.modelDocument.filepathdir):
#6.3.3 filename check
extLinkElt = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended")
if extLinkElt is None:# no ext link element
val.modelXbrl.error((val.EFM60303 + ".noLinkElement", "GFM.1.01.01.noLinkElement"),
_('Invalid linkbase file name: %(filename)s, has no extended link element, cannot determine link type.'),
modelObject=modelDocument, filename=modelDocument.basename,
messageCodes=("EFM.6.03.03.noLinkElement", "EFM.6.23.01.noLinkElement", "GFM.1.01.01.noLinkElement"))
elif extLinkElt.localName not in extLinkEltFileNameEnding:
val.modelXbrl.error("EFM.6.03.02",
_('Invalid linkbase link element %(linkElement)s in %(filename)s'),
modelObject=modelDocument, linkElement=extLinkElt.localName, filename=modelDocument.basename)
else:
m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9])(_[a-z]{3}).xml$", modelDocument.basename)
expectedSuffix = extLinkEltFileNameEnding[extLinkElt.localName]
if m and m.group(2) == expectedSuffix:
try: # check date value
datetime.datetime.strptime(m.group(1),"%Y%m%d").date()
# date and format are ok, check "should" part of 6.3.3
if val.fileNameBasePart:
expectedFilename = "{0}-{1}{2}.xml".format(val.fileNameBasePart, val.fileNameDatePart, expectedSuffix)
if modelDocument.basename != expectedFilename:
val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"),
_('Linkbase name warning: %(filename)s should match %(expectedFilename)s'),
modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename)
except ValueError:
val.modelXbrl.error((val.EFM60303, "GFM.1.01.01"),
_('Invalid linkbase base file name part (date) in "{base}-{yyyymmdd}_{suffix}.xml": %(filename)s'),
modelObject=modelDocument, filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
else:
val.modelXbrl.error((val.EFM60303, "GFM.1.01.01"),
_('Invalid linkbase name, must match "{base}-{yyyymmdd}%(expectedSuffix)s.xml": %(filename)s'),
modelObject=modelDocument, filename=modelDocument.basename, expectedSuffix=expectedSuffix,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
if extLinkElt.localName == "labelLink":
if labelRels is None:
labelRels = val.modelXbrl.relationshipSet(XbrlConst.conceptLabel)
for labelElt in XmlUtil.children(extLinkElt, XbrlConst.link, "label"):
# 6.10.9
if XbrlConst.isNumericRole(labelElt.role):
for rel in labelRels.toModelObject(labelElt):
if rel.fromModelObject is not None and not rel.fromModelObject.isNumeric:
val.modelXbrl.error("EFM.6.10.09",
_("Label of non-numeric concept %(concept)s has a numeric role: %(role)s"),
modelObject=(labelElt, rel.fromModelObject), concept=rel.fromModelObject.qname, role=labelElt.role)
def uriAuthorityValid(self, uri):
if self.standardTaxonomiesUrl:
return UrlUtil.authority(uri) in self.standardAuthorities
return True # no standard authorities to test
def loadStandardTaxonomiesDict(self):
if self.selection:
self.standardTaxonomiesDict = defaultdict(set)
self.familyHrefs = defaultdict(set)
self.standardLocalHrefs = defaultdict(set)
self.standardAuthorities = set()
self.standardPrefixes = {}
if not self.standardTaxonomiesUrl:
return
basename = os.path.basename(self.standardTaxonomiesUrl)
self.modelManager.cntlr.showStatus(_("parsing {0}").format(basename))
file = None
try:
from arelle.FileSource import openXmlFileStream
for filepath in (self.standardTaxonomiesUrl,
os.path.join(self.modelManager.cntlr.configDir,"xbrlschemafiles.xml")):
file = openXmlFileStream(self.modelManager.cntlr, filepath, stripDeclaration=True)[0]
xmldoc = etree.parse(file)
file.close()
for locElt in xmldoc.iter(tag="Loc"):
href = None
localHref = None
namespaceUri = None
prefix = None
attType = None
family = None
elements = None
version = None
for childElt in locElt.iterchildren():
ln = childElt.tag
value = childElt.text.strip()
if ln == "Href":
href = value
elif ln == "LocalHref":
localHref = value
elif ln == "Namespace":
namespaceUri = value
elif ln == "Prefix":
prefix = value
elif ln == "AttType":
attType = value
elif ln == "Family":
family = value
elif ln == "Elements":
elements = value
elif ln == "Version":
version = value
if href:
if namespaceUri and (attType == "SCH" or attType == "ENT"):
self.standardTaxonomiesDict[namespaceUri].add(href)
if localHref:
self.standardLocalHrefs[namespaceUri].add(localHref)
authority = UrlUtil.authority(namespaceUri)
self.standardAuthorities.add(authority)
if family == "BASE":
self.baseTaxonomyNamespaces.add(namespaceUri)
if prefix:
self.standardPrefixes[namespaceUri] = prefix
if href not in self.standardTaxonomiesDict:
self.standardTaxonomiesDict[href] = "Allowed" + attType
if family:
self.familyHrefs[family].add(ErxlLoc(family, version, href, attType, elements, namespaceUri))
elif attType == "SCH" and family == "BASE":
self.baseTaxonomyNamespaces.add(namespaceUri)
except (EnvironmentError,
etree.LxmlError) as err:
self.modelManager.cntlr.addToLog("{0}: import error: {1}".format(basename,err))
etree.clear_error_log()
if file:
file.close()
def checkFilingDTS(val, modelDocument, isEFM, isGFM, visited):
global targetNamespaceDatePattern, efmFilenamePattern, htmlFileNamePattern, roleTypePattern, arcroleTypePattern, \
arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet, \
namespacesConflictPattern
if targetNamespaceDatePattern is None:
targetNamespaceDatePattern = re.compile(
r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|"
r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|")
efmFilenamePattern = re.compile(
r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml|\.htm)$")
htmlFileNamePattern = re.compile(
r"^[a-zA-Z0-9][._a-zA-Z0-9-]*(\.htm)$")
roleTypePattern = re.compile(r"^.*/role/[^/\s]+$")
arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$")
arcroleDefinitionPattern = re.compile(
r"^.*[^\\s]+.*$") # at least one non-whitespace character
namePattern = re.compile(
"[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]"
) # u20ac=Euro, u00a3=pound sterling
linkroleDefinitionBalanceIncomeSheet = re.compile(
r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)",
re.IGNORECASE)
namespacesConflictPattern = re.compile(
r"http://(xbrl\.us|fasb\.org|xbrl\.sec\.gov)/(dei|us-types|us-roles|rr)/([0-9]{4}-[0-9]{2}-[0-9]{2})$"
)
nonDomainItemNameProblemPattern = re.compile(
r"({0})|(FirstQuarter|SecondQuarter|ThirdQuarter|FourthQuarter|[1-4]Qtr|Qtr[1-4]|ytd|YTD|HalfYear)(?:$|[A-Z\W])"
.format(re.sub(r"\W", "", (val.entityRegistrantName or "").title())))
visited.append(modelDocument)
for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items(
):
#6.07.01 no includes
if modelDocumentReference.referenceType == "include":
val.modelXbrl.error(
("EFM.6.07.01", "GFM.1.03.01"),
_("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"
),
modelObject=modelDocumentReference.referringModelObject,
schema=os.path.basename(modelDocument.uri),
include=os.path.basename(referencedDocument.uri))
if referencedDocument not in visited and referencedDocument.inDTS: # ignore EdgarRenderer added non-DTS documents
checkFilingDTS(val, referencedDocument, isEFM, isGFM, visited)
if val.disclosureSystem.standardTaxonomiesDict is None:
pass
if isEFM:
if modelDocument.uri in val.disclosureSystem.standardTaxonomiesDict:
if modelDocument.targetNamespace:
# check for duplicates of us-types, dei, and rr taxonomies
match = namespacesConflictPattern.match(
modelDocument.targetNamespace)
if match is not None:
val.standardNamespaceConflicts[match.group(2)].add(
modelDocument)
else:
if len(modelDocument.basename) > 32:
val.modelXbrl.error(
"EFM.5.01.01.tooManyCharacters",
_("Document file name %(filename)s must not exceed 32 characters."
),
modelObject=modelDocument,
filename=modelDocument.basename)
if modelDocument.type == ModelDocument.Type.INLINEXBRL:
if not htmlFileNamePattern.match(modelDocument.basename):
val.modelXbrl.error(
"EFM.5.01.01",
_("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with .htm."
),
modelObject=modelDocument,
filename=modelDocument.basename)
elif not efmFilenamePattern.match(modelDocument.basename):
val.modelXbrl.error(
"EFM.5.01.01",
_("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with .xsd or .xml."
),
modelObject=modelDocument,
filename=modelDocument.basename)
if (modelDocument.type == ModelDocument.Type.SCHEMA
and modelDocument.targetNamespace
not in val.disclosureSystem.baseTaxonomyNamespaces
and modelDocument.uri.startswith(val.modelXbrl.uriDir)):
val.hasExtensionSchema = True
# check schema contents types
# 6.7.3 check namespace for standard authority
targetNamespaceAuthority = UrlUtil.authority(
modelDocument.targetNamespace)
if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities:
val.modelXbrl.error(
("EFM.6.07.03", "GFM.1.03.03"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s is a disallowed authority"
),
modelObject=modelDocument,
schema=os.path.basename(modelDocument.uri),
targetNamespace=modelDocument.targetNamespace,
targetNamespaceAuthority=UrlUtil.authority(
modelDocument.targetNamespace, includeScheme=False))
# 6.7.4 check namespace format
if modelDocument.targetNamespace is None or not modelDocument.targetNamespace.startswith(
"http://"):
match = None
else:
targetNamespaceDate = modelDocument.targetNamespace[
len(targetNamespaceAuthority):]
match = targetNamespaceDatePattern.match(targetNamespaceDate)
if match is not None:
try:
if match.lastindex == 3:
date = datetime.date(int(match.group(1)),
int(match.group(2)),
int(match.group(3)))
elif match.lastindex == 6:
date = datetime.date(int(match.group(4)),
int(match.group(5)),
int(match.group(6)))
else:
match = None
except ValueError:
match = None
if match is None:
val.modelXbrl.error(
("EFM.6.07.04", "GFM.1.03.04"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s must have format http://{authority}/{versionDate}"
),
modelObject=modelDocument,
schema=os.path.basename(modelDocument.uri),
targetNamespace=modelDocument.targetNamespace)
elif val.fileNameDate and date > val.fileNameDate:
val.modelXbrl.info(
("EFM.6.07.06", "GFM.1.03.06"),
_("Warning: Taxonomy schema %(schema)s namespace %(targetNamespace)s has date later than document name date %(docNameDate)s"
),
modelObject=modelDocument,
schema=os.path.basename(modelDocument.uri),
targetNamespace=modelDocument.targetNamespace,
docNameDate=val.fileNameDate)
if modelDocument.targetNamespace is not None:
# 6.7.5 check prefix for _
authority = UrlUtil.authority(modelDocument.targetNamespace)
if not re.match(r"(http://|https://|ftp://|urn:)\w+", authority):
val.modelXbrl.error(
("EFM.6.07.05", "GFM.1.03.05"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s must be a valid URL with a valid authority for the namespace."
),
modelObject=modelDocument,
schema=os.path.basename(modelDocument.uri),
targetNamespace=modelDocument.targetNamespace)
prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement,
modelDocument.targetNamespace)
if not prefix:
val.modelXbrl.error(
("EFM.6.07.07", "GFM.1.03.07"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s missing prefix for the namespace."
),
modelObject=modelDocument,
schema=os.path.basename(modelDocument.uri),
targetNamespace=modelDocument.targetNamespace)
elif "_" in prefix:
val.modelXbrl.error(
("EFM.6.07.07", "GFM.1.03.07"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s prefix %(prefix)s must not have an '_'"
),
modelObject=modelDocument,
schema=os.path.basename(modelDocument.uri),
targetNamespace=modelDocument.targetNamespace,
prefix=prefix)
for modelConcept in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.w3.org/2001/XMLSchema}element"):
if isinstance(modelConcept, ModelConcept):
# 6.7.16 name not duplicated in standard taxonomies
name = modelConcept.get("name")
if name is None:
name = ""
if modelConcept.get("ref") is not None:
continue # don't validate ref's here
for c in val.modelXbrl.nameConcepts.get(name, []):
if c.modelDocument != modelDocument:
if not c.modelDocument.uri.startswith(
val.modelXbrl.uriDir):
val.modelXbrl.error(
("EFM.6.07.16", "GFM.1.03.18"),
_("Concept %(concept)s is also defined in standard taxonomy schema schema %(standardSchema)s"
),
modelObject=(modelConcept, c),
concept=modelConcept.qname,
standardSchema=os.path.basename(
c.modelDocument.uri),
standardConcept=c.qname)
# 6.7.17 id properly formed
_id = modelConcept.id
requiredId = (prefix
if prefix is not None else "") + "_" + name
if _id != requiredId:
val.modelXbrl.error(
("EFM.6.07.17", "GFM.1.03.19"),
_("Concept %(concept)s id %(id)s should be %(requiredId)s"
),
modelObject=modelConcept,
concept=modelConcept.qname,
id=_id,
requiredId=requiredId)
# 6.7.18 nillable is true
nillable = modelConcept.get("nillable")
if nillable != "true" and modelConcept.isItem:
val.modelXbrl.error(
("EFM.6.07.18", "GFM.1.03.20"),
_("Taxonomy schema %(schema)s element %(concept)s nillable %(nillable)s should be 'true'"
),
modelObject=modelConcept,
schema=os.path.basename(modelDocument.uri),
concept=name,
nillable=nillable)
# 6.7.19 not tuple
if modelConcept.isTuple:
val.modelXbrl.error(
("EFM.6.07.19", "GFM.1.03.21"),
_("Concept %(concept)s is a tuple"),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.7.20 no typed domain ref
if modelConcept.isTypedDimension:
val.modelXbrl.error(
("EFM.6.07.20", "GFM.1.03.22"),
_("Concept %(concept)s has typedDomainRef %(typedDomainRef)s"
),
modelObject=modelConcept,
concept=modelConcept.qname,
typedDomainRef=modelConcept.typedDomainElement.
qname if modelConcept.typedDomainElement
is not None else modelConcept.typedDomainRef)
# 6.7.21 abstract must be duration
isDuration = modelConcept.periodType == "duration"
if modelConcept.isAbstract and not isDuration:
val.modelXbrl.error(
("EFM.6.07.21", "GFM.1.03.23"),
_("Taxonomy schema %(schema)s element %(concept)s is abstract but period type is not duration"
),
modelObject=modelConcept,
schema=os.path.basename(modelDocument.uri),
concept=modelConcept.qname)
# 6.7.22 abstract must be stringItemType
''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08
if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType:
val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"),
_("Concept %(concept)s is abstract but type is not xbrli:stringItemType"),
modelObject=modelConcept, concept=modelConcept.qname)
'''
substitutionGroupQname = modelConcept.substitutionGroupQname
# 6.7.23 Axis must be subs group dimension
if name.endswith("Axis") ^ (
substitutionGroupQname
== XbrlConst.qnXbrldtDimensionItem):
val.modelXbrl.error(
("EFM.6.07.23", "GFM.1.03.25"),
_("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.7.24 Table must be subs group hypercube
if name.endswith("Table") ^ (
substitutionGroupQname
== XbrlConst.qnXbrldtHypercubeItem):
val.modelXbrl.error(
("EFM.6.07.24", "GFM.1.03.26"),
_("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"
),
modelObject=modelConcept,
schema=os.path.basename(modelDocument.uri),
concept=modelConcept.qname)
# 6.7.25 if neither hypercube or dimension, substitution group must be item
if substitutionGroupQname not in (
None, XbrlConst.qnXbrldtDimensionItem,
XbrlConst.qnXbrldtHypercubeItem,
XbrlConst.qnXbrliItem):
val.modelXbrl.error(
("EFM.6.07.25", "GFM.1.03.27"),
_("Concept %(concept)s has disallowed substitution group %(substitutionGroup)s"
),
modelObject=modelConcept,
concept=modelConcept.qname,
substitutionGroup=modelConcept.
substitutionGroupQname)
# 6.7.26 Table must be subs group hypercube
if name.endswith(
"LineItems") and modelConcept.abstract != "true":
val.modelXbrl.error(
("EFM.6.07.26", "GFM.1.03.28"),
_("Concept %(concept)s is a LineItems but not abstract"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.7.27 type domainMember must end with Domain or Member
conceptType = modelConcept.type
isDomainItemType = conceptType is not None and conceptType.isDomainItemType
endsWithDomainOrMember = name.endswith(
"Domain") or name.endswith("Member")
if isDomainItemType != endsWithDomainOrMember:
val.modelXbrl.error(
("EFM.6.07.27", "GFM.1.03.29"),
_("Concept %(concept)s must end with Domain or Member for type of domainItemType"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.7.28 domainItemType must be duration
if isDomainItemType and not isDuration:
val.modelXbrl.error(
("EFM.6.07.28", "GFM.1.03.30"),
_("Concept %(concept)s is a domainItemType and must be periodType duration"
),
modelObject=modelConcept,
concept=modelConcept.qname)
#6.7.31 (version 27) fractions
if modelConcept.isFraction:
val.modelXbrl.error(
"EFM.6.07.31",
_("Concept %(concept)s is a fraction"),
modelObject=modelConcept,
concept=modelConcept.qname)
#6.7.32 (version 27) instant non numeric
if modelConcept.isItem and (not modelConcept.isNumeric
and not isDuration
and not modelConcept.isAbstract
and not isDomainItemType):
val.modelXbrl.error(
"EFM.6.07.32",
_("Taxonomy schema %(schema)s element %(concept)s is non-numeric but period type is not duration"
),
modelObject=modelConcept,
schema=os.path.basename(modelDocument.uri),
concept=modelConcept.qname)
# 6.8.5 semantic check, check LC3 name
if name:
if not name[0].isupper():
val.modelXbrl.log(
"ERROR-SEMANTIC", ("EFM.6.08.05.firstLetter",
"GFM.2.03.05.firstLetter"),
_("Concept %(concept)s name must start with a capital letter"
),
modelObject=modelConcept,
concept=modelConcept.qname)
if namePattern.search(name):
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.05.disallowedCharacter",
"GFM.2.03.05.disallowedCharacter"),
_("Concept %(concept)s has disallowed name character"
),
modelObject=modelConcept,
concept=modelConcept.qname)
if len(name) > 200:
val.modelXbrl.log(
"ERROR-SEMANTIC",
"EFM.6.08.05.nameLength",
_("Concept %(concept)s name length %(namelength)s exceeds 200 characters"
),
modelObject=modelConcept,
concept=modelConcept.qname,
namelength=len(name))
if isEFM:
label = modelConcept.label(lang="en-US",
fallbackToQname=False)
if label:
# allow Joe's Bar, N.A. to be JoesBarNA -- remove ', allow A. as not article "a"
lc3name = ''.join(
re.sub(r"['.-]", "", (
w[0] or w[2] or w[3] or w[4])).title()
for w in re.findall(
r"((\w+')+\w+)|(A[.-])|([.-]A(?=\W|$))|(\w+)",
label
) # EFM implies this should allow - and . re.findall(r"[\w\-\.]+", label)
if w[4].lower() not in ("the", "a", "an"))
if not (name == lc3name or
(name and lc3name and lc3name[0].isdigit()
and name[1:] == lc3name and
(name[0].isalpha() or name[0] == '_'))):
val.modelXbrl.log(
"WARNING-SEMANTIC",
"EFM.6.08.05.LC3",
_("Concept %(concept)s should match expected LC3 composition %(lc3name)s"
),
modelObject=modelConcept,
concept=modelConcept.qname,
lc3name=lc3name)
if conceptType is not None:
# 6.8.6 semantic check
if not isDomainItemType and conceptType.qname != XbrlConst.qnXbrliDurationItemType:
nameProblems = nonDomainItemNameProblemPattern.findall(
name)
if any(
any(t) for t in nameProblems
): # list of tuples with possibly nonempty strings
val.modelXbrl.log(
"WARNING-SEMANTIC",
("EFM.6.08.06", "GFM.2.03.06"),
_("Concept %(concept)s should not contain company or period information, found: %(matches)s"
),
modelObject=modelConcept,
concept=modelConcept.qname,
matches=", ".join(''.join(t)
for t in nameProblems))
if conceptType.qname == XbrlConst.qnXbrliMonetaryItemType:
if not modelConcept.balance:
# 6.8.11 may not appear on a income or balance statement
if any(
linkroleDefinitionBalanceIncomeSheet.
match(roleType.definition)
for rel in val.modelXbrl.
relationshipSet(XbrlConst.parentChild).
toModelObject(modelConcept)
for roleType in val.modelXbrl.
roleTypes.get(rel.linkrole, ())):
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.11", "GFM.2.03.11"),
_("Concept %(concept)s must have a balance because it appears in a statement of income or balance sheet"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.11.5 semantic check, must have a documentation label
stdLabel = modelConcept.label(
lang="en-US", fallbackToQname=False)
defLabel = modelConcept.label(
preferredLabel=XbrlConst.
documentationLabel,
lang="en-US",
fallbackToQname=False)
if not defLabel or ( # want different words than std label
stdLabel
and re.findall(r"\w+", stdLabel)
== re.findall(r"\w+", defLabel)):
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.11.05", "GFM.2.04.04"),
_("Concept %(concept)s is monetary without a balance and must have a documentation label that disambiguates its sign"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.8.16 semantic check
if conceptType.qname == XbrlConst.qnXbrliDateItemType and modelConcept.periodType != "duration":
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.16", "GFM.2.03.16"),
_("Concept %(concept)s of type xbrli:dateItemType must have periodType duration"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.8.17 semantic check
if conceptType.qname == XbrlConst.qnXbrliStringItemType and modelConcept.periodType != "duration":
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.17", "GFM.2.03.17"),
_("Concept %(concept)s of type xbrli:stringItemType must have periodType duration"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.7.8 check for embedded linkbase
for e in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
if isinstance(e, ModelObject):
val.modelXbrl.error(
("EFM.6.07.08", "GFM.1.03.08"),
_("Taxonomy schema %(schema)s contains an embedded linkbase"
),
modelObject=e,
schema=modelDocument.basename)
break
requiredUsedOns = {
XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink,
XbrlConst.qnLinkDefinitionLink
}
standardUsedOns = {
XbrlConst.qnLinkLabel,
XbrlConst.qnLinkReference,
XbrlConst.qnLinkDefinitionArc,
XbrlConst.qnLinkCalculationArc,
XbrlConst.qnLinkPresentationArc,
XbrlConst.qnLinkLabelArc,
XbrlConst.qnLinkReferenceArc,
# per WH, private footnote arc and footnore resource roles are not allowed
XbrlConst.qnLinkFootnoteArc,
XbrlConst.qnLinkFootnote,
}
# 6.7.9 role types authority
for e in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.xbrl.org/2003/linkbase}roleType"):
if isinstance(e, ModelObject):
roleURI = e.get("roleURI")
if targetNamespaceAuthority != UrlUtil.authority(roleURI):
val.modelXbrl.error(
("EFM.6.07.09", "GFM.1.03.09"),
_("RoleType %(roleType)s does not match authority %(targetNamespaceAuthority)s"
),
modelObject=e,
roleType=roleURI,
targetNamespaceAuthority=targetNamespaceAuthority,
targetNamespace=modelDocument.targetNamespace)
# 6.7.9 end with .../role/lc3 name
if not roleTypePattern.match(roleURI):
val.modelXbrl.warning(
("EFM.6.07.09.roleEnding", "GFM.1.03.09"),
"RoleType %(roleType)s should end with /role/{LC3name}",
modelObject=e,
roleType=roleURI)
# 6.7.10 only one role type declaration in DTS
modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI)
if modelRoleTypes is not None:
modelRoleType = modelRoleTypes[0]
definition = modelRoleType.definitionNotStripped
usedOns = modelRoleType.usedOns
if len(modelRoleTypes) == 1:
# 6.7.11 used on's for pre, cal, def if any has a used on
if not usedOns.isdisjoint(requiredUsedOns) and len(
requiredUsedOns - usedOns) > 0:
val.modelXbrl.error(
("EFM.6.07.11", "GFM.1.03.11"),
_("RoleType %(roleType)s missing used on %(usedOn)s"
),
modelObject=e,
roleType=roleURI,
usedOn=requiredUsedOns - usedOns)
# 6.7.12 definition match pattern
if (val.disclosureSystem.roleDefinitionPattern
is not None and
(definition is None or not val.disclosureSystem.
roleDefinitionPattern.match(definition))):
val.modelXbrl.error(
("EFM.6.07.12", "GFM.1.03.12-14"),
_("RoleType %(roleType)s definition \"%(definition)s\" must match {Sortcode} - {Type} - {Title}"
),
modelObject=e,
roleType=roleURI,
definition=(definition or ""))
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log(
"ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("RoleType %(roleuri)s is defined using role types already defined by standard roles for: %(qnames)s"
),
modelObject=e,
roleuri=roleURI,
qnames=', '.join(
str(qn) for qn in usedOns & standardUsedOns))
# 6.7.13 arcrole types authority
for e in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.xbrl.org/2003/linkbase}arcroleType"):
if isinstance(e, ModelObject):
arcroleURI = e.get("arcroleURI")
if targetNamespaceAuthority != UrlUtil.authority(arcroleURI):
val.modelXbrl.error(
("EFM.6.07.13", "GFM.1.03.15"),
_("ArcroleType %(arcroleType)s does not match authority %(targetNamespaceAuthority)s"
),
modelObject=e,
arcroleType=arcroleURI,
targetNamespaceAuthority=targetNamespaceAuthority,
targetNamespace=modelDocument.targetNamespace)
# 6.7.13 end with .../arcrole/lc3 name
if not arcroleTypePattern.match(arcroleURI):
val.modelXbrl.warning(
("EFM.6.07.13.arcroleEnding", "GFM.1.03.15"),
_("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"
),
modelObject=e,
arcroleType=arcroleURI)
# 6.7.15 definition match pattern
modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI]
definition = modelRoleTypes[0].definition
if definition is None or not arcroleDefinitionPattern.match(
definition):
val.modelXbrl.error(
("EFM.6.07.15", "GFM.1.03.17"),
_("ArcroleType %(arcroleType)s definition must be non-empty"
),
modelObject=e,
arcroleType=arcroleURI)
# semantic checks
usedOns = modelRoleTypes[0].usedOns
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log(
"ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("ArcroleType %(arcroleuri)s is defined using role types already defined by standard arcroles for: %(qnames)s"
),
modelObject=e,
arcroleuri=arcroleURI,
qnames=', '.join(
str(qn) for qn in usedOns & standardUsedOns))
#6.3.3 filename check
m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9]).xsd$",
modelDocument.basename)
if m:
try: # check date value
datetime.datetime.strptime(m.group(1), "%Y%m%d").date()
# date and format are ok, check "should" part of 6.3.3
if val.fileNameBasePart:
expectedFilename = "{0}-{1}.xsd".format(
val.fileNameBasePart, val.fileNameDatePart)
if modelDocument.basename != expectedFilename:
val.modelXbrl.log(
"WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance",
"GFM.1.01.01.matchInstance"),
_('Schema file name warning: %(filename)s, should match %(expectedFilename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
expectedFilename=expectedFilename)
except ValueError:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid schema file base name part (date) in "{base}-{yyyymmdd}.xsd": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
else:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid schema file name, must match "{base}-{yyyymmdd}.xsd": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
elif modelDocument.type == ModelDocument.Type.LINKBASE:
# if it is part of the submission (in same directory) check name
labelRels = None
if modelDocument.filepath.startswith(
val.modelXbrl.modelDocument.filepathdir):
#6.3.3 filename check
extLinkElt = XmlUtil.descendant(
modelDocument.xmlRootElement, XbrlConst.link, "*",
"{http://www.w3.org/1999/xlink}type", "extended")
if extLinkElt is None: # no ext link element
val.modelXbrl.error(
(val.EFM60303 + ".noLinkElement",
"GFM.1.01.01.noLinkElement"),
_('Invalid linkbase file name: %(filename)s, has no extended link element, cannot determine link type.'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03.noLinkElement",
"EFM.6.23.01.noLinkElement",
"GFM.1.01.01.noLinkElement"))
elif extLinkElt.localName not in extLinkEltFileNameEnding:
val.modelXbrl.error(
"EFM.6.03.02",
_('Invalid linkbase link element %(linkElement)s in %(filename)s'
),
modelObject=modelDocument,
linkElement=extLinkElt.localName,
filename=modelDocument.basename)
else:
m = re.match(
r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9])(_[a-z]{3}).xml$",
modelDocument.basename)
expectedSuffix = extLinkEltFileNameEnding[extLinkElt.localName]
if m and m.group(2) == expectedSuffix:
try: # check date value
datetime.datetime.strptime(m.group(1), "%Y%m%d").date()
# date and format are ok, check "should" part of 6.3.3
if val.fileNameBasePart:
expectedFilename = "{0}-{1}{2}.xml".format(
val.fileNameBasePart, val.fileNameDatePart,
expectedSuffix)
if modelDocument.basename != expectedFilename:
val.modelXbrl.log(
"WARNING-SEMANTIC",
("EFM.6.03.03.matchInstance",
"GFM.1.01.01.matchInstance"),
_('Linkbase name warning: %(filename)s should match %(expectedFilename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
expectedFilename=expectedFilename)
except ValueError:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid linkbase base file name part (date) in "{base}-{yyyymmdd}_{suffix}.xml": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01",
"GFM.1.01.01"))
else:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid linkbase name, must match "{base}-{yyyymmdd}%(expectedSuffix)s.xml": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
expectedSuffix=expectedSuffix,
messageCodes=("EFM.6.03.03", "EFM.6.23.01",
"GFM.1.01.01"))
if extLinkElt.localName == "labelLink":
if labelRels is None:
labelRels = val.modelXbrl.relationshipSet(
XbrlConst.conceptLabel)
for labelElt in XmlUtil.children(extLinkElt,
XbrlConst.link, "label"):
# 6.10.9
if XbrlConst.isNumericRole(labelElt.role):
for rel in labelRels.toModelObject(labelElt):
if rel.fromModelObject is not None and not rel.fromModelObject.isNumeric:
val.modelXbrl.error(
"EFM.6.10.09",
_("Label of non-numeric concept %(concept)s has a numeric role: %(role)s"
),
modelObject=(labelElt,
rel.fromModelObject),
concept=rel.fromModelObject.qname,
role=labelElt.role)
def checkFilingDTS(val, modelDocument, isEFM, isGFM, visited):
global targetNamespaceDatePattern, efmFilenamePattern, htmlFileNamePattern, roleTypePattern, arcroleTypePattern, \
arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet
if targetNamespaceDatePattern is None:
targetNamespaceDatePattern = re.compile(
r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|"
r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|")
efmFilenamePattern = re.compile(
r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml|\.htm)$")
htmlFileNamePattern = re.compile(
r"^[a-zA-Z0-9][._a-zA-Z0-9-]*(\.htm)$")
roleTypePattern = re.compile(r"^.*/role/[^/\s]+$")
arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$")
arcroleDefinitionPattern = re.compile(
r"^.*[^\\s]+.*$") # at least one non-whitespace character
namePattern = re.compile(
"[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]"
) # u20ac=Euro, u00a3=pound sterling
linkroleDefinitionBalanceIncomeSheet = re.compile(
r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)",
re.IGNORECASE)
nonDomainItemNameProblemPattern = re.compile(
r"({0})|(FirstQuarter|SecondQuarter|ThirdQuarter|FourthQuarter|[1-4]Qtr|Qtr[1-4]|ytd|YTD|HalfYear)(?:$|[A-Z\W])"
.format(re.sub(r"\W", "", (val.entityRegistrantName or "").title())))
visited.append(modelDocument)
for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items(
):
#6.07.01 no includes
if modelDocumentReference.referenceType == "include":
val.modelXbrl.error(
("EFM.6.07.01", "GFM.1.03.01"),
_("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"
),
modelObject=modelDocumentReference.referringModelObject,
schema=modelDocument.basename,
include=referencedDocument.basename)
if referencedDocument not in visited and (
referencedDocument.inDTS or referencedDocument.type
== ModelDocument.Type.INLINEXBRLDOCUMENTSET
): # ignore EdgarRenderer added non-DTS documents
checkFilingDTS(val, referencedDocument, isEFM, isGFM, visited)
if modelDocument.type == ModelDocument.Type.INLINEXBRLDOCUMENTSET:
return # nothing to check in inline document set surrogate parent
if val.disclosureSystem.standardTaxonomiesDict is None:
pass
if isEFM:
if modelDocument.uri in val.disclosureSystem.standardTaxonomiesDict:
if modelDocument.targetNamespace:
# check for duplicates of us-types, dei, and rr taxonomies
match = standardNamespacesPattern.match(
modelDocument.targetNamespace)
if match is not None:
conflictClass = match.group(2) or match.group(5)
if (conflictClass == 'us-gaap' and
match.group(3) < '2018') or conflictClass == 'srt':
val.standardNamespaceConflicts['srt+us-gaap'].add(
modelDocument
) # ignore non-srt multi-usgaap in Filing.py
if conflictClass == 'us-gaap' or conflictClass == 'ifrs-full':
val.standardNamespaceConflicts['ifrs+us-gaap'].add(
modelDocument)
if conflictClass not in ('us-gaap', 'srt'):
val.standardNamespaceConflicts[conflictClass].add(
modelDocument)
else:
if len(modelDocument.basename) > 32:
val.modelXbrl.error(
"EFM.5.01.01.tooManyCharacters",
_("Document file name %(filename)s must not exceed 32 characters."
),
edgarCode="cp-0501-File-Name-Length",
modelObject=modelDocument,
filename=modelDocument.basename)
if modelDocument.type != ModelDocument.Type.INLINEXBRLDOCUMENTSET:
if modelDocument.type == ModelDocument.Type.INLINEXBRL:
_pattern = htmlFileNamePattern
_suffix = ".htm"
else:
_pattern = efmFilenamePattern
_suffix = ".xsd or .xml"
if not _pattern.match(modelDocument.basename):
val.modelXbrl.error(
"EFM.5.01.01",
_("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with %(suffix)s."
),
edgarCode="cp-0501-File-Name",
modelObject=modelDocument,
filename=modelDocument.basename,
suffix=_suffix)
if (modelDocument.type == ModelDocument.Type.SCHEMA
and modelDocument.targetNamespace
not in val.disclosureSystem.baseTaxonomyNamespaces
and modelDocument.uri.startswith(val.modelXbrl.uriDir)):
val.hasExtensionSchema = True
# check schema contents types
# 6.7.3 check namespace for standard authority
targetNamespaceAuthority = UrlUtil.authority(
modelDocument.targetNamespace)
if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities:
val.modelXbrl.error(
("EFM.6.07.03", "GFM.1.03.03"),
_("The target namespace, %(targetNamespace)s cannot have the same authority (%(targetNamespaceAuthority)s) as a standard "
"taxonomy, in %(schema)s. Please change your target namespace."
),
edgarCode="fs-0703-Extension-Has-Standard-Namespace-Authority",
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace,
targetNamespaceAuthority=UrlUtil.authority(
modelDocument.targetNamespace, includeScheme=False))
# 6.7.4 check namespace format
if modelDocument.targetNamespace is None or not modelDocument.targetNamespace.startswith(
"http://"):
match = None
else:
targetNamespaceDate = modelDocument.targetNamespace[
len(targetNamespaceAuthority):]
match = targetNamespaceDatePattern.match(targetNamespaceDate)
if match is not None:
try:
if match.lastindex == 3:
date = datetime.date(int(match.group(1)),
int(match.group(2)),
int(match.group(3)))
elif match.lastindex == 6:
date = datetime.date(int(match.group(4)),
int(match.group(5)),
int(match.group(6)))
else:
match = None
except ValueError:
match = None
if match is None:
val.modelXbrl.error(
("EFM.6.07.04", "GFM.1.03.04"),
_("You did not adhere to the requirements for target namespace, for %(targetNamespace)s in %(schema)s. "
"Please recheck your submission and adhere to the target namespace requirements."
),
edgarCode="cp-0704-Taxonomy-Valid-Target-Namespace",
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace)
elif val.fileNameDate and date > val.fileNameDate:
val.modelXbrl.info(
("EFM.6.07.06", "GFM.1.03.06"),
_("Warning: Taxonomy schema %(schema)s namespace %(targetNamespace)s has date later than document name date %(docNameDate)s"
),
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace,
docNameDate=val.fileNameDate)
if modelDocument.targetNamespace is not None:
# 6.7.5 check prefix for _
authority = UrlUtil.authority(modelDocument.targetNamespace)
if not re.match(r"(http://|https://|ftp://|urn:)\w+", authority):
val.modelXbrl.error(
("EFM.6.07.05", "GFM.1.03.05"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s must be a valid URI with a valid authority for the namespace."
),
edgarCode="du-0705-Namespace-Authority",
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace)
# may be multiple prefixes for namespace
prefixes = [
(prefix if prefix is not None else "")
for prefix, NS in modelDocument.xmlRootElement.nsmap.items()
if NS == modelDocument.targetNamespace
]
if not prefixes:
prefix = None
val.modelXbrl.error(
("EFM.6.07.07", "GFM.1.03.07"),
_("The schema does not supply a prefix for %(targetNamespace)s without an underscore character, in file %(schema)s. "
"Please provide or change a prefix"),
edgarCode="du-0707-Recommended-Prefix-Disallowed",
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace)
else:
prefix = prefixes[0]
if len(prefixes) > 1:
val.modelXbrl.error(
("EFM.6.07.07", "GFM.1.03.07"),
_("The schema does not supply a prefix for %(targetNamespace)s without an underscore character, in file %(schema)s. "
"Please provide or change a prefix"),
edgarCode="du-0707-Recommended-Prefix-Disallowed",
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace,
prefix=", ".join(prefixes))
elif "_" in prefix:
val.modelXbrl.error(
("EFM.6.07.07", "GFM.1.03.07"),
_("The schema does not supply a prefix for %(targetNamespace)s without an underscore character, in file %(schema)s. "
"Please provide or change a prefix"),
edgarCode="du-0707-Recommended-Prefix-Disallowed",
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace,
prefix=prefix)
for modelConcept in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.w3.org/2001/XMLSchema}element"):
if isinstance(modelConcept, ModelConcept):
# 6.7.16 name not duplicated in standard taxonomies
name = modelConcept.get("name")
if name is None:
name = ""
if modelConcept.get("ref") is not None:
continue # don't validate ref's here
for c in val.modelXbrl.nameConcepts.get(name, []):
if c.modelDocument != modelDocument:
if not c.modelDocument.uri.startswith(
val.modelXbrl.uriDir):
val.modelXbrl.error(
("EFM.6.07.16", "GFM.1.03.18"),
_("Your extension taxonomy contains an element, %(concept)s, which has the same name as an element in the base taxonomy, "
"%(standardConcept)s. Please ensure that this extension is appropriate and if so, please change the extension concept name."
),
edgarCode=
"cp-0716-Element-Name-Same-As-Base",
modelObject=(modelConcept, c),
concept=modelConcept.qname,
standardSchema=c.modelDocument.basename,
standardConcept=c.qname)
# 6.7.17 id properly formed
_id = modelConcept.id
requiredId = (prefix
if prefix is not None else "") + "_" + name
if _id != requiredId:
val.modelXbrl.error(
("EFM.6.07.17", "GFM.1.03.19"),
_("You did not adhere to the declarations for concepts by containing an 'id' attribute whose value begins with the recommended "
"namespace prefix of the taxonomy, followed by an underscore, followed by an element name, for the concept %(concept)s. "
"Please recheck your submission."),
edgarCode="cp-0717-Element-Id",
modelObject=modelConcept,
concept=modelConcept.qname,
id=_id,
requiredId=requiredId)
# 6.7.18 nillable is true
nillable = modelConcept.get("nillable")
if nillable != "true" and modelConcept.isItem:
val.modelXbrl.error(
("EFM.6.07.18", "GFM.1.03.20"),
_("Element %(concept)s is declared without a 'true' value for the nillable attribute. Please set the value to 'true'."
),
edgarCode="du-0718-Nillable-Not-True",
modelObject=modelConcept,
schema=modelDocument.basename,
concept=name,
nillable=nillable)
# 6.7.19 not tuple
if modelConcept.isTuple:
val.modelXbrl.error(
("EFM.6.07.19", "GFM.1.03.21"),
_("You provided an extension concept which is a tuple, %(concept)s. Please remove tuples and check your submission."
),
edgarCode="cp-0719-No-Tuple-Element",
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.7.20 no typed domain ref
if modelConcept.isTypedDimension:
val.modelXbrl.error(
("EFM.6.07.20", "GFM.1.03.22"),
_("There is an xbrldt:typedDomainRef attribute on %(concept)s (%(typedDomainRef)s). Please remove it."
),
edgarCode="du-0720-Typed-Domain-Ref-Disallowed",
modelObject=modelConcept,
concept=modelConcept.qname,
typedDomainRef=modelConcept.typedDomainElement.
qname if modelConcept.typedDomainElement
is not None else modelConcept.typedDomainRef)
# 6.7.21 abstract must be duration
isDuration = modelConcept.periodType == "duration"
if modelConcept.isAbstract and not isDuration:
val.modelXbrl.error(
("EFM.6.07.21", "GFM.1.03.23"),
_("Element %(concept)s is declared as an abstract item with period type 'instant'. Please change it to 'duration' or "
"make the element not abstract."),
edgarCode="du-0721-Abstract-Is-Instant",
modelObject=modelConcept,
schema=modelDocument.basename,
concept=modelConcept.qname)
# 6.7.22 abstract must be stringItemType
''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08
if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType:
val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"),
_("Concept %(concept)s is abstract but type is not xbrli:stringItemType"),
modelObject=modelConcept, concept=modelConcept.qname)
'''
substitutionGroupQname = modelConcept.substitutionGroupQname
# 6.7.23 Axis must be subs group dimension
if name.endswith("Axis") ^ (
substitutionGroupQname
== XbrlConst.qnXbrldtDimensionItem):
val.modelXbrl.error(
("EFM.6.07.23", "GFM.1.03.25"),
_("The substitution group 'xbrldt:dimensionItem' is only consistent with an element name that ends with 'Axis'. "
"Please change %(conceptLocalName)s or change the substitutionGroup."
),
edgarCode="du-0723-Axis-Dimension-Name-Mismatch",
modelObject=modelConcept,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName)
# 6.7.24 Table must be subs group hypercube
if name.endswith("Table") ^ (
substitutionGroupQname
== XbrlConst.qnXbrldtHypercubeItem):
val.modelXbrl.error(
("EFM.6.07.24", "GFM.1.03.26"),
_("The substitution group 'xbrldt:hypercubeItem' is only allowed with an element name that ends with 'Table'. "
"Please change %(conceptLocalName)s or change the substitutionGroup."
),
edgarCode="du-0724-Table-Hypercube-Name-Mismatch",
modelObject=modelConcept,
schema=modelDocument.basename,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName)
# 6.7.25 if neither hypercube or dimension, substitution group must be item
if substitutionGroupQname not in (
None, XbrlConst.qnXbrldtDimensionItem,
XbrlConst.qnXbrldtHypercubeItem,
XbrlConst.qnXbrliItem):
val.modelXbrl.error(
("EFM.6.07.25", "GFM.1.03.27"),
_("The substitution group attribute value %(substitutionGroup)s of element %(conceptLocalName)s is not allowed. "
"Please change it to one of 'xbrli:item', 'xbrldt:dimensionItem' or 'xbrldt:hypercubeItem'."
),
edgarCode="du-0725-Substitution-Group-Custom",
modelObject=modelConcept,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName,
substitutionGroup=modelConcept.
substitutionGroupQname)
# 6.7.26 Table must be subs group hypercube
if name.endswith(
"LineItems") and modelConcept.abstract != "true":
val.modelXbrl.error(
("EFM.6.07.26", "GFM.1.03.28"),
_("The element %(conceptLocalName)s ends with 'LineItems' but is not abstract. Please change %(conceptLocalName)s or "
"the value of the 'abstract' attribute."),
edgarCode=
"du-0726-LineItems-Abstract-Name-Mismatch",
modelObject=modelConcept,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName)
# 6.7.27 type domainMember must end with Domain or Member
conceptType = modelConcept.type
isDomainItemType = conceptType is not None and conceptType.isDomainItemType
endsWithDomainOrMember = name.endswith(
"Domain") or name.endswith("Member")
if isDomainItemType != endsWithDomainOrMember:
val.modelXbrl.error(
("EFM.6.07.27", "GFM.1.03.29"),
_("The type 'us-types:domainItemType' is only allowed with an element name that ends with 'Domain' or 'Member'. "
"Please change %(conceptLocalName)s or change the type."
),
edgarCode="du-0727-Domain-Type-Name-Mismatch",
modelObject=modelConcept,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName)
# 6.7.28 domainItemType must be duration
if isDomainItemType and not isDuration:
val.modelXbrl.error(
("EFM.6.07.28", "GFM.1.03.30"),
_("Element %(conceptLocalName)s is declared as a us-types:domainItemType with period type 'instant'. "
"Please change it to 'duration' or change the item type."
),
edgarCode="du-0728-Domain-Member-Is-Instant",
modelObject=modelConcept,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName)
#6.7.31 (version 27) fractions
if modelConcept.isFraction:
val.modelXbrl.error(
"EFM.6.07.31",
_("Element %(concept)s is declared as a fraction item type. Change or remove the declaration."
),
edgarCode="du-0731-Fraction-Item-Type",
modelObject=modelConcept,
concept=modelConcept.qname)
#6.7.32 (version 27) instant non numeric
if modelConcept.isItem and (not modelConcept.isNumeric
and not isDuration
and not modelConcept.isAbstract
and not isDomainItemType):
val.modelXbrl.error(
"EFM.6.07.32",
_("Declaration of element %(concept)s in %(schema)s must have xbrli:periodType of 'duration' because its base type is not numeric."
),
edgarCode="rq-0732-Nonnnumeric-Must-Be-Duration",
modelObject=modelConcept,
schema=modelDocument.basename,
concept=modelConcept.qname)
# 6.8.5 semantic check, check LC3 name
if name:
if not name[0].isupper():
val.modelXbrl.log(
"ERROR-SEMANTIC", ("EFM.6.08.05.firstLetter",
"GFM.2.03.05.firstLetter"),
_("Concept %(concept)s name must start with a capital letter"
),
modelObject=modelConcept,
concept=modelConcept.qname)
if namePattern.search(name):
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.05.disallowedCharacter",
"GFM.2.03.05.disallowedCharacter"),
_("Concept %(concept)s has disallowed name character"
),
modelObject=modelConcept,
concept=modelConcept.qname)
if len(name) > 200:
val.modelXbrl.log(
"ERROR-SEMANTIC",
"EFM.6.08.05.nameLength",
_("Concept %(concept)s name length %(namelength)s exceeds 200 characters"
),
modelObject=modelConcept,
concept=modelConcept.qname,
namelength=len(name))
if isEFM:
label = modelConcept.label(lang="en-US",
fallbackToQname=False)
if label:
# allow Joe's Bar, N.A. to be JoesBarNA -- remove ', allow A. as not article "a"
lc3name = ''.join(
re.sub(r"['.-]", "", (
w[0] or w[2] or w[3] or w[4])).title()
for w in re.findall(
r"((\w+')+\w+)|(A[.-])|([.-]A(?=\W|$))|(\w+)",
label
) # EFM implies this should allow - and . re.findall(r"[\w\-\.]+", label)
if w[4].lower() not in ("the", "a", "an"))
if not (name == lc3name or
(name and lc3name and lc3name[0].isdigit()
and name[1:] == lc3name and
(name[0].isalpha() or name[0] == '_'))):
val.modelXbrl.log(
"WARNING-SEMANTIC",
"EFM.6.08.05.LC3",
_("Concept %(concept)s should match expected LC3 composition %(lc3name)s"
),
modelObject=modelConcept,
concept=modelConcept.qname,
lc3name=lc3name)
if conceptType is not None:
# 6.8.6 semantic check
if not isDomainItemType and conceptType.qname != XbrlConst.qnXbrliDurationItemType:
nameProblems = nonDomainItemNameProblemPattern.findall(
name)
if any(
any(t) for t in nameProblems
): # list of tuples with possibly nonempty strings
val.modelXbrl.log(
"WARNING-SEMANTIC",
("EFM.6.08.06", "GFM.2.03.06"),
_("Concept %(concept)s should not contain company or period information, found: %(matches)s"
),
modelObject=modelConcept,
concept=modelConcept.qname,
matches=", ".join(''.join(t)
for t in nameProblems))
if conceptType.qname == XbrlConst.qnXbrliMonetaryItemType:
if not modelConcept.balance:
# 6.8.11 may not appear on a income or balance statement
if any(
linkroleDefinitionBalanceIncomeSheet.
match(roleType.definition)
for rel in val.modelXbrl.
relationshipSet(XbrlConst.parentChild).
toModelObject(modelConcept)
for roleType in val.modelXbrl.
roleTypes.get(rel.linkrole, ())):
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.11", "GFM.2.03.11"),
_("Concept %(concept)s must have a balance because it appears in a statement of income or balance sheet"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.11.5 semantic check, must have a documentation label
stdLabel = modelConcept.label(
lang="en-US", fallbackToQname=False)
defLabel = modelConcept.label(
preferredLabel=XbrlConst.
documentationLabel,
lang="en-US",
fallbackToQname=False)
if not defLabel or ( # want different words than std label
stdLabel
and re.findall(r"\w+", stdLabel)
== re.findall(r"\w+", defLabel)):
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.11.05", "GFM.2.04.04"),
_("Concept %(concept)s is monetary without a balance and must have a documentation label that disambiguates its sign"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.8.16 semantic check
if conceptType.qname == XbrlConst.qnXbrliDateItemType and modelConcept.periodType != "duration":
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.16", "GFM.2.03.16"),
_("Concept %(concept)s of type xbrli:dateItemType must have periodType duration"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.8.17 semantic check
if conceptType.qname == XbrlConst.qnXbrliStringItemType and modelConcept.periodType != "duration":
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.17", "GFM.2.03.17"),
_("Concept %(concept)s of type xbrli:stringItemType must have periodType duration"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.7.8 check for embedded linkbase
for e in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
if isinstance(e, ModelObject):
val.modelXbrl.error(
("EFM.6.07.08", "GFM.1.03.08"),
_("Your filing contained embedded linkbases in %(schema)s. Please recheck your submission and remove all embedded linkbases."
),
edgarCode="cp-0708-No-Embedded-Linkbases",
modelObject=e,
schema=modelDocument.basename)
break
requiredUsedOns = {
XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink,
XbrlConst.qnLinkDefinitionLink
}
standardUsedOns = {
XbrlConst.qnLinkLabel,
XbrlConst.qnLinkReference,
XbrlConst.qnLinkDefinitionArc,
XbrlConst.qnLinkCalculationArc,
XbrlConst.qnLinkPresentationArc,
XbrlConst.qnLinkLabelArc,
XbrlConst.qnLinkReferenceArc,
# per WH, private footnote arc and footnore resource roles are not allowed
XbrlConst.qnLinkFootnoteArc,
XbrlConst.qnLinkFootnote,
}
# 6.7.9 role types authority
for e in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.xbrl.org/2003/linkbase}roleType"):
if isinstance(e, ModelObject):
roleURI = e.get("roleURI")
if targetNamespaceAuthority != UrlUtil.authority(roleURI):
val.modelXbrl.error(
("EFM.6.07.09", "GFM.1.03.09"),
_("Role %(roleType)s does not begin with %(targetNamespace)s's scheme and authority. "
"Please change the role URI or target namespace URI."
),
edgarCode="du-0709-Role-Namespace-Mismatch",
modelObject=e,
roleType=roleURI,
targetNamespaceAuthority=targetNamespaceAuthority,
targetNamespace=modelDocument.targetNamespace)
# 6.7.9 end with .../role/lc3 name
if not roleTypePattern.match(roleURI):
val.modelXbrl.warning(
("EFM.6.07.09.roleEnding", "GFM.1.03.09"),
"RoleType %(roleType)s should end with /role/{LC3name}",
modelObject=e,
roleType=roleURI)
# 6.7.10 only one role type declaration in DTS
modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI)
if modelRoleTypes is not None:
modelRoleType = modelRoleTypes[0]
definition = modelRoleType.definitionNotStripped
usedOns = modelRoleType.usedOns
if len(modelRoleTypes) == 1:
# 6.7.11 used on's for pre, cal, def if any has a used on
if not usedOns.isdisjoint(requiredUsedOns) and len(
requiredUsedOns - usedOns) > 0:
val.modelXbrl.error(
("EFM.6.07.11", "GFM.1.03.11"),
_("The role %(roleType)s did not provide a usedOn element for all three link types (presentation, "
"calculation and definition), missing %(usedOn)s. Change the declaration to be for all three types of link, and resubmit."
),
edgarCode=
"du-0711-Role-Type-Declaration-Incomplete",
modelObject=e,
roleType=roleURI,
usedOn=requiredUsedOns - usedOns)
# 6.7.12 definition match pattern
if (val.disclosureSystem.roleDefinitionPattern
is not None and
(definition is None or not val.disclosureSystem.
roleDefinitionPattern.match(definition))):
val.modelXbrl.error(
("EFM.6.07.12", "GFM.1.03.12-14"),
_("The definition '%(definition)s' of role %(roleType)s does not match the expected format. "
"Please check that the definition matches {number} - {type} - {text}."
),
edgarCode="rq-0712-Role-Definition-Mismatch",
modelObject=e,
roleType=roleURI,
definition=(definition or ""))
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log(
"ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("RoleType %(roleuri)s is defined using role types already defined by standard roles for: %(qnames)s"
),
modelObject=e,
roleuri=roleURI,
qnames=', '.join(
str(qn) for qn in usedOns & standardUsedOns))
# 6.7.13 arcrole types authority
for e in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.xbrl.org/2003/linkbase}arcroleType"):
if isinstance(e, ModelObject):
arcroleURI = e.get("arcroleURI")
if targetNamespaceAuthority != UrlUtil.authority(arcroleURI):
val.modelXbrl.error(
("EFM.6.07.13", "GFM.1.03.15"),
_("Relationship role %(arcroleType)s does not begin with %(targetNamespace)s's scheme and authority. "
"Please change the relationship role URI or target namespace URI."
),
edgarCode="du-0713-Arcrole-Namespace-Mismatch",
modelObject=e,
arcroleType=arcroleURI,
targetNamespaceAuthority=targetNamespaceAuthority,
targetNamespace=modelDocument.targetNamespace)
# 6.7.13 end with .../arcrole/lc3 name
if not arcroleTypePattern.match(arcroleURI):
val.modelXbrl.warning(
("EFM.6.07.13.arcroleEnding", "GFM.1.03.15"),
_("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"
),
modelObject=e,
arcroleType=arcroleURI)
# 6.7.15 definition match pattern
modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI]
definition = modelRoleTypes[0].definition
if definition is None or not arcroleDefinitionPattern.match(
definition):
val.modelXbrl.error(
("EFM.6.07.15", "GFM.1.03.17"),
_("Relationship role declaration %(arcroleType)s is missing a definition. Please provide a definition."
),
edgarCode="du-0715-Arcrole-Definition-Missing",
modelObject=e,
arcroleType=arcroleURI)
# semantic checks
usedOns = modelRoleTypes[0].usedOns
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log(
"ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("ArcroleType %(arcroleuri)s is defined using role types already defined by standard arcroles for: %(qnames)s"
),
modelObject=e,
arcroleuri=arcroleURI,
qnames=', '.join(
str(qn) for qn in usedOns & standardUsedOns))
#6.3.3 filename check
m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9]).xsd$",
modelDocument.basename)
if m:
try: # check date value
datetime.datetime.strptime(m.group(1), "%Y%m%d").date()
# date and format are ok, check "should" part of 6.3.3
if val.fileNameBasePart:
expectedFilename = "{0}-{1}.xsd".format(
val.fileNameBasePart, val.fileNameDatePart)
if modelDocument.basename != expectedFilename:
val.modelXbrl.log(
"WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance",
"GFM.1.01.01.matchInstance"),
_('Schema file name warning: %(filename)s, should match %(expectedFilename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
expectedFilename=expectedFilename)
except ValueError:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid schema file base name part (date) in "{base}-{yyyymmdd}.xsd": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
else:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid schema file name, must match "{base}-{yyyymmdd}.xsd": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
elif modelDocument.type == ModelDocument.Type.LINKBASE:
# if it is part of the submission (in same directory) check name
labelRels = None
if modelDocument.filepath.startswith(
val.modelXbrl.modelDocument.filepathdir):
#6.3.3 filename check
extLinkElt = XmlUtil.descendant(
modelDocument.xmlRootElement, XbrlConst.link, "*",
"{http://www.w3.org/1999/xlink}type", "extended")
if extLinkElt is None: # no ext link element
val.modelXbrl.error(
(val.EFM60303 + ".noLinkElement",
"GFM.1.01.01.noLinkElement"),
_('Invalid linkbase file name: %(filename)s, has no extended link element, cannot determine link type.'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03.noLinkElement",
"EFM.6.23.01.noLinkElement",
"GFM.1.01.01.noLinkElement"))
elif extLinkElt.localName not in extLinkEltFileNameEnding:
val.modelXbrl.error(
"EFM.6.03.02",
_('Invalid linkbase link element %(linkElement)s in %(filename)s'
),
modelObject=modelDocument,
linkElement=extLinkElt.localName,
filename=modelDocument.basename)
else:
m = re.match(
r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9])(_[a-z]{3}).xml$",
modelDocument.basename)
expectedSuffix = extLinkEltFileNameEnding[extLinkElt.localName]
if m and m.group(2) == expectedSuffix:
try: # check date value
datetime.datetime.strptime(m.group(1), "%Y%m%d").date()
# date and format are ok, check "should" part of 6.3.3
if val.fileNameBasePart:
expectedFilename = "{0}-{1}{2}.xml".format(
val.fileNameBasePart, val.fileNameDatePart,
expectedSuffix)
if modelDocument.basename != expectedFilename:
val.modelXbrl.log(
"WARNING-SEMANTIC",
("EFM.6.03.03.matchInstance",
"GFM.1.01.01.matchInstance"),
_('Linkbase name warning: %(filename)s should match %(expectedFilename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
expectedFilename=expectedFilename)
except ValueError:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid linkbase base file name part (date) in "{base}-{yyyymmdd}_{suffix}.xml": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01",
"GFM.1.01.01"))
else:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid linkbase name, must match "{base}-{yyyymmdd}%(expectedSuffix)s.xml": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
expectedSuffix=expectedSuffix,
messageCodes=("EFM.6.03.03", "EFM.6.23.01",
"GFM.1.01.01"))
if extLinkElt.localName == "labelLink":
if labelRels is None:
labelRels = val.modelXbrl.relationshipSet(
XbrlConst.conceptLabel)
for labelElt in XmlUtil.children(extLinkElt,
XbrlConst.link, "label"):
# 6.10.9
if XbrlConst.isNumericRole(labelElt.role):
for rel in labelRels.toModelObject(labelElt):
if rel.fromModelObject is not None and not rel.fromModelObject.isNumeric:
val.modelXbrl.error(
"EFM.6.10.09",
_("Non-numeric element %(concept)s has a label role for numeric elements: %(role)s. "
"Please change the role attribute."),
edgarCode="du-1009-Numeric-Label-Role",
modelObject=(labelElt,
rel.fromModelObject),
concept=rel.fromModelObject.qname,
role=labelElt.role)
def modelObjectDocumentUri(modelObject):
return UrlUtil.ensureUrl(modelObject.modelDocument.uri)
def logArguments(self, codes, msg, codedArgs):
# determine logCode
messageCode = None
for argCode in codes if isinstance(codes, tuple) else (codes, ):
if (isinstance(argCode, ModelValue.QName)
or (self.modelManager.disclosureSystem.EFM
and argCode.startswith("EFM"))
or (self.modelManager.disclosureSystem.GFM
and argCode.startswith("GFM"))
or (self.modelManager.disclosureSystem.HMRC
and argCode.startswith("HMRC"))
or (self.modelManager.disclosureSystem.SBRNL
and argCode.startswith("SBR.NL"))
or argCode[0:3] not in ("EFM", "GFM", "HMR", "SBR")):
messageCode = argCode
break
# determine message and extra arguments
fmtArgs = {}
extras = {"messageCode": messageCode}
for argName, argValue in codedArgs.items():
if argName in ("modelObject", "modelXbrl", "modelDocument"):
try:
entryUrl = self.modelDocument.uri
except AttributeError:
entryUrl = self.entryLoadingUrl
refs = []
for arg in (argValue if isinstance(argValue,
(tuple, list)) else
(argValue, )):
if arg is not None:
if isinstance(arg, _STR_BASE):
objectUrl = arg
else:
try:
objectUrl = arg.modelDocument.uri
except AttributeError:
try:
objectUrl = self.modelDocument.uri
except AttributeError:
objectUrl = self.entryLoadingUrl
file = UrlUtil.relativeUri(entryUrl, objectUrl)
ref = {}
if isinstance(arg, ModelObject):
ref["href"] = file + "#" + XmlUtil.elementFragmentIdentifier(
arg)
ref["sourceLine"] = arg.sourceline
ref["objectId"] = arg.objectId()
else:
ref["href"] = file
refs.append(ref)
extras["refs"] = refs
elif argName == "sourceLine":
if isinstance(
argValue,
_INT_TYPES): # must be sortable with int's in logger
extras["sourceLine"] = argValue
elif argName != "exc_info":
if isinstance(
argValue,
(ModelValue.QName, ModelObject, bool, FileNamedStringIO)):
fmtArgs[argName] = str(argValue)
elif isinstance(argValue, _INT_TYPES):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale,
'%i', argValue)
elif isinstance(argValue, float):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale,
'%f', argValue)
else:
fmtArgs[argName] = argValue
if "refs" not in extras:
try:
file = os.path.basename(self.modelDocument.uri)
except AttributeError:
try:
file = os.path.basename(self.entryLoadingUrl)
except:
file = ""
extras["refs"] = [{"href": file}]
return (messageCode, (msg, fmtArgs) if fmtArgs else (msg, ), extras)
def validateValue(modelXbrl,
elt,
attrTag,
baseXsdType,
value,
isNillable=False,
isNil=False,
facets=None):
if baseXsdType:
try:
'''
if (len(value) == 0 and attrTag is None and not isNillable and
baseXsdType not in ("anyType", "string", "normalizedString", "token", "NMTOKEN", "anyURI", "noContent")):
raise ValueError("missing value for not nillable element")
'''
xValid = VALID
whitespaceReplace = (baseXsdType == "normalizedString")
whitespaceCollapse = (not whitespaceReplace
and baseXsdType != "string")
pattern = baseXsdTypePatterns.get(baseXsdType)
if facets:
if "pattern" in facets:
pattern = facets["pattern"]
# note multiple patterns are or'ed togetner, which isn't yet implemented!
if "whiteSpace" in facets:
whitespaceReplace, whitespaceCollapse = {
"preserve": (False, False),
"replace": (True, False),
"collapse": (False, True)
}[facets["whiteSpace"]]
if whitespaceReplace:
value = normalizeWhitespacePattern.sub(' ', value)
elif whitespaceCollapse:
value = collapseWhitespacePattern.sub(' ', value.strip())
if pattern is not None and pattern.match(value) is None:
raise ValueError("pattern facet " +
facets["pattern"].pattern if facets and
"pattern" in facets else "pattern mismatch")
if facets:
if "enumeration" in facets and value not in facets[
"enumeration"]:
raise ValueError("{0} is not in {1}".format(
value, facets["enumeration"]))
if "length" in facets and len(value) != facets["length"]:
raise ValueError("length {0}, expected {1}".format(
len(value), facets["length"]))
if "minLength" in facets and len(value) < facets["minLength"]:
raise ValueError("length {0}, minLength {1}".format(
len(value), facets["minLength"]))
if "maxLength" in facets and len(value) > facets["maxLength"]:
raise ValueError("length {0}, maxLength {1}".format(
len(value), facets["maxLength"]))
if baseXsdType == "noContent":
if len(value) > 0 and not value.isspace():
raise ValueError("value content not permitted")
xValue = sValue = None
elif baseXsdType in {
"string", "normalizedString", "language", "token",
"NMTOKEN", "Name", "NCName", "IDREF", "ENTITY"
}:
xValue = sValue = value
elif baseXsdType == "ID":
xValue = sValue = value
xValid = VALID_ID
elif baseXsdType == "anyURI":
if value: # allow empty strings to be valid anyURIs
if UrlUtil.relativeUrlPattern.match(value) is None:
raise ValueError("IETF RFC 2396 4.3 syntax")
# encode PSVI xValue similarly to Xerces and other implementations
xValue = anyURI(UrlUtil.anyUriQuoteForPSVI(value))
sValue = value
elif not value and isNil and isNillable: # rest of types get None if nil/empty value
xValue = sValue = None
elif baseXsdType in ("decimal", "float", "double"):
xValue = sValue = float(value)
if facets:
if "totalDigits" in facets and len(value.replace(
".", "")) > facets["totalDigits"]:
raise ValueError("totalDigits facet {0}".format(
facets["totalDigits"]))
if "fractionDigits" in facets and (
'.' in value and len(value[value.index('.') + 1:])
> facets["fractionDigits"]):
raise ValueError("fraction digits facet {0}".format(
facets["fractionDigits"]))
if "maxInclusive" in facets and xValue > facets[
"maxInclusive"]:
raise ValueError(" > maxInclusive {0}".format(
facets["maxInclusive"]))
if "maxExclusive" in facets and xValue >= facets[
"maxExclusive"]:
raise ValueError(" >= maxInclusive {0}".format(
facets["maxExclusive"]))
if "minInclusive" in facets and xValue < facets[
"minInclusive"]:
raise ValueError(" < minInclusive {0}".format(
facets["minInclusive"]))
if "minExclusive" in facets and xValue <= facets[
"minExclusive"]:
raise ValueError(" <= minExclusive {0}".format(
facets["minExclusive"]))
elif baseXsdType in {
"integer", "nonPositiveInteger", "negativeInteger",
"nonNegativeInteger", "positiveInteger", "long",
"unsignedLong", "int", "unsignedInt", "short",
"unsignedShort", "byte", "unsignedByte"
}:
xValue = sValue = _INT(value)
if ((baseXsdType
in {"nonNegativeInteger", "unsignedLong", "unsignedInt"}
and xValue < 0)
or (baseXsdType == "nonPositiveInteger" and xValue > 0)
or (baseXsdType == "positiveInteger" and xValue <= 0)
or (baseXsdType == "byte" and not -128 <= xValue < 127)
or
(baseXsdType == "unsignedByte" and not 0 <= xValue < 255)
or
(baseXsdType == "short" and not -32768 <= xValue < 32767)
or (baseXsdType == "unsignedShort"
and not 0 <= xValue < 65535)
or (baseXsdType == "positiveInteger" and xValue <= 0)):
raise ValueError("{0} is not {1}".format(
value, baseXsdType))
if facets:
if "totalDigits" in facets and len(value.replace(
".", "")) > facets["totalDigits"]:
raise ValueError("totalDigits facet {0}".format(
facets["totalDigits"]))
if "fractionDigits" in facets and (
'.' in value and len(value[value.index('.') + 1:])
> facets["fractionDigits"]):
raise ValueError("fraction digits facet {0}".format(
facets["fractionDigits"]))
if "maxInclusive" in facets and xValue > facets[
"maxInclusive"]:
raise ValueError(" > maxInclusive {0}".format(
facets["maxInclusive"]))
if "maxExclusive" in facets and xValue >= facets[
"maxExclusive"]:
raise ValueError(" >= maxInclusive {0}".format(
facets["maxExclusive"]))
if "minInclusive" in facets and xValue < facets[
"minInclusive"]:
raise ValueError(" < minInclusive {0}".format(
facets["minInclusive"]))
if "minExclusive" in facets and xValue <= facets[
"minExclusive"]:
raise ValueError(" <= minExclusive {0}".format(
facets["minExclusive"]))
elif baseXsdType == "boolean":
if value in ("true", "1"):
xValue = sValue = True
elif value in ("false", "0"):
xValue = sValue = False
else:
raise ValueError
elif baseXsdType == "QName":
xValue = qname(elt,
value,
castException=ValueError,
prefixException=ValueError)
sValue = value
''' not sure here, how are explicitDimensions validated, but bad units not?
if xValue.namespaceURI in modelXbrl.namespaceDocs:
if (xValue not in modelXbrl.qnameConcepts and
xValue not in modelXbrl.qnameTypes and
xValue not in modelXbrl.qnameAttributes and
xValue not in modelXbrl.qnameAttributeGroups):
raise ValueError("qname not defined " + str(xValue))
'''
elif baseXsdType in ("XBRLI_DECIMALSUNION",
"XBRLI_PRECISIONUNION"):
xValue = sValue = value if value == "INF" else _INT(value)
elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
xValue = sValue = _INT(value)
if xValue == 0:
raise ValueError("invalid value")
elif baseXsdType == "XBRLI_DATEUNION":
xValue = dateTime(value,
type=DATEUNION,
castException=ValueError)
sValue = value
elif baseXsdType == "dateTime":
xValue = dateTime(value,
type=DATETIME,
castException=ValueError)
sValue = value
elif baseXsdType == "date":
xValue = dateTime(value, type=DATE, castException=ValueError)
sValue = value
elif baseXsdType == "regex-pattern":
# for facet compiling
try:
sValue = value
if value in xmlSchemaPatterns:
xValue = xmlSchemaPatterns[value]
else:
if r"\i" in value or r"\c" in value:
value = value.replace(r"\i", iNameChar).replace(
r"\c", cNameChar)
xValue = re.compile(value +
"$") # must match whole string
except Exception as err:
raise ValueError(err)
else:
if baseXsdType in lexicalPatterns:
match = lexicalPatterns[baseXsdType].match(value)
if match is None:
raise ValueError("lexical pattern mismatch")
if baseXsdType == "gMonthDay":
month, day, zSign, zHrMin, zHr, zMin = match.groups()
if int(day) > {
2: 29,
4: 30,
6: 30,
9: 30,
11: 30,
1: 31,
3: 31,
5: 31,
7: 31,
8: 31,
10: 31,
12: 31
}[int(month)]:
raise ValueError(
"invalid day {0} for month {1}".format(
day, month))
xValue = value
sValue = value
except ValueError as err:
if ModelInlineFact is not None and isinstance(
elt, ModelInlineFact):
errElt = "{0} fact {1}".format(elt.elementQname, elt.qname)
else:
errElt = elt.elementQname
if attrTag:
modelXbrl.error(
"xmlSchema:valueError",
_("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"
),
modelObject=elt,
element=errElt,
attribute=XmlUtil.clarkNotationToPrefixedName(
elt, attrTag, isAttribute=True),
typeName=baseXsdType,
value=value,
error=err)
else:
modelXbrl.error(
"xmlSchema:valueError",
_("Element %(element)s type %(typeName)s value error: %(value)s, %(error)s"
),
modelObject=elt,
element=errElt,
typeName=baseXsdType,
value=value,
error=err)
xValue = None
sValue = value
xValid = INVALID
else:
xValue = sValue = None
xValid = UNKNOWN
if attrTag:
try: # dynamically allocate attributes (otherwise given shared empty set)
xAttributes = elt.xAttributes
except AttributeError:
elt.xAttributes = xAttributes = {}
xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue,
sValue, value)
else:
elt.xValid = xValid
elt.xValue = xValue
elt.sValue = sValue
def validateValue(modelXbrl, elt, attrTag, baseXsdType, value, isNillable=False, isNil=False, facets=None):
if baseXsdType:
try:
'''
if (len(value) == 0 and attrTag is None and not isNillable and
baseXsdType not in ("anyType", "string", "normalizedString", "token", "NMTOKEN", "anyURI", "noContent")):
raise ValueError("missing value for not nillable element")
'''
xValid = VALID
whitespaceReplace = (baseXsdType == "normalizedString")
whitespaceCollapse = (not whitespaceReplace and baseXsdType != "string")
isList = baseXsdType in {"IDREFS", "ENTITIES", "NMTOKENS"}
if isList:
baseXsdType = baseXsdType[:-1] # remove plural
pattern = baseXsdTypePatterns.get(baseXsdType)
if facets:
if "pattern" in facets:
pattern = facets["pattern"]
# note multiple patterns are or'ed togetner, which isn't yet implemented!
if "whiteSpace" in facets:
whitespaceReplace, whitespaceCollapse = {"preserve":(False,False), "replace":(True,False), "collapse":(False,True)}[facets["whiteSpace"]]
if whitespaceReplace:
value = normalizeWhitespacePattern.sub(' ', value)
elif whitespaceCollapse:
value = collapseWhitespacePattern.sub(' ', value.strip())
if baseXsdType == "noContent":
if len(value) > 0 and not value.isspace():
raise ValueError("value content not permitted")
# note that sValue and xValue are not innerText but only text elements on specific element (or attribute)
xValue = sValue = None
xValid = VALID_NO_CONTENT # notify others that element may contain subelements (for stringValue needs)
elif not value and isNil and isNillable: # rest of types get None if nil/empty value
xValue = sValue = None
else:
if pattern is not None:
if ((isList and any(pattern.match(v) is None for v in value.split())) or
(not isList and pattern.match(value) is None)):
raise ValueError("pattern facet " + facets["pattern"].pattern if facets and "pattern" in facets else "pattern mismatch")
if facets:
if "enumeration" in facets and value not in facets["enumeration"]:
raise ValueError("{0} is not in {1}".format(value, facets["enumeration"]))
if "length" in facets and len(value) != facets["length"]:
raise ValueError("length {0}, expected {1}".format(len(value), facets["length"]))
if "minLength" in facets and len(value) < facets["minLength"]:
raise ValueError("length {0}, minLength {1}".format(len(value), facets["minLength"]))
if "maxLength" in facets and len(value) > facets["maxLength"]:
raise ValueError("length {0}, maxLength {1}".format(len(value), facets["maxLength"]))
if baseXsdType in {"string", "normalizedString", "language", "token", "NMTOKEN","Name","NCName","IDREF","ENTITY"}:
xValue = sValue = value
elif baseXsdType == "ID":
xValue = sValue = value
xValid = VALID_ID
elif baseXsdType == "anyURI":
if value: # allow empty strings to be valid anyURIs
if UrlUtil.relativeUrlPattern.match(value) is None:
raise ValueError("IETF RFC 2396 4.3 syntax")
# encode PSVI xValue similarly to Xerces and other implementations
xValue = anyURI(UrlUtil.anyUriQuoteForPSVI(value))
sValue = value
elif baseXsdType in ("decimal", "float", "double"):
if baseXsdType == "decimal":
if decimalPattern.match(value) is None:
raise ValueError("lexical pattern mismatch")
xValue = Decimal(value)
sValue = float(value) # s-value uses Number (float) representation
else:
if floatPattern.match(value) is None:
raise ValueError("lexical pattern mismatch")
xValue = sValue = float(value)
if facets:
if "totalDigits" in facets and len(value.replace(".","")) > facets["totalDigits"]:
raise ValueError("totalDigits facet {0}".format(facets["totalDigits"]))
if "fractionDigits" in facets and ( '.' in value and
len(value[value.index('.') + 1:]) > facets["fractionDigits"]):
raise ValueError("fraction digits facet {0}".format(facets["fractionDigits"]))
if "maxInclusive" in facets and xValue > facets["maxInclusive"]:
raise ValueError(" > maxInclusive {0}".format(facets["maxInclusive"]))
if "maxExclusive" in facets and xValue >= facets["maxExclusive"]:
raise ValueError(" >= maxInclusive {0}".format(facets["maxExclusive"]))
if "minInclusive" in facets and xValue < facets["minInclusive"]:
raise ValueError(" < minInclusive {0}".format(facets["minInclusive"]))
if "minExclusive" in facets and xValue <= facets["minExclusive"]:
raise ValueError(" <= minExclusive {0}".format(facets["minExclusive"]))
elif baseXsdType in {"integer",
"nonPositiveInteger","negativeInteger","nonNegativeInteger","positiveInteger",
"long","unsignedLong",
"int","unsignedInt",
"short","unsignedShort",
"byte","unsignedByte"}:
xValue = sValue = _INT(value)
if ((baseXsdType in {"nonNegativeInteger","unsignedLong","unsignedInt"}
and xValue < 0) or
(baseXsdType == "nonPositiveInteger" and xValue > 0) or
(baseXsdType == "positiveInteger" and xValue <= 0) or
(baseXsdType == "byte" and not -128 <= xValue < 127) or
(baseXsdType == "unsignedByte" and not 0 <= xValue < 255) or
(baseXsdType == "short" and not -32768 <= xValue < 32767) or
(baseXsdType == "unsignedShort" and not 0 <= xValue < 65535) or
(baseXsdType == "positiveInteger" and xValue <= 0)):
raise ValueError("{0} is not {1}".format(value, baseXsdType))
if facets:
if "totalDigits" in facets and len(value.replace(".","")) > facets["totalDigits"]:
raise ValueError("totalDigits facet {0}".format(facets["totalDigits"]))
if "fractionDigits" in facets and ( '.' in value and
len(value[value.index('.') + 1:]) > facets["fractionDigits"]):
raise ValueError("fraction digits facet {0}".format(facets["fractionDigits"]))
if "maxInclusive" in facets and xValue > facets["maxInclusive"]:
raise ValueError(" > maxInclusive {0}".format(facets["maxInclusive"]))
if "maxExclusive" in facets and xValue >= facets["maxExclusive"]:
raise ValueError(" >= maxInclusive {0}".format(facets["maxExclusive"]))
if "minInclusive" in facets and xValue < facets["minInclusive"]:
raise ValueError(" < minInclusive {0}".format(facets["minInclusive"]))
if "minExclusive" in facets and xValue <= facets["minExclusive"]:
raise ValueError(" <= minExclusive {0}".format(facets["minExclusive"]))
elif baseXsdType == "boolean":
if value in ("true", "1"):
xValue = sValue = True
elif value in ("false", "0"):
xValue = sValue = False
else: raise ValueError
elif baseXsdType == "QName":
xValue = qnameEltPfxName(elt, value, prefixException=ValueError)
#xValue = qname(elt, value, castException=ValueError, prefixException=ValueError)
sValue = value
''' not sure here, how are explicitDimensions validated, but bad units not?
if xValue.namespaceURI in modelXbrl.namespaceDocs:
if (xValue not in modelXbrl.qnameConcepts and
xValue not in modelXbrl.qnameTypes and
xValue not in modelXbrl.qnameAttributes and
xValue not in modelXbrl.qnameAttributeGroups):
raise ValueError("qname not defined " + str(xValue))
'''
elif baseXsdType in ("XBRLI_DECIMALSUNION", "XBRLI_PRECISIONUNION"):
xValue = sValue = value if value == "INF" else _INT(value)
elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
xValue = sValue = _INT(value)
if xValue == 0:
raise ValueError("invalid value")
elif baseXsdType == "XBRLI_DATEUNION":
xValue = dateTime(value, type=DATEUNION, castException=ValueError)
sValue = value
elif baseXsdType == "dateTime":
xValue = dateTime(value, type=DATETIME, castException=ValueError)
sValue = value
elif baseXsdType == "date":
xValue = dateTime(value, type=DATE, castException=ValueError)
sValue = value
elif baseXsdType == "regex-pattern":
# for facet compiling
try:
sValue = value
if value in xmlSchemaPatterns:
xValue = xmlSchemaPatterns[value]
else:
if r"\i" in value or r"\c" in value:
value = value.replace(r"\i", iNameChar).replace(r"\c", cNameChar)
xValue = re_compile(value + "$") # must match whole string
except Exception as err:
raise ValueError(err)
elif baseXsdType == "fraction":
sValue = value
xValue = Fraction("/".join(elt.fractionValue))
else:
if baseXsdType in lexicalPatterns:
match = lexicalPatterns[baseXsdType].match(value)
if match is None:
raise ValueError("lexical pattern mismatch")
if baseXsdType == "gMonthDay":
month, day, zSign, zHrMin, zHr, zMin = match.groups()
if int(day) > {2:29, 4:30, 6:30, 9:30, 11:30, 1:31, 3:31, 5:31, 7:31, 8:31, 10:31, 12:31}[int(month)]:
raise ValueError("invalid day {0} for month {1}".format(day, month))
xValue = gMonthDay(month, day)
elif baseXsdType == "gYearMonth":
year, month, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gYearMonth(year, month)
elif baseXsdType == "gYear":
year, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gYear(year)
elif baseXsdType == "gMonth":
month, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gMonth(month)
elif baseXsdType == "gDay":
day, zSign, zHrMin, zHr, zMin = match.groups()
xValue = gDay(day)
else:
xValue = value
else: # no lexical pattern, forget compiling value
xValue = value
sValue = value
except (ValueError, InvalidOperation) as err:
if ModelInlineValueObject is not None and isinstance(elt, ModelInlineValueObject):
errElt = "{0} fact {1}".format(elt.elementQname, elt.qname)
else:
errElt = elt.elementQname
if attrTag:
modelXbrl.error("xmlSchema:valueError",
_("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"),
modelObject=elt,
element=errElt,
attribute=XmlUtil.clarkNotationToPrefixedName(elt,attrTag,isAttribute=True),
typeName=baseXsdType,
value=value if len(value) < 31 else value[:30] + '...',
error=err)
else:
modelXbrl.error("xmlSchema:valueError",
_("Element %(element)s type %(typeName)s value error: %(value)s, %(error)s"),
modelObject=elt,
element=errElt,
typeName=baseXsdType,
value=value if len(value) < 31 else value[:30] + '...',
error=err)
xValue = None
sValue = value
xValid = INVALID
else:
xValue = sValue = None
xValid = UNKNOWN
if attrTag:
try: # dynamically allocate attributes (otherwise given shared empty set)
xAttributes = elt.xAttributes
except AttributeError:
elt.xAttributes = xAttributes = {}
xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue, sValue, value)
else:
elt.xValid = xValid
elt.xValue = xValue
elt.sValue = sValue
def loadStandardTaxonomiesDict(self):
if self.selection:
self.standardTaxonomiesDict = defaultdict(set)
self.familyHrefs = defaultdict(set)
self.standardLocalHrefs = defaultdict(set)
self.standardAuthorities = set()
self.standardPrefixes = {}
if not self.standardTaxonomiesUrl:
return
basename = os.path.basename(self.standardTaxonomiesUrl)
self.modelManager.cntlr.showStatus(
_("parsing {0}").format(basename))
file = None
try:
from arelle.FileSource import openXmlFileStream
for filepath in (self.standardTaxonomiesUrl,
os.path.join(
self.modelManager.cntlr.configDir,
"xbrlschemafiles.xml")):
file = openXmlFileStream(self.modelManager.cntlr,
filepath,
stripDeclaration=True)[0]
xmldoc = etree.parse(file)
file.close()
for erxlElt in xmldoc.iter(tag="Erxl"):
v = erxlElt.get("version")
if v and re.match(r"[0-9]+([.][0-9]+)*$", v):
vSplit = v.split('.') # at least 3 digits always!
self.version = tuple(
int(n) for n in vSplit) + tuple(
0 for n in range(3 - len(vSplit)))
break
for locElt in xmldoc.iter(tag="Loc"):
href = None
localHref = None
namespaceUri = None
prefix = None
attType = None
family = None
elements = None
version = None
for childElt in locElt.iterchildren():
ln = childElt.tag
value = childElt.text.strip()
if ln == "Href":
href = value
elif ln == "LocalHref":
localHref = value
elif ln == "Namespace":
namespaceUri = value
elif ln == "Prefix":
prefix = value
elif ln == "AttType":
attType = value
elif ln == "Family":
family = value
elif ln == "Elements":
elements = value
elif ln == "Version":
version = value
if href:
if namespaceUri and (attType == "SCH"
or attType == "ENT"):
self.standardTaxonomiesDict[namespaceUri].add(
href)
if localHref:
self.standardLocalHrefs[namespaceUri].add(
localHref)
authority = UrlUtil.authority(namespaceUri)
self.standardAuthorities.add(authority)
if family == "BASE":
self.baseTaxonomyNamespaces.add(
namespaceUri)
if prefix:
self.standardPrefixes[
namespaceUri] = prefix
if href not in self.standardTaxonomiesDict:
self.standardTaxonomiesDict[
href] = "Allowed" + attType
if family:
self.familyHrefs[family].add(
ErxlLoc(family, version, href, attType,
elements, namespaceUri))
elif attType == "SCH" and family == "BASE":
self.baseTaxonomyNamespaces.add(namespaceUri)
except (EnvironmentError, etree.LxmlError) as err:
self.modelManager.cntlr.addToLog(
"{0}: import error: {1}".format(basename, err))
etree.clear_error_log()
if file:
file.close()
def validateValue(modelXbrl,
elt,
attrTag,
baseXsdType,
value,
isNillable=False,
facets=None):
if baseXsdType:
try:
if (len(value) == 0 and not attrTag is None and not isNillable
and baseXsdType
not in ("anyType", "string", "normalizedString", "token",
"NMTOKEN", "anyURI", "noContent")):
raise ValueError("missing value for not nillable element")
xValid = VALID
whitespaceReplace = (baseXsdType == "normalizedString")
whitespaceCollapse = (not whitespaceReplace
and baseXsdType != "string")
pattern = baseXsdTypePatterns.get(baseXsdType)
if facets:
if "pattern" in facets:
pattern = facets["pattern"]
# note multiple patterns are or'ed togetner, which isn't yet implemented!
if "whiteSpace" in facets:
whitespaceReplace, whitespaceCollapse = {
"preserve": (False, False),
"replace": (True, False),
"collapse": (False, True)
}
if whitespaceReplace:
value = normalizeWhitespacePattern.sub(' ', value)
elif whitespaceCollapse:
value = collapseWhitespacePattern.sub(' ', value.strip())
if pattern is not None and pattern.match(value) is None:
raise ValueError("pattern facet " +
facets["pattern"].pattern if facets and
"pattern" in facets else "pattern mismatch")
if facets:
if "enumeration" in facets and value not in facets[
"enumeration"]:
raise ValueError("is not in {1}".format(
value, facets["enumeration"]))
if "length" in facets and len(value) != facets["length"]:
raise ValueError("length {0}, expected {1}".format(
len(value), facets["length"]))
if "minLength" in facets and len(value) < facets["minLength"]:
raise ValueError("length {0}, minLength {1}".format(
len(value), facets["minLength"]))
if "maxLength" in facets and len(value) > facets["maxLength"]:
raise ValueError("length {0}, maxLength {1}".format(
len(value), facets["maxLength"]))
if baseXsdType == "noContent":
if len(value) > 0 and not value.isspace():
raise ValueError("value content not permitted")
xValue = sValue = None
elif baseXsdType in {
"string", "normalizedString", "language", "token",
"NMTOKEN", "Name", "NCName", "IDREF", "ENTITY"
}:
xValue = sValue = value
elif baseXsdType == "ID":
xValue = sValue = value
xValid = VALID_ID
elif baseXsdType == "anyURI":
xValue = anyURI(value)
sValue = value
if xValue and not UrlUtil.isValid(
xValue): # allow empty strings to be valid anyURIs
raise ValueError("invalid anyURI value")
elif not value: # rest of types get None if nil/empty value
xValue = sValue = None
elif baseXsdType in ("decimal", "float", "double"):
xValue = sValue = float(value)
if facets:
if "totalDigits" in facets and len(value.replace(
".", "")) > facets["totalDigits"]:
raise ValueError("totalDigits facet {0}".format(
facets["totalDigits"]))
if "fractionDigits" in facets and (
'.' in value and len(value[value.index('.') + 1:])
> facets["fractionDigits"]):
raise ValueError("fraction digits facet {0}".format(
facets["fractionDigits"]))
elif baseXsdType in ("integer", ):
xValue = sValue = int(value)
elif baseXsdType == "boolean":
if value in ("true", "1"):
xValue = sValue = True
elif value in ("false", "0"):
xValue = sValue = False
else:
raise ValueError
elif baseXsdType == "QName":
xValue = qname(elt, value, castException=ValueError)
sValue = value
''' not sure here, how are explicitDimensions validated, but bad units not?
if xValue.namespaceURI in modelXbrl.namespaceDocs:
if (xValue not in modelXbrl.qnameConcepts and
xValue not in modelXbrl.qnameTypes and
xValue not in modelXbrl.qnameAttributes and
xValue not in modelXbrl.qnameAttributeGroups):
raise ValueError("qname not defined " + str(xValue))
'''
elif baseXsdType in ("XBRLI_DECIMALSUNION",
"XBRLI_PRECISIONUNION"):
xValue = sValue = value if value == "INF" else int(value)
elif baseXsdType in ("XBRLI_NONZERODECIMAL"):
xValue = sValue = int(value)
if xValue == 0:
raise ValueError("invalid value")
elif baseXsdType == "XBRLI_DATEUNION":
xValue = dateTime(value,
type=DATEUNION,
castException=ValueError)
sValue = value
elif baseXsdType == "dateTime":
xValue = dateTime(value,
type=DATETIME,
castException=ValueError)
sValue = value
elif baseXsdType == "date":
xValue = dateTime(value, type=DATE, castException=ValueError)
sValue = value
elif baseXsdType == "regex-pattern":
# for facet compiling
try:
xValue = re.compile(value + "$") # must match whole string
sValue = value
except Exception as err:
raise ValueError(err)
else:
if baseXsdType in lexicalPatterns:
match = lexicalPatterns[baseXsdType].match(value)
if match is None:
raise ValueError("lexical pattern mismatch")
if baseXsdType == "gMonthDay":
month, day, zSign, zHrMin, zHr, zMin = match.groups()
if int(day) > {
2: 29,
4: 30,
6: 30,
9: 30,
11: 30,
1: 31,
3: 31,
5: 31,
7: 31,
8: 31,
10: 31,
12: 31
}[int(month)]:
raise ValueError(
"invalid day {0} for month {1}".format(
day, month))
xValue = value
sValue = value
except ValueError as err:
if ModelInlineFact is not None and isinstance(
elt, ModelInlineFact):
errElt = "{0} fact {1}".format(elt.elementQname, elt.qname)
else:
errElt = elt.elementQname
if attrTag:
modelXbrl.error(
"xmlSchema:valueError",
_("Element %(element)s attribute %(attribute)s type %(typeName)s value error: %(value)s, %(error)s"
),
modelObject=elt,
element=errElt,
attribute=XmlUtil.clarkNotationToPrefixedName(
elt, attrTag, isAttribute=True),
typeName=baseXsdType,
value=value,
error=err)
else:
modelXbrl.error(
"xmlSchema:valueError",
_("Element %(element)s type %(typeName)s value error: %(value)s, %(error)s"
),
modelObject=elt,
element=errElt,
typeName=baseXsdType,
value=value,
error=err)
xValue = None
sValue = value
xValid = INVALID
else:
xValue = sValue = None
xValid = UNKNOWN
if attrTag:
elt.xAttributes[attrTag] = ModelAttribute(elt, attrTag, xValid, xValue,
sValue, value)
else:
elt.xValid = xValid
elt.xValue = xValue
elt.sValue = sValue
def uriAuthorityValid(self, uri):
return UrlUtil.authority(uri) in self.standardAuthorities
