def expected(self): if self.localName == "testcase": return self.document.basename[:4] #starts with PASS or FAIL errorElement = XmlUtil.descendant(self, None, "error") if errorElement is not None: return ModelValue.qname(errorElement, XmlUtil.text(errorElement)) resultElement = XmlUtil.descendant(self, None, "result") if resultElement is not None: expected = resultElement.get("expected") if expected: return expected for assertElement in XmlUtil.children(resultElement, None, "assert"): num = assertElement.get("num") if len(num) == 5: return "EFM.{0}.{1}.{2}".format(num[0],num[1:3],num[3:6]) asserTests = {} for atElt in XmlUtil.children(resultElement, None, "assertionTests"): try: asserTests[atElt.get("assertionID")] = (_INT(atElt.get("countSatisfied")),_INT(atElt.get("countNotSatisfied"))) except ValueError: pass if asserTests: return asserTests elif self.get("result"): return self.get("result") return None
def reference(self): efmNameElts = XmlUtil.children(self.getparent(), None, "name") for efmNameElt in efmNameElts: if efmNameElt is not None and efmNameElt.text.startswith("EDGAR"): return efmNameElt.text referenceElement = XmlUtil.descendant(self, None, "reference") if referenceElement is not None: # formula test suite return "{0}#{1}".format( referenceElement.get("specification"), referenceElement.get("id")) referenceElement = XmlUtil.descendant(self, None, "documentationReference") if referenceElement is not None: # w3c test suite return referenceElement.get("{http://www.w3.org/1999/xlink}href") descriptionElement = XmlUtil.descendant(self, None, "description") if descriptionElement is not None and descriptionElement.get( "reference"): return descriptionElement.get("reference") # xdt test suite if self.getparent().get("description"): return self.getparent().get( "description") # base spec 2.1 test suite functRegistryRefElt = XmlUtil.descendant(self.getparent(), None, "reference") if functRegistryRefElt is not None: # function registry return functRegistryRefElt.get( "{http://www.w3.org/1999/xlink}href") return None
def cfcnTest(self): # tuple of (expression, element holding the expression) try: return self._cfcnTest except AttributeError: self._cfcnTest = None if self.localName == "test-case": # xpath testcase outputFileElement = XmlUtil.descendant(self, None, "output-file") if outputFileElement is not None and outputFileElement.get("compare") == "Text": filepath = ( self.modelDocument.filepathdir + "/" + "ExpectedTestResults/" + self.get("FilePath") + outputFileElement.text ) if os.sep != "/": filepath = filepath.replace("/", os.sep) with io.open(filepath, "rt", encoding="utf-8") as f: self._cfcnTest = ("xs:string($result) eq '{0}'".format(f.read()), self) else: testElement = XmlUtil.descendant(self, XbrlConst.cfcn, "test") if testElement is not None: self._cfcnTest = (XmlUtil.innerText(testElement), testElement) elif self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms": output = self.get("output") if output: self._cfcnTest = ("$result eq '{0}'".format(output.replace("'", "''")), self) return self._cfcnTest
def readMeFirstUris(self): try: return self._readMeFirstUris except AttributeError: self._readMeFirstUris = [] # first look if any plugin method to get readme first URIs if not any( pluginXbrlMethod(self) for pluginXbrlMethod in pluginClassMethods( "ModelTestcaseVariation.ReadMeFirstUris")): if self.localName == "testGroup": #w3c testcase instanceTestElement = XmlUtil.descendant( self, None, "instanceTest") if instanceTestElement is not None: # take instance first self._readMeFirstUris.append( XmlUtil.descendantAttr( instanceTestElement, None, "instanceDocument", "{http://www.w3.org/1999/xlink}href")) else: schemaTestElement = XmlUtil.descendant( self, None, "schemaTest") if schemaTestElement is not None: self._readMeFirstUris.append( XmlUtil.descendantAttr( schemaTestElement, None, "schemaDocument", "{http://www.w3.org/1999/xlink}href")) elif self.localName == "test-case": #xpath testcase inputFileElement = XmlUtil.descendant( self, None, "input-file") if inputFileElement is not None: # take instance first self._readMeFirstUris.append( "TestSources/" + inputFileElement.text + ".xml") else: # default built-in method for readme first uris for anElement in self.iterdescendants(): if isinstance(anElement, ModelObject) and anElement.get( "readMeFirst") == "true": if anElement.get( "{http://www.w3.org/1999/xlink}href"): uri = anElement.get( "{http://www.w3.org/1999/xlink}href") else: uri = XmlUtil.innerText(anElement) if anElement.get("name"): self._readMeFirstUris.append((ModelValue.qname( anElement, anElement.get("name")), uri)) elif anElement.get("dts"): self._readMeFirstUris.append( (anElement.get("dts"), uri)) else: self._readMeFirstUris.append(uri) if not self._readMeFirstUris: # provide a dummy empty instance document self._readMeFirstUris.append( os.path.join(self.modelXbrl.modelManager.cntlr.configDir, "empty-instance.xml")) return self._readMeFirstUris
def measures(self): try: return self._measures except AttributeError: if self.isDivide: self._measures = (measuresOf(XmlUtil.descendant(self, XbrlConst.xbrli, "unitNumerator")), measuresOf(XmlUtil.descendant(self, XbrlConst.xbrli, "unitDenominator"))) else: self._measures = (measuresOf(self),[]) return self._measures
def resultXbrlInstanceUri(self): for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ResultXbrlInstanceUri"): resultInstanceUri = pluginXbrlMethod(self) if resultInstanceUri is not None: return resultInstanceUri or None # (empty string returns None) resultInstance = XmlUtil.descendant(XmlUtil.descendant(self, None, "result"), None, "instance") if resultInstance is not None: return XmlUtil.text(resultInstance) return None
def severityLevel(self): for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ExpectedSeverity"): severityLevelName = pluginXbrlMethod(self) if severityLevelName: # ignore plug in if not a plug-in-recognized test case return logging._checkLevel(severityLevelName) # default behavior without plugins # SEC error cases have <assert severity={err|wrn}>... if (XmlUtil.descendant(self, None, "assert", attrName="severity", attrValue="wrn") is not None or XmlUtil.descendant(self, None, "result", attrName="severity", attrValue="warning") is not None): return logging._checkLevel("WARNING") return logging._checkLevel("INCONSISTENCY")
def measures(self): """([QName],[Qname]) -- Returns a tuple of multiply measures list and divide members list (empty if not a divide element). Each list of QNames is in prefixed-name order.""" try: return self._measures except AttributeError: if self.isDivide: self._measures = (measuresOf(XmlUtil.descendant(self, XbrlConst.xbrli, "unitNumerator")), measuresOf(XmlUtil.descendant(self, XbrlConst.xbrli, "unitDenominator"))) else: self._measures = (measuresOf(self),[]) return self._measures
def expected(self): for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ExpectedResult"): expected = pluginXbrlMethod(self) if expected: return expected # default behavior without plugins if self.localName == "testcase": return self.document.basename[:4] #starts with PASS or FAIL elif self.localName == "testGroup": #w3c testcase instanceTestElement = XmlUtil.descendant(self, None, "instanceTest") if instanceTestElement is not None: # take instance first return XmlUtil.descendantAttr(instanceTestElement, None, "expected", "validity") else: schemaTestElement = XmlUtil.descendant(self, None, "schemaTest") if schemaTestElement is not None: return XmlUtil.descendantAttr(schemaTestElement, None, "expected", "validity") resultElement = XmlUtil.descendant(self, None, "result") if resultElement is not None: expected = resultElement.get("expected") if expected and resultElement.get("nonStandardErrorCodes") == "true": # if @expected and @nonStandardErrorCodes then use expected instead of error codes return expected errorElement = XmlUtil.descendant(self, None, "error") resultElement = XmlUtil.descendant(self, None, "result") if errorElement is not None and not errorElement.get("nonStandardErrorCodes"): _errorText = XmlUtil.text(errorElement) if ' ' in _errorText: # list of tokens return _errorText return ModelValue.qname(errorElement, _errorText) # turn into a QName if resultElement is not None: if expected: return expected for assertElement in XmlUtil.children(resultElement, None, "assert"): num = assertElement.get("num") if num == "99999": # inline test, use name as expected return assertElement.get("name") if len(num) == 5: return "EFM.{0}.{1}.{2}".format(num[0],num[1:3],num[3:6]) asserTests = {} for atElt in XmlUtil.children(resultElement, None, "assertionTests"): try: asserTests[atElt.get("assertionID")] = (_INT(atElt.get("countSatisfied")),_INT(atElt.get("countNotSatisfied"))) except ValueError: pass if asserTests: return asserTests elif self.get("result"): return self.get("result") return None
def dir(self): self.open() if not self.isOpen: return None elif self.filesDir is not None: return self.filesDir elif self.isZip: files = [] for zipinfo in self.fs.infolist(): files.append(zipinfo.filename) self.filesDir = files elif self.isTarGz: self.filesDir = self.fs.getnames() elif self.isEis: files = [] for docElt in self.eisDocument.iter(tag=u"{http://www.sec.gov/edgar/common}document"): outfn = docElt.findtext(u"{http://www.sec.gov/edgar/common}conformedName") if outfn: files.append(outfn); self.filesDir = files elif self.isXfd: files = [] for data in self.xfdDocument.iter(tag=u"data"): outfn = data.findtext(u"filename") if outfn: if len(outfn) > 2 and outfn[0].isalpha() and \ outfn[1] == u':' and outfn[2] == u'\\': continue files.append(outfn); self.filesDir = files elif self.isRss: files = [] # return title, descr, pubdate, linst doc edgr = u"http://www.sec.gov/Archives/edgar" try: for dsElt in XmlUtil.descendants(self.rssDocument, None, u"item"): instDoc = None for instDocElt in XmlUtil.descendants(dsElt, edgr, u"xbrlFile"): if instDocElt.get(u"(http://www.sec.gov/Archives/edgar}description").endswith(u"INSTANCE DOCUMENT"): instDoc = instDocElt.get(u"(http://www.sec.gov/Archives/edgar}url") break if not instDoc: continue files.append(( XmlUtil.text(XmlUtil.descendant(dsElt, None, u"title")), # tooltip u"{0}\n {1}\n {2}\n {3}\n {4}".format( XmlUtil.text(XmlUtil.descendant(dsElt, edgr, u"companyName")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, u"formType")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, u"filingDate")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, u"cikNumber")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, u"period"))), XmlUtil.text(XmlUtil.descendant(dsElt, None, u"description")), XmlUtil.text(XmlUtil.descendant(dsElt, None, u"pubDate")), instDoc)) self.filesDir = files except (EnvironmentError, etree.LxmlError), err: pass
def pubDate(self): try: return self._pubDate except AttributeError: from arelle.UrlUtil import parseRfcDatetime self._pubDate = parseRfcDatetime(XmlUtil.text(XmlUtil.descendant(self, None, "pubDate"))) return self._pubDate
def resultIsTable(self): result = XmlUtil.descendant(self, None, "result") if result is not None : child = XmlUtil.child(result, None, "table") if child is not None and XmlUtil.text(child).endswith(".xml"): return True return False
def cfcnCall(self): # tuple of (expression, element holding the expression) try: return self._cfcnCall except AttributeError: self._cfcnCall = None if self.localName == "test-case": # xpath testcase queryElement = XmlUtil.descendant(self, None, "query") if queryElement is not None: filepath = ( self.modelDocument.filepathdir + "/" + "Queries/XQuery/" + self.get("FilePath") + queryElement.get("name") + ".xq" ) if os.sep != "/": filepath = filepath.replace("/", os.sep) with io.open(filepath, "rt", encoding="utf-8") as f: self._cfcnCall = (f.read(), self) else: for callElement in XmlUtil.descendants(self, XbrlConst.cfcn, "call"): self._cfcnCall = (XmlUtil.innerText(callElement), callElement) break if self._cfcnCall is None and self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms": name = self.getparent().get("name") input = self.get("input") if name and input: self._cfcnCall = ("{0}('{1}')".format(name, input.replace("'", "''")), self) return self._cfcnCall
def resultTableUri(self): result = XmlUtil.descendant(self, None, "result") if result is not None: child = XmlUtil.child(result, None, "table") if child is not None: return os.path.join(self.modelDocument.outpath, XmlUtil.text(child)) return None
def resultInfosetUri(self): result = XmlUtil.descendant(self, None, "result") if result is not None: child = XmlUtil.child(result, None, "file") return os.path.join( self.modelDocument.outpath, XmlUtil.text(child if child is not None else result)) return None
def resultIsInfoset(self): if self.modelDocument.outpath: result = XmlUtil.descendant(self, None, "result") if result is not None: return XmlUtil.child(result, None, "file") is not None or XmlUtil.text( result).endswith(".xml") return False
def acceptanceDatetime(self): try: return self._acceptanceDatetime except AttributeError: import datetime self._acceptanceDatetime = None date = XmlUtil.text(XmlUtil.descendant(self, edgr, u"acceptanceDatetime")) if date and len(date) == 14: self._acceptanceDatetime = datetime.datetime(_INT(date[0:4]),_INT(date[4:6]),_INT(date[6:8]),_INT(date[8:10]),_INT(date[10:12]),_INT(date[12:14])) return self._acceptanceDatetime
def unit_denominator(xc, p, args): if len(args) != 1: raise XPathContext.FunctionNumArgs() if len(args[0]) != 1: raise XPathContext.FunctionArgType(1,"xbrl:unit") unit = args[0][0] if isinstance(unit,ModelObject) and \ unit.localName == "unit" and unit.namespaceURI == XbrlConst.xbrli: measuresParent = XmlUtil.descendant(unit, XbrlConst.xbrli, "unitDenominator") if measuresParent is None: return [] return XmlUtil.descendants(measuresParent, XbrlConst.xbrli, "measure") raise XPathContext.FunctionArgType(1,"xbrl:unit")
def unit_numerator(xc, p, args): if len(args) != 1: raise XPathContext.FunctionNumArgs() if len(args[0]) != 1: raise XPathContext.FunctionArgType(1,"xbrl:unit") unit = args[0][0] if isinstance(unit,ModelObject.ModelObject): unit = unit.element if isinstance(unit,xml.dom.Node) and unit.nodeType == 1 and \ unit.localName == "unit" and unit.namespaceURI == XbrlConst.xbrli: measuresParent = XmlUtil.descendant(unit, XbrlConst.xbrli, "unitNumerator") if measuresParent is None: measuresParent = unit return XmlUtil.descendants(measuresParent, XbrlConst.xbrli, "measure") raise XPathContext.FunctionArgType(1,"xbrl:unit")
def filingDate(self): try: return self._filingDate except AttributeError: import datetime self._filingDate = None date = XmlUtil.text(XmlUtil.descendant(self, edgr, "filingDate")) d = date.split("/") if d and len(d) == 3: self._filingDate = datetime.date(_INT(d[2]),_INT(d[0]),_INT(d[1])) return self._filingDate
def dir(self): self.open() if not self.isOpen: return None elif self.filesDir is not None: return self.filesDir elif self.isZip: files = [] for zipinfo in self.fs.infolist(): files.append(zipinfo.filename) self.filesDir = files elif self.isXfd: files = [] for data in self.xfdDocument.getElementsByTagName("data"): outfn = XmlUtil.text(data.getElementsByTagName("filename")[0]) if len(outfn) > 1: if len(outfn) > 2 and outfn[0].isalpha() and \ outfn[1] == ':' and outfn[2] == '\\': continue files.append(outfn); self.filesDir = files elif self.isRss: files = [] # return title, descr, pubdate, linst doc edgr = "http://www.sec.gov/Archives/edgar" try: for dsElt in self.rssDocument.getElementsByTagName("item"): instDoc = None for instDocElt in XmlUtil.descendants(dsElt, edgr, "xbrlFile"): if instDocElt.getAttributeNS(edgr,"description").endswith("INSTANCE DOCUMENT"): instDoc = instDocElt.getAttributeNS(edgr,"url") break if not instDoc: continue files.append(( XmlUtil.text(XmlUtil.descendant(dsElt, None, "title")), # tooltip "{0}\n {1}\n {2}\n {3}\n {4}".format( XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "companyName")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "formType")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "filingDate")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "cikNumber")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "period"))), XmlUtil.text(XmlUtil.descendant(dsElt, None, "description")), XmlUtil.text(XmlUtil.descendant(dsElt, None, "pubDate")), instDoc)) self.filesDir = files except (EnvironmentError, xml.parsers.expat.ExpatError, xml.dom.DOMException) as err: pass return self.filesDir
def expected(self): for pluginXbrlMethod in pluginClassMethods(u"ModelTestcaseVariation.ExpectedResult"): expected = pluginXbrlMethod(self) if expected: return expected # default behavior without plugins if self.localName == u"testcase": return self.document.basename[:4] #starts with PASS or FAIL elif self.localName == u"testGroup": #w3c testcase instanceTestElement = XmlUtil.descendant(self, None, u"instanceTest") if instanceTestElement is not None: # take instance first return XmlUtil.descendantAttr(instanceTestElement, None, u"expected", u"validity") else: schemaTestElement = XmlUtil.descendant(self, None, u"schemaTest") if schemaTestElement is not None: return XmlUtil.descendantAttr(schemaTestElement, None, u"expected", u"validity") errorElement = XmlUtil.descendant(self, None, u"error") if errorElement is not None: return ModelValue.qname(errorElement, XmlUtil.text(errorElement)) resultElement = XmlUtil.descendant(self, None, u"result") if resultElement is not None: expected = resultElement.get(u"expected") if expected: return expected for assertElement in XmlUtil.children(resultElement, None, u"assert"): num = assertElement.get(u"num") if len(num) == 5: return u"EFM.{0}.{1}.{2}".format(num[0],num[1:3],num[3:6]) asserTests = {} for atElt in XmlUtil.children(resultElement, None, u"assertionTests"): try: asserTests[atElt.get(u"assertionID")] = (_INT(atElt.get(u"countSatisfied")),_INT(atElt.get(u"countNotSatisfied"))) except ValueError: pass if asserTests: return asserTests elif self.get(u"result"): return self.get(u"result") return None
def name(self): try: return self._name except AttributeError: if self.get("name"): self._name = self.get("name") else: nameElement = XmlUtil.descendant(self, None, "name" if self.localName != "testcase" else "number") if nameElement is not None: self._name = XmlUtil.innerText(nameElement) else: self._name = None return self._name
def cfcnTest(self): # tuple of (expression, element holding the expression) try: return self._cfcnTest except AttributeError: self._cfcnTest = None testElement = XmlUtil.descendant(self, XbrlConst.cfcn, "test") if testElement is not None: self._cfcnTest = (XmlUtil.innerText(testElement), testElement) elif self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms": output = self.get("output") if output: self._cfcnTest = ("$result eq '{0}'".format(output.replace("'","''")), self) return self._cfcnTest
def parent_child(args, parentName, childName, findDescendant=False): if len(args) != 1: raise XPathContext.FunctionNumArgs() if len(args[0]) != 1: raise XPathContext.FunctionArgType(1,"xbrl:" + parentName) parent = args[0][0] if isinstance(parent,ModelObject) and \ parent.localName == parentName and parent.namespaceURI == XbrlConst.xbrli: if childName.startswith('@'): return parent.get(childName[1:]) elif childName == 'text()': return XmlUtil.textNotStripped(parent) elif childName == 'strip-text()': return XmlUtil.text(parent) elif findDescendant: return XmlUtil.descendant(parent, XbrlConst.xbrli, childName) else: return XmlUtil.child(parent, XbrlConst.xbrli, childName) raise XPathContext.FunctionArgType(1,"xbrl:" + parentName)
def cfcnCall(self): # tuple of (expression, element holding the expression) try: return self._cfcnCall except AttributeError: self._cfcnCall = None if self.localName == "test-case": #xpath testcase queryElement = XmlUtil.descendant(self, None, "query") if queryElement is not None: filepath = (self.modelDocument.filepathdir + "/" + "Queries/XQuery/" + self.get("FilePath") + queryElement.get("name") + '.xq') if os.sep != "/": filepath = filepath.replace("/", os.sep) with io.open(filepath, 'rt', encoding='utf-8') as f: self._cfcnCall = (f.read(), self) else: for callElement in XmlUtil.descendants(self, XbrlConst.cfcn, "call"): self._cfcnCall = (XmlUtil.innerText(callElement), callElement) break if self._cfcnCall is None and self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms": name = self.getparent().get("name") input = self.get("input") if name and input: self._cfcnCall = ("{0}('{1}')".format(name, input.replace("'","''")), self) return self._cfcnCall
def period(self): per = XmlUtil.text(XmlUtil.descendant(self, edgr, "period")) if per and len(per) == 8: return "{0}-{1}-{2}".format(per[0:4], per[4:6], per[6:8]) return None
def dir(self): self.open() if not self.isOpen: return None elif self.filesDir is not None: return self.filesDir elif self.isZip: files = [] for zipinfo in self.fs.infolist(): files.append(zipinfo.filename) self.filesDir = files elif self.isXfd: files = [] for data in self.xfdDocument.getElementsByTagName("data"): outfn = XmlUtil.text(data.getElementsByTagName("filename")[0]) if len(outfn) > 1: if len(outfn) > 2 and outfn[0].isalpha() and \ outfn[1] == ':' and outfn[2] == '\\': continue files.append(outfn) self.filesDir = files elif self.isRss: files = [] # return title, descr, pubdate, linst doc edgr = "http://www.sec.gov/Archives/edgar" try: for dsElt in self.rssDocument.getElementsByTagName("item"): instDoc = None for instDocElt in XmlUtil.descendants( dsElt, edgr, "xbrlFile"): if instDocElt.getAttributeNS( edgr, "description").endswith("INSTANCE DOCUMENT"): instDoc = instDocElt.getAttributeNS(edgr, "url") break if not instDoc: continue files.append(( XmlUtil.text(XmlUtil.descendant(dsElt, None, "title")), # tooltip "{0}\n {1}\n {2}\n {3}\n {4}".format( XmlUtil.text( XmlUtil.descendant(dsElt, edgr, "companyName")), XmlUtil.text( XmlUtil.descendant(dsElt, edgr, "formType")), XmlUtil.text( XmlUtil.descendant(dsElt, edgr, "filingDate")), XmlUtil.text( XmlUtil.descendant(dsElt, edgr, "cikNumber")), XmlUtil.text( XmlUtil.descendant(dsElt, edgr, "period"))), XmlUtil.text( XmlUtil.descendant(dsElt, None, "description")), XmlUtil.text(XmlUtil.descendant( dsElt, None, "pubDate")), instDoc)) self.filesDir = files except (EnvironmentError, xml.parsers.expat.ExpatError, xml.dom.DOMException) as err: pass return self.filesDir
def fiscalYearEnd(self): yrEnd = XmlUtil.text(XmlUtil.descendant(self, edgr, "fiscalYearEnd")) if yrEnd and len(yrEnd) == 4: return "{0}-{1}".format(yrEnd[0:2],yrEnd[2:4]) return None
def resultInfosetUri(self): result = XmlUtil.descendant(self, None, "result") if result is not None: child = XmlUtil.child(result, None, "file") return os.path.join(self.modelDocument.outpath, XmlUtil.text(child if child is not None else result)) return None
def description(self): nameElement = XmlUtil.descendant(self, None, ("description", "documentation")) if nameElement is not None: return XmlUtil.innerText(nameElement) return None
def resultIsVersioningReport(self): return XmlUtil.descendant(XmlUtil.descendant(self, None, "result"), None, "versioningReport") is not None
def resultIsXbrlInstance(self): return XmlUtil.descendant(XmlUtil.descendant(self, None, "result"), None, "instance") is not None
def checkFilingDTS(val, modelDocument, visited): global targetNamespaceDatePattern, efmFilenamePattern, roleTypePattern, arcroleTypePattern, \ arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet, \ namespacesConflictPattern if targetNamespaceDatePattern is None: targetNamespaceDatePattern = re.compile( r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|" r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|") efmFilenamePattern = re.compile( r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml)$") roleTypePattern = re.compile(r"^.*/role/[^/\s]+$") arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$") arcroleDefinitionPattern = re.compile( r"^.*[^\\s]+.*$") # at least one non-whitespace character namePattern = re.compile( "[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]" ) # u20ac=Euro, u00a3=pound sterling linkroleDefinitionBalanceIncomeSheet = re.compile( r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)", re.IGNORECASE) namespacesConflictPattern = re.compile( r"http://(xbrl\.us|fasb\.org|xbrl\.sec\.gov)/(dei|us-types|us-roles|rr)/([0-9]{4}-[0-9]{2}-[0-9]{2})$" ) visited.append(modelDocument) for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items( ): #6.07.01 no includes if "include" in modelDocumentReference.referenceTypes: val.modelXbrl.error( "SBR.NL.2.2.0.18", _("Taxonomy schema %(schema)s includes %(include)s, only import is allowed" ), modelObject=modelDocumentReference.referringModelObject, schema=os.path.basename(modelDocument.uri), include=os.path.basename(referencedDocument.uri)) if referencedDocument not in visited: checkFilingDTS(val, referencedDocument, visited) if val.disclosureSystem.standardTaxonomiesDict is None: pass if (modelDocument.type == ModelDocument.Type.SCHEMA and modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and modelDocument.uri.startswith(val.modelXbrl.uriDir)): # check schema contents types definesLinkroles = False definesArcroles = False definesLinkParts = False definesAbstractItems = False definesNonabstractItems = False definesConcepts = False definesTuples = False definesPresentationTuples = False definesSpecificationTuples = False definesTypes = False definesEnumerations = False definesDimensions = False definesDomains = False definesHypercubes = False genrlSpeclRelSet = val.modelXbrl.relationshipSet( XbrlConst.generalSpecial) for modelConcept in modelDocument.xmlRootElement.iterdescendants( tag="{http://www.w3.org/2001/XMLSchema}element"): if isinstance(modelConcept, ModelConcept): # 6.7.16 name not duplicated in standard taxonomies name = modelConcept.get("name") if name is None: name = "" if modelConcept.get("ref") is not None: continue # don't validate ref's here for c in val.modelXbrl.nameConcepts.get(name, []): if c.modelDocument != modelDocument: if not (genrlSpeclRelSet.isRelated( modelConcept, "child", c) or genrlSpeclRelSet.isRelated( c, "child", modelConcept)): val.modelXbrl.error( "SBR.NL.2.2.2.02", _("Concept %(concept)s is also defined in standard taxonomy schema %(standardSchema)s without a general-special relationship" ), modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename( c.modelDocument.uri)) ''' removed RH 2011-12-23 corresponding set up of table in ValidateFiling if val.validateSBRNL and name in val.nameWordsTable: if not any( any( genrlSpeclRelSet.isRelated(c, "child", modelConcept) for c in val.modelXbrl.nameConcepts.get(partialWordName, [])) for partialWordName in val.nameWordsTable[name]): val.modelXbrl.error("SBR.NL.2.3.2.01", _("Concept %(specialName)s is appears to be missing a general-special relationship to %(generalNames)s"), modelObject=c, specialName=modelConcept.qname, generalNames=', or to '.join(val.nameWordsTable[name])) ''' if modelConcept.isTuple: if modelConcept.substitutionGroupQname.localName == "presentationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith( "/basis/sbr/xbrl/xbrl-syntax-extension" ): # namespace may change each year definesPresentationTuples = True elif modelConcept.substitutionGroupQname.localName == "specificationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith( "/basis/sbr/xbrl/xbrl-syntax-extension" ): # namespace may change each year definesSpecificationTuples = True else: definesTuples = True definesConcepts = True if modelConcept.isAbstract: val.modelXbrl.error( "SBR.NL.2.2.2.03", _("Concept %(concept)s is an abstract tuple"), modelObject=modelConcept, concept=modelConcept.qname) if tupleCycle(val, modelConcept): val.modelXbrl.error( "SBR.NL.2.2.2.07", _("Tuple %(concept)s has a tuple cycle"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.get( "nillable") != "false" and modelConcept.isRoot: val.modelXbrl.error( "SBR.NL.2.2.2.17", #don't want default, just what was really there _("Tuple %(concept)s must have nillable='false'"), modelObject=modelConcept, concept=modelConcept.qname) elif modelConcept.isItem: definesConcepts = True if modelConcept.abstract == "true": if modelConcept.isRoot: if modelConcept.get( "nillable" ) != "false": #don't want default, just what was really there val.modelXbrl.error( "SBR.NL.2.2.2.16", _("Abstract root concept %(concept)s must have nillable='false'" ), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.typeQname != XbrlConst.qnXbrliStringItemType: val.modelXbrl.error( "SBR.NL.2.2.2.21", _("Abstract root concept %(concept)s must have type='xbrli:stringItemType'" ), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.balance: val.modelXbrl.error( "SBR.NL.2.2.2.22", _("Abstract concept %(concept)s must not have a balance attribute" ), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.isHypercubeItem: definesHypercubes = True elif modelConcept.isDimensionItem: definesDimensions = True elif modelConcept.substitutionGroupQname and modelConcept.substitutionGroupQname.localName in ( "domainItem", "domainMemberItem"): definesDomains = True elif modelConcept.isItem: definesAbstractItems = True else: # not abstract if modelConcept.isItem: definesNonabstractItems = True if not (modelConcept.label( preferredLabel=XbrlConst.documentationLabel, fallbackToQname=False, lang="nl") or val.modelXbrl.relationshipSet( XbrlConst.conceptReference ).fromModelObject(c) or modelConcept.genLabel( role=XbrlConst.genDocumentationLabel, lang="nl") or val.modelXbrl.relationshipSet( XbrlConst.elementReference). fromModelObject(c)): val.modelXbrl.error( "SBR.NL.2.2.2.28", _("Concept %(concept)s must have a documentation label or reference" ), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.balance and not modelConcept.instanceOfType( XbrlConst.qnXbrliMonetaryItemType): val.modelXbrl.error( "SBR.NL.2.2.2.24", _("Non-monetary concept %(concept)s must not have a balance attribute" ), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.isLinkPart: definesLinkParts = True val.modelXbrl.error( "SBR.NL.2.2.5.01", _("Link:part concept %(concept)s is not allowed"), modelObject=modelConcept, concept=modelConcept.qname) if not modelConcept.genLabel(fallbackToQname=False, lang="nl"): val.modelXbrl.error( "SBR.NL.2.2.5.02", _("Link part definition %(concept)s must have a generic label in language 'nl'" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.9 role types authority for e in modelDocument.xmlRootElement.iterdescendants( tag="{http://www.xbrl.org/2003/linkbase}roleType"): if isinstance(e, ModelObject): roleURI = e.get("roleURI") # 6.7.10 only one role type declaration in DTS modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI) if modelRoleTypes is not None: modelRoleType = modelRoleTypes[0] definition = modelRoleType.definitionNotStripped usedOns = modelRoleType.usedOns if usedOns & XbrlConst.standardExtLinkQnames or XbrlConst.qnGenLink in usedOns: definesLinkroles = True if not e.genLabel(): val.modelXbrl.error( "SBR.NL.2.2.3.03", _("Link RoleType %(roleType)s missing a generic standard label" ), modelObject=e, roleType=roleURI) nlLabel = e.genLabel(lang="nl") if definition != nlLabel: val.modelXbrl.error( "SBR.NL.2.2.3.04", _("Link RoleType %(roleType)s definition does not match NL standard generic label, \ndefinition: %(definition)s \nNL label: %(label)s" ), modelObject=e, roleType=roleURI, definition=definition, label=nlLabel) if definition and (definition[0].isspace() or definition[-1].isspace()): val.modelXbrl.error( "SBR.NL.2.2.3.07", _('Link RoleType %(roleType)s definition has leading or trailing spaces: "%(definition)s"' ), modelObject=e, roleType=roleURI, definition=definition) # 6.7.13 arcrole types authority for e in modelDocument.xmlRootElement.iterdescendants( tag="{http://www.xbrl.org/2003/linkbase}arcroleType"): if isinstance(e, ModelObject): arcroleURI = e.get("arcroleURI") definesArcroles = True val.modelXbrl.error( "SBR.NL.2.2.4.01", _("Arcrole type definition is not allowed: %(arcroleURI)s" ), modelObject=e, arcroleURI=arcroleURI) for appinfoElt in modelDocument.xmlRootElement.iter( tag="{http://www.w3.org/2001/XMLSchema}appinfo"): for nonLinkElt in appinfoElt.iterdescendants(): if isinstance(nonLinkElt, ModelObject ) and nonLinkElt.namespaceURI != XbrlConst.link: val.modelXbrl.error( "SBR.NL.2.2.11.05", _("Appinfo contains disallowed non-link element %(element)s" ), modelObject=nonLinkElt, element=nonLinkElt.qname) for cplxTypeElt in modelDocument.xmlRootElement.iter( tag="{http://www.w3.org/2001/XMLSchema}complexType"): choiceElt = cplxTypeElt.find( "{http://www.w3.org/2001/XMLSchema}choice") if choiceElt is not None: val.modelXbrl.error( "SBR.NL.2.2.11.09", _("ComplexType contains disallowed xs:choice element"), modelObject=choiceElt) for cplxContentElt in modelDocument.xmlRootElement.iter( tag="{http://www.w3.org/2001/XMLSchema}complexContent"): if XmlUtil.descendantAttr( cplxContentElt, "http://www.w3.org/2001/XMLSchema", ("extension", "restriction"), "base") != "sbr:placeholder": val.modelXbrl.error("SBR.NL.2.2.11.10", _("ComplexContent is disallowed"), modelObject=cplxContentElt) for typeEltTag in ("{http://www.w3.org/2001/XMLSchema}complexType", "{http://www.w3.org/2001/XMLSchema}simpleType"): for typeElt in modelDocument.xmlRootElement.iter(tag=typeEltTag): definesTypes = True name = typeElt.get("name") if name: if not name[0].islower() or not name.isalnum(): val.modelXbrl.error( "SBR.NL.3.2.8.09", _("Type name attribute must be lower camelcase: %(name)s." ), modelObject=typeElt, name=name) for enumElt in modelDocument.xmlRootElement.iter( tag="{http://www.w3.org/2001/XMLSchema}enumeration"): definesEnumerations = True if any(not valueElt.genLabel(lang="nl") for valueElt in enumElt.iter( tag="{http://www.w3.org/2001/XMLSchema}value")): val.modelXbrl.error( "SBR.NL.2.2.7.05", _("Enumeration element has value(s) without generic label." ), modelObject=enumElt) if (definesLinkroles + definesArcroles + definesLinkParts + definesAbstractItems + definesNonabstractItems + definesTuples + definesPresentationTuples + definesSpecificationTuples + definesTypes + definesEnumerations + definesDimensions + definesDomains + definesHypercubes) != 1: schemaContents = [] if definesLinkroles: schemaContents.append(_("linkroles")) if definesArcroles: schemaContents.append(_("arcroles")) if definesLinkParts: schemaContents.append(_("link parts")) if definesAbstractItems: schemaContents.append(_("abstract items")) if definesNonabstractItems: schemaContents.append(_("nonabstract items")) if definesTuples: schemaContents.append(_("tuples")) if definesPresentationTuples: schemaContents.append(_("sbrPresentationTuples")) if definesSpecificationTuples: schemaContents.append(_("sbrSpecificationTuples")) if definesTypes: schemaContents.append(_("types")) if definesEnumerations: schemaContents.append(_("enumerations")) if definesDimensions: schemaContents.append(_("dimensions")) if definesDomains: schemaContents.append(_("domains")) if definesHypercubes: schemaContents.append(_("hypercubes")) if schemaContents: if not ((definesTuples or definesPresentationTuples or definesSpecificationTuples) and not (definesLinkroles or definesArcroles or definesLinkParts or definesAbstractItems or definesTypes or definesDimensions or definesDomains or definesHypercubes)): val.modelXbrl.error( "SBR.NL.2.2.1.01", _("Taxonomy schema may only define one of these: %(contents)s" ), modelObject=modelDocument, contents=', '.join(schemaContents)) elif not any(refDoc.inDTS and refDoc.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces for refDoc in modelDocument.referencesDocument.keys( )): # no linkbase ref or includes val.modelXbrl.error( "SBR.NL.2.2.1.01", _("Taxonomy schema must be a DTS entrypoint OR define linkroles OR arcroles OR link:parts OR context fragments OR abstract items OR tuples OR non-abstract elements OR types OR enumerations OR dimensions OR domains OR hypercubes" ), modelObject=modelDocument) if definesConcepts ^ any( # xor so either concepts and no label LB or no concepts and has label LB (refDoc.type == ModelDocument.Type.LINKBASE and XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "labelLink") is not None) for refDoc in modelDocument.referencesDocument.keys()): # no label linkbase val.modelXbrl.error( "SBR.NL.2.2.1.02", _("A schema that defines concepts MUST have a linked 2.1 label linkbase" ), modelObject=modelDocument) if ( definesNonabstractItems or definesTuples ) and not any( # was xor but changed to and not per RH 1/11/12 (refDoc.type == ModelDocument.Type.LINKBASE and (XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "referenceLink") is not None or XmlUtil.descendant( refDoc.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/documentation") is not None)) for refDoc in modelDocument.referencesDocument.keys()): val.modelXbrl.error( "SBR.NL.2.2.1.03", _("A schema that defines non-abstract items MUST have a linked (2.1) reference linkbase AND/OR a label linkbase with @xlink:role=documentation" ), modelObject=modelDocument) elif modelDocument.type == ModelDocument.Type.LINKBASE: pass visited.remove(modelDocument)
def companyName(self): return XmlUtil.text(XmlUtil.descendant(self, edgr, "companyName"))
def readMeFirstUris(self): try: return self._readMeFirstUris except AttributeError: self._readMeFirstUris = [] self.readMeFirstElements = [] # first look if any plugin method to get readme first URIs if not any( pluginXbrlMethod(self) for pluginXbrlMethod in pluginClassMethods( "ModelTestcaseVariation.ReadMeFirstUris")): if self.localName == "testGroup": #w3c testcase instanceTestElement = XmlUtil.descendant( self, None, "instanceTest") if instanceTestElement is not None: # take instance first self._readMeFirstUris.append( XmlUtil.descendantAttr( instanceTestElement, None, "instanceDocument", "{http://www.w3.org/1999/xlink}href")) self.readMeFirstElements.append(instanceTestElement) else: schemaTestElement = XmlUtil.descendant( self, None, "schemaTest") if schemaTestElement is not None: self._readMeFirstUris.append( XmlUtil.descendantAttr( schemaTestElement, None, "schemaDocument", "{http://www.w3.org/1999/xlink}href")) self.readMeFirstElements.append(schemaTestElement) elif self.localName == "test-case": #xpath testcase inputFileElement = XmlUtil.descendant( self, None, "input-file") if inputFileElement is not None: # take instance first self._readMeFirstUris.append("TestSources/" + inputFileElement.text + ".xml") self.readMeFirstElements.append(inputFileElement) elif self.resultIsTaxonomyPackage: self._readMeFirstUris.append( os.path.join(self.modelDocument.filepathdir, "tests", self.get("name") + ".zip")) self.readMeFirstElements.append(self) else: # default built-in method for readme first uris for anElement in self.iterdescendants(): if isinstance(anElement, ModelObject) and anElement.get( "readMeFirst") == "true": if anElement.get( "{http://www.w3.org/1999/xlink}href"): uri = anElement.get( "{http://www.w3.org/1999/xlink}href") else: uri = XmlUtil.innerText(anElement) if anElement.get("name"): self._readMeFirstUris.append( (ModelValue.qname(anElement, anElement.get("name")), uri)) self.readMeFirstElements.append(anElement) elif anElement.get("dts"): self._readMeFirstUris.append( (anElement.get("dts"), uri)) self.readMeFirstElements.append(anElement) else: self._readMeFirstUris.append(uri) self.readMeFirstElements.append(anElement) if not self._readMeFirstUris: # provide a dummy empty instance document self._readMeFirstUris.append( os.path.join(self.modelXbrl.modelManager.cntlr.configDir, "empty-instance.xml")) self.readMeFirstElements.append(None) return self._readMeFirstUris
def expected(self): for pluginXbrlMethod in pluginClassMethods( "ModelTestcaseVariation.ExpectedResult"): expected = pluginXbrlMethod(self) if expected: return expected # default behavior without plugins if self.localName == "testcase": return self.document.basename[:4] #starts with PASS or FAIL elif self.localName == "testGroup": #w3c testcase instanceTestElement = XmlUtil.descendant(self, None, "instanceTest") if instanceTestElement is not None: # take instance first return XmlUtil.descendantAttr(instanceTestElement, None, "expected", "validity") else: schemaTestElement = XmlUtil.descendant(self, None, "schemaTest") if schemaTestElement is not None: return XmlUtil.descendantAttr(schemaTestElement, None, "expected", "validity") resultElement = XmlUtil.descendant(self, None, "result") if resultElement is not None: expected = resultElement.get("expected") if expected and resultElement.get( "nonStandardErrorCodes") == "true": # if @expected and @nonStandardErrorCodes then use expected instead of error codes return expected errorElements = XmlUtil.descendants(self, None, "error") resultElement = XmlUtil.descendant(self, None, "result") if isinstance(errorElements, list) and len(errorElements) > 0: return [ ModelValue.qname(e, e.stringValue) if not e.get("nonStandardErrorCodes") else e.stringValue for e in errorElements ] #else: # errorElement = errorElements[1] # if errorElement is not None and not errorElement.get("nonStandardErrorCodes"): # _errorText = XmlUtil.text(errorElement) # if ' ' in _errorText: # list of tokens # return _errorText # return ModelValue.qname(errorElement, _errorText) # turn into a QName if resultElement is not None: if expected: return expected for assertElement in XmlUtil.children(resultElement, None, "assert"): num = assertElement.get("num") if num == "99999": # inline test, use name as expected return assertElement.get("name") if len(num) == 5: return "EFM.{0}.{1}.{2}".format(num[0], num[1:3], num[3:6]) asserTests = {} for atElt in XmlUtil.children(resultElement, None, "assertionTests"): try: asserTests[atElt.get("assertionID")] = ( _INT(atElt.get("countSatisfied")), _INT(atElt.get("countNotSatisfied"))) except ValueError: pass if asserTests: return asserTests elif self.get("result"): return self.get("result") return None
def assignedSic(self): return XmlUtil.text(XmlUtil.descendant(self, edgr, "assignedSic"))
def fiscalYearEnd(self): yrEnd = XmlUtil.text(XmlUtil.descendant(self, edgr, "fiscalYearEnd")) if yrEnd and len(yrEnd) == 4: return "{0}-{1}".format(yrEnd[0:2], yrEnd[2:4]) return None
def checkFilingDTS(val, modelDocument, isEFM, isGFM, visited): global targetNamespaceDatePattern, efmFilenamePattern, htmlFileNamePattern, roleTypePattern, arcroleTypePattern, \ arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet if targetNamespaceDatePattern is None: targetNamespaceDatePattern = re.compile( r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|" r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|") efmFilenamePattern = re.compile( r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml|\.htm)$") htmlFileNamePattern = re.compile( r"^[a-zA-Z0-9][._a-zA-Z0-9-]*(\.htm)$") roleTypePattern = re.compile(r"^.*/role/[^/\s]+$") arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$") arcroleDefinitionPattern = re.compile( r"^.*[^\\s]+.*$") # at least one non-whitespace character namePattern = re.compile( "[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]" ) # u20ac=Euro, u00a3=pound sterling linkroleDefinitionBalanceIncomeSheet = re.compile( r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)", re.IGNORECASE) nonDomainItemNameProblemPattern = re.compile( r"({0})|(FirstQuarter|SecondQuarter|ThirdQuarter|FourthQuarter|[1-4]Qtr|Qtr[1-4]|ytd|YTD|HalfYear)(?:$|[A-Z\W])" .format(re.sub(r"\W", "", (val.entityRegistrantName or "").title()))) visited.append(modelDocument) for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items( ): #6.07.01 no includes if modelDocumentReference.referenceType == "include": val.modelXbrl.error( ("EFM.6.07.01", "GFM.1.03.01"), _("Taxonomy schema %(schema)s includes %(include)s, only import is allowed" ), modelObject=modelDocumentReference.referringModelObject, schema=modelDocument.basename, include=referencedDocument.basename) if referencedDocument not in visited and ( referencedDocument.inDTS or referencedDocument.type == ModelDocument.Type.INLINEXBRLDOCUMENTSET ): # ignore EdgarRenderer added non-DTS documents checkFilingDTS(val, referencedDocument, isEFM, isGFM, visited) if modelDocument.type == ModelDocument.Type.INLINEXBRLDOCUMENTSET: return # nothing to check in inline document set surrogate parent if val.disclosureSystem.standardTaxonomiesDict is None: pass if isEFM: if modelDocument.uri in val.disclosureSystem.standardTaxonomiesDict: if modelDocument.targetNamespace: # check for duplicates of us-types, dei, and rr taxonomies match = standardNamespacesPattern.match( modelDocument.targetNamespace) if match is not None: conflictClass = match.group(2) or match.group(5) if (conflictClass == 'us-gaap' and match.group(3) < '2018') or conflictClass == 'srt': val.standardNamespaceConflicts['srt+us-gaap'].add( modelDocument ) # ignore non-srt multi-usgaap in Filing.py if conflictClass == 'us-gaap' or conflictClass == 'ifrs-full': val.standardNamespaceConflicts['ifrs+us-gaap'].add( modelDocument) if conflictClass not in ('us-gaap', 'srt'): val.standardNamespaceConflicts[conflictClass].add( modelDocument) else: if len(modelDocument.basename) > 32: val.modelXbrl.error( "EFM.5.01.01.tooManyCharacters", _("Document file name %(filename)s must not exceed 32 characters." ), edgarCode="cp-0501-File-Name-Length", modelObject=modelDocument, filename=modelDocument.basename) if modelDocument.type != ModelDocument.Type.INLINEXBRLDOCUMENTSET: if modelDocument.type == ModelDocument.Type.INLINEXBRL: _pattern = htmlFileNamePattern _suffix = ".htm" else: _pattern = efmFilenamePattern _suffix = ".xsd or .xml" if not _pattern.match(modelDocument.basename): val.modelXbrl.error( "EFM.5.01.01", _("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with %(suffix)s." ), edgarCode="cp-0501-File-Name", modelObject=modelDocument, filename=modelDocument.basename, suffix=_suffix) if (modelDocument.type == ModelDocument.Type.SCHEMA and modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and modelDocument.uri.startswith(val.modelXbrl.uriDir)): val.hasExtensionSchema = True # check schema contents types # 6.7.3 check namespace for standard authority targetNamespaceAuthority = UrlUtil.authority( modelDocument.targetNamespace) if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities: val.modelXbrl.error( ("EFM.6.07.03", "GFM.1.03.03"), _("The target namespace, %(targetNamespace)s cannot have the same authority (%(targetNamespaceAuthority)s) as a standard " "taxonomy, in %(schema)s. Please change your target namespace." ), edgarCode="fs-0703-Extension-Has-Standard-Namespace-Authority", modelObject=modelDocument, schema=modelDocument.basename, targetNamespace=modelDocument.targetNamespace, targetNamespaceAuthority=UrlUtil.authority( modelDocument.targetNamespace, includeScheme=False)) # 6.7.4 check namespace format if modelDocument.targetNamespace is None or not modelDocument.targetNamespace.startswith( "http://"): match = None else: targetNamespaceDate = modelDocument.targetNamespace[ len(targetNamespaceAuthority):] match = targetNamespaceDatePattern.match(targetNamespaceDate) if match is not None: try: if match.lastindex == 3: date = datetime.date(int(match.group(1)), int(match.group(2)), int(match.group(3))) elif match.lastindex == 6: date = datetime.date(int(match.group(4)), int(match.group(5)), int(match.group(6))) else: match = None except ValueError: match = None if match is None: val.modelXbrl.error( ("EFM.6.07.04", "GFM.1.03.04"), _("You did not adhere to the requirements for target namespace, for %(targetNamespace)s in %(schema)s. " "Please recheck your submission and adhere to the target namespace requirements." ), edgarCode="cp-0704-Taxonomy-Valid-Target-Namespace", modelObject=modelDocument, schema=modelDocument.basename, targetNamespace=modelDocument.targetNamespace) elif val.fileNameDate and date > val.fileNameDate: val.modelXbrl.info( ("EFM.6.07.06", "GFM.1.03.06"), _("Warning: Taxonomy schema %(schema)s namespace %(targetNamespace)s has date later than document name date %(docNameDate)s" ), modelObject=modelDocument, schema=modelDocument.basename, targetNamespace=modelDocument.targetNamespace, docNameDate=val.fileNameDate) if modelDocument.targetNamespace is not None: # 6.7.5 check prefix for _ authority = UrlUtil.authority(modelDocument.targetNamespace) if not re.match(r"(http://|https://|ftp://|urn:)\w+", authority): val.modelXbrl.error( ("EFM.6.07.05", "GFM.1.03.05"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must be a valid URI with a valid authority for the namespace." ), edgarCode="du-0705-Namespace-Authority", modelObject=modelDocument, schema=modelDocument.basename, targetNamespace=modelDocument.targetNamespace) # may be multiple prefixes for namespace prefixes = [ (prefix if prefix is not None else "") for prefix, NS in modelDocument.xmlRootElement.nsmap.items() if NS == modelDocument.targetNamespace ] if not prefixes: prefix = None val.modelXbrl.error( ("EFM.6.07.07", "GFM.1.03.07"), _("The schema does not supply a prefix for %(targetNamespace)s without an underscore character, in file %(schema)s. " "Please provide or change a prefix"), edgarCode="du-0707-Recommended-Prefix-Disallowed", modelObject=modelDocument, schema=modelDocument.basename, targetNamespace=modelDocument.targetNamespace) else: prefix = prefixes[0] if len(prefixes) > 1: val.modelXbrl.error( ("EFM.6.07.07", "GFM.1.03.07"), _("The schema does not supply a prefix for %(targetNamespace)s without an underscore character, in file %(schema)s. " "Please provide or change a prefix"), edgarCode="du-0707-Recommended-Prefix-Disallowed", modelObject=modelDocument, schema=modelDocument.basename, targetNamespace=modelDocument.targetNamespace, prefix=", ".join(prefixes)) elif "_" in prefix: val.modelXbrl.error( ("EFM.6.07.07", "GFM.1.03.07"), _("The schema does not supply a prefix for %(targetNamespace)s without an underscore character, in file %(schema)s. " "Please provide or change a prefix"), edgarCode="du-0707-Recommended-Prefix-Disallowed", modelObject=modelDocument, schema=modelDocument.basename, targetNamespace=modelDocument.targetNamespace, prefix=prefix) for modelConcept in modelDocument.xmlRootElement.iterdescendants( tag="{http://www.w3.org/2001/XMLSchema}element"): if isinstance(modelConcept, ModelConcept): # 6.7.16 name not duplicated in standard taxonomies name = modelConcept.get("name") if name is None: name = "" if modelConcept.get("ref") is not None: continue # don't validate ref's here for c in val.modelXbrl.nameConcepts.get(name, []): if c.modelDocument != modelDocument: if not c.modelDocument.uri.startswith( val.modelXbrl.uriDir): val.modelXbrl.error( ("EFM.6.07.16", "GFM.1.03.18"), _("Your extension taxonomy contains an element, %(concept)s, which has the same name as an element in the base taxonomy, " "%(standardConcept)s. Please ensure that this extension is appropriate and if so, please change the extension concept name." ), edgarCode= "cp-0716-Element-Name-Same-As-Base", modelObject=(modelConcept, c), concept=modelConcept.qname, standardSchema=c.modelDocument.basename, standardConcept=c.qname) # 6.7.17 id properly formed _id = modelConcept.id requiredId = (prefix if prefix is not None else "") + "_" + name if _id != requiredId: val.modelXbrl.error( ("EFM.6.07.17", "GFM.1.03.19"), _("You did not adhere to the declarations for concepts by containing an 'id' attribute whose value begins with the recommended " "namespace prefix of the taxonomy, followed by an underscore, followed by an element name, for the concept %(concept)s. " "Please recheck your submission."), edgarCode="cp-0717-Element-Id", modelObject=modelConcept, concept=modelConcept.qname, id=_id, requiredId=requiredId) # 6.7.18 nillable is true nillable = modelConcept.get("nillable") if nillable != "true" and modelConcept.isItem: val.modelXbrl.error( ("EFM.6.07.18", "GFM.1.03.20"), _("Element %(concept)s is declared without a 'true' value for the nillable attribute. Please set the value to 'true'." ), edgarCode="du-0718-Nillable-Not-True", modelObject=modelConcept, schema=modelDocument.basename, concept=name, nillable=nillable) # 6.7.19 not tuple if modelConcept.isTuple: val.modelXbrl.error( ("EFM.6.07.19", "GFM.1.03.21"), _("You provided an extension concept which is a tuple, %(concept)s. Please remove tuples and check your submission." ), edgarCode="cp-0719-No-Tuple-Element", modelObject=modelConcept, concept=modelConcept.qname) # 6.7.20 no typed domain ref if modelConcept.isTypedDimension: val.modelXbrl.error( ("EFM.6.07.20", "GFM.1.03.22"), _("There is an xbrldt:typedDomainRef attribute on %(concept)s (%(typedDomainRef)s). Please remove it." ), edgarCode="du-0720-Typed-Domain-Ref-Disallowed", modelObject=modelConcept, concept=modelConcept.qname, typedDomainRef=modelConcept.typedDomainElement. qname if modelConcept.typedDomainElement is not None else modelConcept.typedDomainRef) # 6.7.21 abstract must be duration isDuration = modelConcept.periodType == "duration" if modelConcept.isAbstract and not isDuration: val.modelXbrl.error( ("EFM.6.07.21", "GFM.1.03.23"), _("Element %(concept)s is declared as an abstract item with period type 'instant'. Please change it to 'duration' or " "make the element not abstract."), edgarCode="du-0721-Abstract-Is-Instant", modelObject=modelConcept, schema=modelDocument.basename, concept=modelConcept.qname) # 6.7.22 abstract must be stringItemType ''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08 if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType: val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"), _("Concept %(concept)s is abstract but type is not xbrli:stringItemType"), modelObject=modelConcept, concept=modelConcept.qname) ''' substitutionGroupQname = modelConcept.substitutionGroupQname # 6.7.23 Axis must be subs group dimension if name.endswith("Axis") ^ ( substitutionGroupQname == XbrlConst.qnXbrldtDimensionItem): val.modelXbrl.error( ("EFM.6.07.23", "GFM.1.03.25"), _("The substitution group 'xbrldt:dimensionItem' is only consistent with an element name that ends with 'Axis'. " "Please change %(conceptLocalName)s or change the substitutionGroup." ), edgarCode="du-0723-Axis-Dimension-Name-Mismatch", modelObject=modelConcept, concept=modelConcept.qname, conceptLocalName=modelConcept.qname.localName) # 6.7.24 Table must be subs group hypercube if name.endswith("Table") ^ ( substitutionGroupQname == XbrlConst.qnXbrldtHypercubeItem): val.modelXbrl.error( ("EFM.6.07.24", "GFM.1.03.26"), _("The substitution group 'xbrldt:hypercubeItem' is only allowed with an element name that ends with 'Table'. " "Please change %(conceptLocalName)s or change the substitutionGroup." ), edgarCode="du-0724-Table-Hypercube-Name-Mismatch", modelObject=modelConcept, schema=modelDocument.basename, concept=modelConcept.qname, conceptLocalName=modelConcept.qname.localName) # 6.7.25 if neither hypercube or dimension, substitution group must be item if substitutionGroupQname not in ( None, XbrlConst.qnXbrldtDimensionItem, XbrlConst.qnXbrldtHypercubeItem, XbrlConst.qnXbrliItem): val.modelXbrl.error( ("EFM.6.07.25", "GFM.1.03.27"), _("The substitution group attribute value %(substitutionGroup)s of element %(conceptLocalName)s is not allowed. " "Please change it to one of 'xbrli:item', 'xbrldt:dimensionItem' or 'xbrldt:hypercubeItem'." ), edgarCode="du-0725-Substitution-Group-Custom", modelObject=modelConcept, concept=modelConcept.qname, conceptLocalName=modelConcept.qname.localName, substitutionGroup=modelConcept. substitutionGroupQname) # 6.7.26 Table must be subs group hypercube if name.endswith( "LineItems") and modelConcept.abstract != "true": val.modelXbrl.error( ("EFM.6.07.26", "GFM.1.03.28"), _("The element %(conceptLocalName)s ends with 'LineItems' but is not abstract. Please change %(conceptLocalName)s or " "the value of the 'abstract' attribute."), edgarCode= "du-0726-LineItems-Abstract-Name-Mismatch", modelObject=modelConcept, concept=modelConcept.qname, conceptLocalName=modelConcept.qname.localName) # 6.7.27 type domainMember must end with Domain or Member conceptType = modelConcept.type isDomainItemType = conceptType is not None and conceptType.isDomainItemType endsWithDomainOrMember = name.endswith( "Domain") or name.endswith("Member") if isDomainItemType != endsWithDomainOrMember: val.modelXbrl.error( ("EFM.6.07.27", "GFM.1.03.29"), _("The type 'us-types:domainItemType' is only allowed with an element name that ends with 'Domain' or 'Member'. " "Please change %(conceptLocalName)s or change the type." ), edgarCode="du-0727-Domain-Type-Name-Mismatch", modelObject=modelConcept, concept=modelConcept.qname, conceptLocalName=modelConcept.qname.localName) # 6.7.28 domainItemType must be duration if isDomainItemType and not isDuration: val.modelXbrl.error( ("EFM.6.07.28", "GFM.1.03.30"), _("Element %(conceptLocalName)s is declared as a us-types:domainItemType with period type 'instant'. " "Please change it to 'duration' or change the item type." ), edgarCode="du-0728-Domain-Member-Is-Instant", modelObject=modelConcept, concept=modelConcept.qname, conceptLocalName=modelConcept.qname.localName) #6.7.31 (version 27) fractions if modelConcept.isFraction: val.modelXbrl.error( "EFM.6.07.31", _("Element %(concept)s is declared as a fraction item type. Change or remove the declaration." ), edgarCode="du-0731-Fraction-Item-Type", modelObject=modelConcept, concept=modelConcept.qname) #6.7.32 (version 27) instant non numeric if modelConcept.isItem and (not modelConcept.isNumeric and not isDuration and not modelConcept.isAbstract and not isDomainItemType): val.modelXbrl.error( "EFM.6.07.32", _("Declaration of element %(concept)s in %(schema)s must have xbrli:periodType of 'duration' because its base type is not numeric." ), edgarCode="rq-0732-Nonnnumeric-Must-Be-Duration", modelObject=modelConcept, schema=modelDocument.basename, concept=modelConcept.qname) # 6.8.5 semantic check, check LC3 name if name: if not name[0].isupper(): val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.05.firstLetter", "GFM.2.03.05.firstLetter"), _("Concept %(concept)s name must start with a capital letter" ), modelObject=modelConcept, concept=modelConcept.qname) if namePattern.search(name): val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.05.disallowedCharacter", "GFM.2.03.05.disallowedCharacter"), _("Concept %(concept)s has disallowed name character" ), modelObject=modelConcept, concept=modelConcept.qname) if len(name) > 200: val.modelXbrl.log( "ERROR-SEMANTIC", "EFM.6.08.05.nameLength", _("Concept %(concept)s name length %(namelength)s exceeds 200 characters" ), modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name)) if isEFM: label = modelConcept.label(lang="en-US", fallbackToQname=False) if label: # allow Joe's Bar, N.A. to be JoesBarNA -- remove ', allow A. as not article "a" lc3name = ''.join( re.sub(r"['.-]", "", ( w[0] or w[2] or w[3] or w[4])).title() for w in re.findall( r"((\w+')+\w+)|(A[.-])|([.-]A(?=\W|$))|(\w+)", label ) # EFM implies this should allow - and . re.findall(r"[\w\-\.]+", label) if w[4].lower() not in ("the", "a", "an")) if not (name == lc3name or (name and lc3name and lc3name[0].isdigit() and name[1:] == lc3name and (name[0].isalpha() or name[0] == '_'))): val.modelXbrl.log( "WARNING-SEMANTIC", "EFM.6.08.05.LC3", _("Concept %(concept)s should match expected LC3 composition %(lc3name)s" ), modelObject=modelConcept, concept=modelConcept.qname, lc3name=lc3name) if conceptType is not None: # 6.8.6 semantic check if not isDomainItemType and conceptType.qname != XbrlConst.qnXbrliDurationItemType: nameProblems = nonDomainItemNameProblemPattern.findall( name) if any( any(t) for t in nameProblems ): # list of tuples with possibly nonempty strings val.modelXbrl.log( "WARNING-SEMANTIC", ("EFM.6.08.06", "GFM.2.03.06"), _("Concept %(concept)s should not contain company or period information, found: %(matches)s" ), modelObject=modelConcept, concept=modelConcept.qname, matches=", ".join(''.join(t) for t in nameProblems)) if conceptType.qname == XbrlConst.qnXbrliMonetaryItemType: if not modelConcept.balance: # 6.8.11 may not appear on a income or balance statement if any( linkroleDefinitionBalanceIncomeSheet. match(roleType.definition) for rel in val.modelXbrl. relationshipSet(XbrlConst.parentChild). toModelObject(modelConcept) for roleType in val.modelXbrl. roleTypes.get(rel.linkrole, ())): val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.11", "GFM.2.03.11"), _("Concept %(concept)s must have a balance because it appears in a statement of income or balance sheet" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.11.5 semantic check, must have a documentation label stdLabel = modelConcept.label( lang="en-US", fallbackToQname=False) defLabel = modelConcept.label( preferredLabel=XbrlConst. documentationLabel, lang="en-US", fallbackToQname=False) if not defLabel or ( # want different words than std label stdLabel and re.findall(r"\w+", stdLabel) == re.findall(r"\w+", defLabel)): val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.11.05", "GFM.2.04.04"), _("Concept %(concept)s is monetary without a balance and must have a documentation label that disambiguates its sign" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.8.16 semantic check if conceptType.qname == XbrlConst.qnXbrliDateItemType and modelConcept.periodType != "duration": val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.16", "GFM.2.03.16"), _("Concept %(concept)s of type xbrli:dateItemType must have periodType duration" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.8.17 semantic check if conceptType.qname == XbrlConst.qnXbrliStringItemType and modelConcept.periodType != "duration": val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.17", "GFM.2.03.17"), _("Concept %(concept)s of type xbrli:stringItemType must have periodType duration" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.8 check for embedded linkbase for e in modelDocument.xmlRootElement.iterdescendants( tag="{http://www.xbrl.org/2003/linkbase}linkbase"): if isinstance(e, ModelObject): val.modelXbrl.error( ("EFM.6.07.08", "GFM.1.03.08"), _("Your filing contained embedded linkbases in %(schema)s. Please recheck your submission and remove all embedded linkbases." ), edgarCode="cp-0708-No-Embedded-Linkbases", modelObject=e, schema=modelDocument.basename) break requiredUsedOns = { XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink, XbrlConst.qnLinkDefinitionLink } standardUsedOns = { XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference, XbrlConst.qnLinkDefinitionArc, XbrlConst.qnLinkCalculationArc, XbrlConst.qnLinkPresentationArc, XbrlConst.qnLinkLabelArc, XbrlConst.qnLinkReferenceArc, # per WH, private footnote arc and footnore resource roles are not allowed XbrlConst.qnLinkFootnoteArc, XbrlConst.qnLinkFootnote, } # 6.7.9 role types authority for e in modelDocument.xmlRootElement.iterdescendants( tag="{http://www.xbrl.org/2003/linkbase}roleType"): if isinstance(e, ModelObject): roleURI = e.get("roleURI") if targetNamespaceAuthority != UrlUtil.authority(roleURI): val.modelXbrl.error( ("EFM.6.07.09", "GFM.1.03.09"), _("Role %(roleType)s does not begin with %(targetNamespace)s's scheme and authority. " "Please change the role URI or target namespace URI." ), edgarCode="du-0709-Role-Namespace-Mismatch", modelObject=e, roleType=roleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace) # 6.7.9 end with .../role/lc3 name if not roleTypePattern.match(roleURI): val.modelXbrl.warning( ("EFM.6.07.09.roleEnding", "GFM.1.03.09"), "RoleType %(roleType)s should end with /role/{LC3name}", modelObject=e, roleType=roleURI) # 6.7.10 only one role type declaration in DTS modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI) if modelRoleTypes is not None: modelRoleType = modelRoleTypes[0] definition = modelRoleType.definitionNotStripped usedOns = modelRoleType.usedOns if len(modelRoleTypes) == 1: # 6.7.11 used on's for pre, cal, def if any has a used on if not usedOns.isdisjoint(requiredUsedOns) and len( requiredUsedOns - usedOns) > 0: val.modelXbrl.error( ("EFM.6.07.11", "GFM.1.03.11"), _("The role %(roleType)s did not provide a usedOn element for all three link types (presentation, " "calculation and definition), missing %(usedOn)s. Change the declaration to be for all three types of link, and resubmit." ), edgarCode= "du-0711-Role-Type-Declaration-Incomplete", modelObject=e, roleType=roleURI, usedOn=requiredUsedOns - usedOns) # 6.7.12 definition match pattern if (val.disclosureSystem.roleDefinitionPattern is not None and (definition is None or not val.disclosureSystem. roleDefinitionPattern.match(definition))): val.modelXbrl.error( ("EFM.6.07.12", "GFM.1.03.12-14"), _("The definition '%(definition)s' of role %(roleType)s does not match the expected format. " "Please check that the definition matches {number} - {type} - {text}." ), edgarCode="rq-0712-Role-Definition-Mismatch", modelObject=e, roleType=roleURI, definition=(definition or "")) if usedOns & standardUsedOns: # semantics check val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"), _("RoleType %(roleuri)s is defined using role types already defined by standard roles for: %(qnames)s" ), modelObject=e, roleuri=roleURI, qnames=', '.join( str(qn) for qn in usedOns & standardUsedOns)) # 6.7.13 arcrole types authority for e in modelDocument.xmlRootElement.iterdescendants( tag="{http://www.xbrl.org/2003/linkbase}arcroleType"): if isinstance(e, ModelObject): arcroleURI = e.get("arcroleURI") if targetNamespaceAuthority != UrlUtil.authority(arcroleURI): val.modelXbrl.error( ("EFM.6.07.13", "GFM.1.03.15"), _("Relationship role %(arcroleType)s does not begin with %(targetNamespace)s's scheme and authority. " "Please change the relationship role URI or target namespace URI." ), edgarCode="du-0713-Arcrole-Namespace-Mismatch", modelObject=e, arcroleType=arcroleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace) # 6.7.13 end with .../arcrole/lc3 name if not arcroleTypePattern.match(arcroleURI): val.modelXbrl.warning( ("EFM.6.07.13.arcroleEnding", "GFM.1.03.15"), _("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}" ), modelObject=e, arcroleType=arcroleURI) # 6.7.15 definition match pattern modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI] definition = modelRoleTypes[0].definition if definition is None or not arcroleDefinitionPattern.match( definition): val.modelXbrl.error( ("EFM.6.07.15", "GFM.1.03.17"), _("Relationship role declaration %(arcroleType)s is missing a definition. Please provide a definition." ), edgarCode="du-0715-Arcrole-Definition-Missing", modelObject=e, arcroleType=arcroleURI) # semantic checks usedOns = modelRoleTypes[0].usedOns if usedOns & standardUsedOns: # semantics check val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"), _("ArcroleType %(arcroleuri)s is defined using role types already defined by standard arcroles for: %(qnames)s" ), modelObject=e, arcroleuri=arcroleURI, qnames=', '.join( str(qn) for qn in usedOns & standardUsedOns)) #6.3.3 filename check m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9]).xsd$", modelDocument.basename) if m: try: # check date value datetime.datetime.strptime(m.group(1), "%Y%m%d").date() # date and format are ok, check "should" part of 6.3.3 if val.fileNameBasePart: expectedFilename = "{0}-{1}.xsd".format( val.fileNameBasePart, val.fileNameDatePart) if modelDocument.basename != expectedFilename: val.modelXbrl.log( "WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"), _('Schema file name warning: %(filename)s, should match %(expectedFilename)s' ), modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename) except ValueError: val.modelXbrl.error( (val.EFM60303, "GFM.1.01.01"), _('Invalid schema file base name part (date) in "{base}-{yyyymmdd}.xsd": %(filename)s' ), modelObject=modelDocument, filename=modelDocument.basename, messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01")) else: val.modelXbrl.error( (val.EFM60303, "GFM.1.01.01"), _('Invalid schema file name, must match "{base}-{yyyymmdd}.xsd": %(filename)s' ), modelObject=modelDocument, filename=modelDocument.basename, messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01")) elif modelDocument.type == ModelDocument.Type.LINKBASE: # if it is part of the submission (in same directory) check name labelRels = None if modelDocument.filepath.startswith( val.modelXbrl.modelDocument.filepathdir): #6.3.3 filename check extLinkElt = XmlUtil.descendant( modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended") if extLinkElt is None: # no ext link element val.modelXbrl.error( (val.EFM60303 + ".noLinkElement", "GFM.1.01.01.noLinkElement"), _('Invalid linkbase file name: %(filename)s, has no extended link element, cannot determine link type.' ), modelObject=modelDocument, filename=modelDocument.basename, messageCodes=("EFM.6.03.03.noLinkElement", "EFM.6.23.01.noLinkElement", "GFM.1.01.01.noLinkElement")) elif extLinkElt.localName not in extLinkEltFileNameEnding: val.modelXbrl.error( "EFM.6.03.02", _('Invalid linkbase link element %(linkElement)s in %(filename)s' ), modelObject=modelDocument, linkElement=extLinkElt.localName, filename=modelDocument.basename) else: m = re.match( r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9])(_[a-z]{3}).xml$", modelDocument.basename) expectedSuffix = extLinkEltFileNameEnding[extLinkElt.localName] if m and m.group(2) == expectedSuffix: try: # check date value datetime.datetime.strptime(m.group(1), "%Y%m%d").date() # date and format are ok, check "should" part of 6.3.3 if val.fileNameBasePart: expectedFilename = "{0}-{1}{2}.xml".format( val.fileNameBasePart, val.fileNameDatePart, expectedSuffix) if modelDocument.basename != expectedFilename: val.modelXbrl.log( "WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"), _('Linkbase name warning: %(filename)s should match %(expectedFilename)s' ), modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename) except ValueError: val.modelXbrl.error( (val.EFM60303, "GFM.1.01.01"), _('Invalid linkbase base file name part (date) in "{base}-{yyyymmdd}_{suffix}.xml": %(filename)s' ), modelObject=modelDocument, filename=modelDocument.basename, messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01")) else: val.modelXbrl.error( (val.EFM60303, "GFM.1.01.01"), _('Invalid linkbase name, must match "{base}-{yyyymmdd}%(expectedSuffix)s.xml": %(filename)s' ), modelObject=modelDocument, filename=modelDocument.basename, expectedSuffix=expectedSuffix, messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01")) if extLinkElt.localName == "labelLink": if labelRels is None: labelRels = val.modelXbrl.relationshipSet( XbrlConst.conceptLabel) for labelElt in XmlUtil.children(extLinkElt, XbrlConst.link, "label"): # 6.10.9 if XbrlConst.isNumericRole(labelElt.role): for rel in labelRels.toModelObject(labelElt): if rel.fromModelObject is not None and not rel.fromModelObject.isNumeric: val.modelXbrl.error( "EFM.6.10.09", _("Non-numeric element %(concept)s has a label role for numeric elements: %(role)s. " "Please change the role attribute."), edgarCode="du-1009-Numeric-Label-Role", modelObject=(labelElt, rel.fromModelObject), concept=rel.fromModelObject.qname, role=labelElt.role)
def loadFromExcel(cntlr, excelFile): from arelle import xlrd from arelle.xlrd.sheet import empty_cell from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname startedAt = time.time() importExcelBook = xlrd.open_workbook(excelFile) controlSheet = importExcelBook.sheet_by_index(1) imports = {"xbrli": ( ("namespace", XbrlConst.xbrli), ("schemaLocation", "http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd") )} # xml of imports importXmlns = {} linkbaseRefs = [] labelLinkbases = [] hasPreLB = hasCalLB = hasDefLB = False # xxxLB structure [ (elr1, def1, "_ELR_", [roots]), (elr2, def2, "_ELR_", [rootw]) ...] # roots = (rootHref, None, "_root_", [children]) # children = (childPrefix, childName, arcrole, [grandChildren]) preLB = [] defLB = [] calLB = [] def lbDepthList(lbStruct, depth, parentList=None): if depth == 0: return lbStruct[-1].childStruct return lbDepthList(lbStruct[-1].childStruct, depth-1, list) extensionElements = {} extensionRoles = {} # key is roleURI, value is role definition extensionLabels = {} # key = (prefix, name, lang, role), value = label text def extensionHref(prefix, name): if prefix == extensionSchemaPrefix: filename = extensionSchemaFilename elif prefix in imports: filename = imports[prefix][1][1] else: return None return "{0}#{1}_{2}".format(filename, prefix, name) isUSGAAP = False for iRow in range(1, controlSheet.nrows): try: row = controlSheet.row(iRow) if (row[0].ctype == xlrd.XL_CELL_EMPTY): # skip if col 1 is empty continue action = row[0].value filetype = row[1].value prefix = row[2].value filename = row[3].value namespaceURI = row[4].value lbType = lang = None if action == "import": imports[prefix] = ( ("namespace", namespaceURI), ("schemaLocation", filename) ) importXmlns[prefix] = namespaceURI if re.match(r"http://[^/]+/us-gaap/", namespaceURI): isUSGAAP = True elif action == "extension": if filetype == "schema": extensionSchemaPrefix = prefix extensionSchemaFilename = filename extensionSchemaNamespaceURI = namespaceURI elif filetype == "linkbase": typeLang = prefix.split() if len(typeLang) > 0: lbType = typeLang[0] else: lbType = "unknown" if len(typeLang) > 1: lang = typeLang[1] else: lang = "en" if lbType == "label": labelLinkbases.append((lang, filename)) elif lbType == "presentation": hasPreLB = True elif lbType == "definition": hasDefLB = True elif lbType == "calculation": hasCalLB = True linkbaseRefs.append( (lbType, filename) ) elif filetype == "role" and namespaceURI: extensionRoles[namespaceURI] = filename except Exception as err: cntlr.addToLog("Exception: {error}, Excel row: {excelRow}" .format(error=err, excelRow=iRow), messageCode="importExcel:exception") importExcelSheet = importExcelBook.sheet_by_index(0) # find column headers row headerCols = {} headerRows = set() # find out which rows are header rows for iRow in range(0, importExcelSheet.nrows): row = importExcelSheet.row(iRow) for iCol, colCell in enumerate(row): v = colCell.value if v in importColumnHeaders: headerCols[importColumnHeaders[v]] = iCol if all(colName in headerCols for colName in ("name", "type", "depth")): # must have these to be a header col # it's a header col headerRows.add(iRow) headerCols.clear() def cellValue(row, header): if header in headerCols: iCol = headerCols[header] if iCol < len(row): return row[iCol].value return '' def checkImport(qname): prefix, sep, localName = qname.partition(":") if sep: if prefix not in imports: if prefix == "xbrldt": imports["xbrldt"] = ("namespace", XbrlConst.xbrldt), ("schemaLocation", "http://www.xbrl.org/2005/xbrldt-2005.xsd") elif prefix == "nonnum": imports["nonnum"] = ("namespace", "http://www.xbrl.org/dtr/type/non-numeric"), ("schemaLocation", "http://www.xbrl.org/dtr/type/nonNumeric-2009-12-16.xsd") else: cntlr.addToLog("Warning: prefix schema file is not imported for: {qname}" .format(qname=qname), messageCode="importExcel:warning") # find header rows currentELR = currentELRdefinition = None for iRow in range(0, importExcelSheet.nrows): useLabels = False try: row = importExcelSheet.row(iRow) isHeaderRow = iRow in headerRows isELRrow = (iRow + 1) in headerRows if isHeaderRow: headerCols.clear() for iCol, colCell in enumerate(row): v = colCell.value if v in importColumnHeaders: headerCols[importColumnHeaders[v]] = iCol elif isELRrow: currentELR = currentELRdefinition = None for colCell in row: v = colCell.value if v.startswith("http://"): currentELR = v elif not currentELRdefinition and v.endswith(" 科目一覧"): currentELRdefinition = v[0:-5] elif not currentELRdefinition: currentELRdefinition = v if currentELR or currentELRdefinition: if hasPreLB: preLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) ) if hasDefLB: defLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) ) if hasCalLB: calLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) ) elif headerCols: prefix = cellValue(row, 'prefix').strip() name = cellValue(row, 'name').strip() if "depth" in headerCols: try: depth = int(cellValue(row, 'depth')) except ValueError: depth = None else: depth = None if prefix == extensionSchemaPrefix and name not in extensionElements: # elements row eltType = cellValue(row, 'type') subsGrp = cellValue(row, 'substitutionGroup') abstract = cellValue(row, 'abstract') nillable = cellValue(row, 'nillable') balance = cellValue(row, 'balance') periodType = cellValue(row, 'periodType') newElt = [ ("name", name), ("id", prefix + "_" + name) ] if eltType: newElt.append( ("type", eltType) ) checkImport(eltType) if subsGrp: newElt.append( ("substitutionGroup", subsGrp) ) checkImport(subsGrp) if abstract: newElt.append( ("abstract", abstract) ) if nillable: newElt.append( ("nillable", nillable) ) if balance: newElt.append( ("{http://www.xbrl.org/2003/instance}balance", balance) ) if periodType: newElt.append( ("{http://www.xbrl.org/2003/instance}periodType", periodType) ) extensionElements[name] = newElt useLabels = True if depth is not None: if hasPreLB: entryList = lbDepthList(preLB, depth) preferredLabel = cellValue(row, 'preferredLabel') if preferredLabel and not preferredLabel.startswith("http://"): preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel if entryList is not None: if depth == 0: entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) ) else: entryList.append( LBentry(prefix=prefix, name=name, arcrole=XbrlConst.parentChild, role=preferredLabel) ) if hasDefLB: entryList = lbDepthList(defLB, depth) if entryList is not None: if depth == 0: entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) ) else: entryList.append( LBentry(prefix=prefix, name=name, arcrole="_dimensions_") ) if hasCalLB: calcParent = cellValue(row, 'calculationParent') calcWeight = cellValue(row, 'calculationWeight') if calcParent and calcWeight: calcParentPrefix, sep, calcParentName = calcParent.partition(":") entryList = lbDepthList(calLB, 0) if entryList is not None: entryList.append( LBentry(prefix=calcParentPrefix, name=calcParentName, isRoot=True, childStruct= [LBentry(prefix=prefix, name=name, arcrole=XbrlConst.summationItem, weight=calcWeight )]) ) # accumulate extension labels if useLabels: prefix = cellValue(row, 'prefix').strip() name = cellValue(row, 'name').strip() preferredLabel = cellValue(row, 'preferredLabel') if preferredLabel and not preferredLabel.startswith("http://"): preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel for colItem, iCol in headerCols.items(): if isinstance(colItem, tuple): colItemType, role, lang = colItem cell = row[iCol] if cell.ctype == xlrd.XL_CELL_EMPTY: values = () elif colItemType == "label": values = (cell.value,) elif colItemType == "labels": values = cell.value.split('\n') else: values = () if preferredLabel: # first label column sets preferredLabel if any role = preferredLabel preferredLabel = None for value in values: extensionLabels[prefix, name, lang, role] = value.strip() except Exception as err: cntlr.addToLog("Exception: {error}, Excel row: {excelRow}" .format(error=err, excelRow=iRow), messageCode="importExcel:exception") if isUSGAAP and hasDefLB: # move line items above table def fixUsggapTableDims(lvl1Struct): foundLineItems = False for i1, lvl1Entry in enumerate(lvl1Struct): for i2, lvl2Entry in enumerate(lvl1Entry.childStruct): for i3, lvl3Entry in enumerate(lvl2Entry.childStruct): if lvl3Entry.name.endswith("LineItems") and lvl2Entry.name.endswith("Table"): foundLineItems = True break if foundLineItems: break else: fixUsggapTableDims(lvl1Entry.childStruct) if foundLineItems: lvl1Struct.insert(i1 + 1, LBentry(prefix=lvl3Entry.prefix, name=lvl3Entry.name, arcrole=lvl1Entry.arcrole, childStruct=lvl3Entry.childStruct)) # must keep lvl1Rel if it is __root__ lvl3Entry.childStruct.insert(0, lvl2Entry) if lvl1Entry.name.endswith("Abstract"): del lvl1Struct[i1] del lvl2Entry.childStruct[i3] pass fixUsggapTableDims(defLB) dts = cntlr.modelManager.create(newDocumentType=ModelDocument.Type.SCHEMA, url=extensionSchemaFilename, isEntry=True, base='', # block pathname from becomming absolute initialXml=''' <schema xmlns="http://www.w3.org/2001/XMLSchema" targetNamespace="{targetNamespace}" attributeFormDefault="unqualified" elementFormDefault="qualified" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:{extensionPrefix}="{targetNamespace}" {importXmlns} xmlns:iod="http://disclosure.edinet-fsa.go.jp/taxonomy/common/2013-03-31/iod" xmlns:nonnum="http://www.xbrl.org/dtr/type/non-numeric" xmlns:link="http://www.xbrl.org/2003/linkbase" xmlns:xbrli="http://www.xbrl.org/2003/instance" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xbrldt="http://xbrl.org/2005/xbrldt"/> '''.format(targetNamespace=extensionSchemaNamespaceURI, extensionPrefix=extensionSchemaPrefix, importXmlns=''.join('xmlns:{0}="{1}"\n'.format(prefix, namespaceURI) for prefix, namespaceURI in importXmlns.items()) ) ) dtsSchemaDocument = dts.modelDocument dtsSchemaDocument.inDTS = True # entry document always in DTS dtsSchemaDocument.targetNamespace = extensionSchemaNamespaceURI # not set until schemaDiscover too late otherwise schemaElt = dtsSchemaDocument.xmlRootElement #foreach linkbase annotationElt = XmlUtil.addChild(schemaElt, XbrlConst.xsd, "annotation") appinfoElt = XmlUtil.addChild(annotationElt, XbrlConst.xsd, "appinfo") for iRow in range(0, importExcelSheet.nrows): try: row = importExcelSheet.row(iRow) if (row[0].ctype == xlrd.XL_CELL_EMPTY): # skip if col 1 is empty continue testDir = row[0].value uriFrom = row[1].value uriTo = row[2].value except Exception as err: cntlr.addToLog("Exception: {error}, Excel row: {excelRow}" .format(error=err, excelRow=iRow), messageCode="loadFromExcel:exception") # add linkbaseRefs appinfoElt = XmlUtil.descendant(schemaElt, XbrlConst.xsd, "appinfo") # don't yet add linkbase refs, want to process imports first to get roleType definitions # add imports for importAttributes in sorted(imports.values()): XmlUtil.addChild(schemaElt, XbrlConst.xsd, "import", attributes=importAttributes) # add elements for eltName, eltAttrs in sorted(extensionElements.items(), key=lambda item: item[0]): XmlUtil.addChild(schemaElt, XbrlConst.xsd, "element", attributes=eltAttrs) # add role definitions (for discovery) for roleURI, roleDefinition in extensionRoles.items(): roleElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "roleType", attributes=(("roleURI", roleURI), ("id", "roleType_" + roleURI.rpartition("/")[2]))) if roleDefinition: XmlUtil.addChild(roleElt, XbrlConst.link, "definition", text=roleDefinition) if hasPreLB: XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:presentationLink") if hasDefLB: XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:definitionLink") if hasCalLB: XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:calculationLink") dtsSchemaDocument.schemaDiscover(schemaElt, False, extensionSchemaNamespaceURI) def addLinkbaseRef(lbType, lbFilename, lbDoc): role = "http://www.xbrl.org/2003/role/{0}LinkbaseRef".format(lbType) lbRefElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "linkbaseRef", attributes=(("{http://www.w3.org/1999/xlink}type", "simple"), ("{http://www.w3.org/1999/xlink}href", lbFilename), ("{http://www.w3.org/1999/xlink}role", role), ("{http://www.w3.org/1999/xlink}arcrole", "http://www.w3.org/1999/xlink/properties/linkbase"), )) dtsSchemaDocument.referencesDocument[lbDoc] = ModelDocumentReference("href", lbRefElt) # label linkbase for lang, filename in labelLinkbases: lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base="", initialXml=""" <link:linkbase xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd" xmlns:link="http://www.xbrl.org/2003/linkbase" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xbrli="http://www.xbrl.org/2003/instance"/> """) lbDoc.inDTS = True addLinkbaseRef("label", filename, lbDoc) lbElt = lbDoc.xmlRootElement linkElt = XmlUtil.addChild(lbElt, XbrlConst.link, "labelLink", attributes=(("{http://www.w3.org/1999/xlink}type", "extended"), ("{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/link"))) locs = set() for labelKey, text in extensionLabels.items(): prefix, name, labelLang, role = labelKey if lang == labelLang: locLabel = prefix + "_" + name if locLabel not in locs: locs.add(locLabel) XmlUtil.addChild(linkElt, XbrlConst.link, "loc", attributes=(("{http://www.w3.org/1999/xlink}type", "locator"), ("{http://www.w3.org/1999/xlink}href", extensionHref(prefix, name)), ("{http://www.w3.org/1999/xlink}label", locLabel))) XmlUtil.addChild(linkElt, XbrlConst.link, "labelArc", attributes=(("{http://www.w3.org/1999/xlink}type", "arc"), ("{http://www.w3.org/1999/xlink}arcrole", "http://www.xbrl.org/2003/arcrole/concept-label"), ("{http://www.w3.org/1999/xlink}from", locLabel), ("{http://www.w3.org/1999/xlink}to", "label_" + locLabel), ("order", 1.0))) XmlUtil.addChild(linkElt, XbrlConst.link, "label", attributes=(("{http://www.w3.org/1999/xlink}type", "resource"), ("{http://www.w3.org/1999/xlink}label", "label_" + locLabel), ("{http://www.w3.org/1999/xlink}role", role), ("{http://www.w3.org/XML/1998/namespace}lang", lang)), text=text) lbDoc.linkbaseDiscover(lbElt) def hrefConcept(prefix, name): qn = schemaElt.prefixedNameQname(prefix + ":" + name) if qn in dts.qnameConcepts: return dts.qnameConcepts[qn] return None def lbTreeWalk(lbType, parentElt, lbStruct, roleRefs, locs=None, fromPrefix=None, fromName=None): order = 1.0 for lbEntry in lbStruct: if lbEntry.isELR: role = "unspecified" if lbEntry.role and lbEntry.role.startswith("http://"): # have a role specified role = lbEntry.role elif lbEntry.name: #may be a definition for linkroleUri, modelRoleTypes in dts.roleTypes.items(): definition = modelRoleTypes[0].definition if lbEntry.name == definition: role = linkroleUri break if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef roleType = modelRoleTypes[0] roleRef = ("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id) roleRefs.add(roleRef) linkElt = XmlUtil.addChild(parentElt, XbrlConst.link, lbType + "Link", attributes=(("{http://www.w3.org/1999/xlink}type", "extended"), ("{http://www.w3.org/1999/xlink}role", role))) locs = set() lbTreeWalk(lbType, linkElt, lbEntry.childStruct, roleRefs, locs) else: toPrefix = lbEntry.prefix toName = lbEntry.name toHref = extensionHref(toPrefix, toName) toLabel = toPrefix + "_" + toName if toHref not in locs: XmlUtil.addChild(parentElt, XbrlConst.link, "loc", attributes=(("{http://www.w3.org/1999/xlink}type", "locator"), ("{http://www.w3.org/1999/xlink}href", toHref), ("{http://www.w3.org/1999/xlink}label", toLabel))) locs.add(toHref) if not lbEntry.isRoot: fromLabel = fromPrefix + "_" + fromName if lbType == "calculation" and lbEntry.weight is not None: otherAttrs = ( ("weight", lbEntry.weight), ) elif lbType == "presentation" and lbEntry.role is not None: otherAttrs = ( ("preferredLabel", lbEntry.role), ) else: otherAttrs = ( ) if lbEntry.arcrole == "_dimensions_": # pick proper consecutive arcrole fromConcept = hrefConcept(fromPrefix, fromName) toConcept = hrefConcept(toPrefix, toName) if toConcept is not None and toConcept.isHypercubeItem: arcrole = XbrlConst.all elif toConcept is not None and toConcept.isDimensionItem: arcrole = XbrlConst.hypercubeDimension elif fromConcept is not None and fromConcept.isDimensionItem: arcrole = XbrlConst.dimensionDomain else: arcrole = XbrlConst.domainMember else: arcrole = lbEntry.arcrole XmlUtil.addChild(parentElt, XbrlConst.link, lbType + "Arc", attributes=(("{http://www.w3.org/1999/xlink}type", "arc"), ("{http://www.w3.org/1999/xlink}arcrole", arcrole), ("{http://www.w3.org/1999/xlink}from", fromLabel), ("{http://www.w3.org/1999/xlink}to", toLabel), ("order", order)) + otherAttrs ) order += 1.0 if lbType != "calculation" or lbEntry.isRoot: lbTreeWalk(lbType, parentElt, lbEntry.childStruct, roleRefs, locs, toPrefix, toName) for hasLB, lbType, lbLB in ((hasPreLB, "presentation", preLB), (hasDefLB, "definition", defLB), (hasCalLB, "calculation", calLB)): if hasLB: for lbRefType, filename in linkbaseRefs: if lbType == lbRefType: # output presentation linkbase lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base='', initialXml=""" <link:linkbase xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd" xmlns:link="http://www.xbrl.org/2003/linkbase" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xbrli="http://www.xbrl.org/2003/instance"/> """) lbDoc.inDTS = True addLinkbaseRef(lbRefType, filename, lbDoc) lbElt = lbDoc.xmlRootElement roleRefs = set() if lbType == "definition": roleRefs.update((("arcroleRef", XbrlConst.all, "http://www.xbrl.org/2005/xbrldt-2005.xsd#all"), ("arcroleRef", XbrlConst.dimensionDefault, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-default"), ("arcroleRef", XbrlConst.dimensionDomain, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-domain"), ("arcroleRef", XbrlConst.domainMember, "http://www.xbrl.org/2005/xbrldt-2005.xsd#domain-member"), ("arcroleRef", XbrlConst.hypercubeDimension, "http://www.xbrl.org/2005/xbrldt-2005.xsd#hypercube-dimension"))) lbTreeWalk(lbType, lbElt, lbLB, roleRefs) firstLinkElt = None for firstLinkElt in lbElt.iterchildren(): break # add arcrole references for roleref, roleURI, href in roleRefs: XmlUtil.addChild(lbElt, XbrlConst.link, roleref, attributes=(("arcroleURI" if roleref == "arcroleRef" else "roleURI", roleURI), ("{http://www.w3.org/1999/xlink}type", "simple"), ("{http://www.w3.org/1999/xlink}href", href)), beforeSibling=firstLinkElt) lbDoc.linkbaseDiscover(lbElt) break #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") return dts
def generateUpdatedTableLB(dts, updatedTableLinkbaseFile): import os, io from arelle import XmlUtil, XbrlConst from arelle.ViewUtil import viewReferences, referenceURI from arelle.ModelRenderingObject import ModelEuAxisCoord if dts.fileSource.isArchive: dts.error( "genTblLB:outFileIsArchive", _("Updated Table Linkbase file cannot be an archive: %(tableLBOutputFile)s." ), modelObject=dts, tableLBOutputFile=updatedTableLinkbaseFile) return tblAxisRelSet = dts.relationshipSet(XbrlConst.euTableAxis) axisMbrRelSet = dts.relationshipSet(XbrlConst.euAxisMember) if len(tblAxisRelSet.modelRelationships) == 0: dts.error( "genTblLB:noInputTables", _("DTS does not contain Eurofiling 2010 tables and axes: %(entryFile)s." ), modelObject=dts, entryFile=dts.uri) return file = io.StringIO(''' <nsmap> <link:linkbase xmlns:label="http://xbrl.org/2008/label" xmlns:gen="http://xbrl.org/2008/generic" xmlns:df="http://xbrl.org/2008/filter/dimension" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:reference="http://xbrl.org/2008/reference" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:link="http://www.xbrl.org/2003/linkbase" xmlns:table="http://xbrl.org/2011/table" xmlns:formula="http://xbrl.org/2008/formula" xsi:schemaLocation=" http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd http://xbrl.org/2008/generic http://www.xbrl.org/2008/generic-link.xsd http://xbrl.org/2008/reference http://www.xbrl.org/2008/generic-reference.xsd http://xbrl.org/2008/label http://www.xbrl.org/2008/generic-label.xsd http://xbrl.org/2011/table http://www.xbrl.org/2011/table.xsd http://xbrl.org/2008/filter/dimension http://www.xbrl.org/2008/dimension-filter.xsd"> <link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/table-filter" xlink:type="simple" xlink:href="http://www.xbrl.org/2011/table.xsd#table-filter"/> <link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/table-axis" xlink:type="simple" xlink:href="http://www.xbrl.org/2011/table.xsd#table-axis"/> <link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/axis-subtree" xlink:type="simple" xlink:href="http://www.xbrl.org/2011/table.xsd#axis-subtree"/> <link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/axis-filter" xlink:type="simple" xlink:href="http://www.xbrl.org/2011/filter-axis.xsd#axis-filter"/> </link:linkbase> </nsmap> <!-- Generated by Arelle(r) http://arelle.org --> ''') from arelle.ModelObjectFactory import parser parser, parserLookupName, parserLookupClass = parser(dts, None) from lxml import etree xmlDocument = etree.parse(file, parser=parser, base_url=updatedTableLinkbaseFile) file.close() nsmapElt = xmlDocument.getroot() #xmlDocument.getroot().init(self) ## is this needed ?? for lbElement in xmlDocument.iter( tag="{http://www.xbrl.org/2003/linkbase}linkbase"): break class DocObj: # fake ModelDocument for namespaces def __init__(self): self.xmlRootElement = lbElement self.xmlDocument = xmlDocument docObj = DocObj() numELRs = 0 numTables = 0 def copyAttrs(fromElt, toElt, attrTags): for attr in attrTags: if fromElt.get(attr): toElt.set(attr, fromElt.get(attr)) def generateTable(newLinkElt, newTblElt, srcTblElt, tblAxisRelSet, axisMbrRelSet, visited): if srcTblElt is not None: for rel in tblAxisRelSet.fromModelObject(srcTblElt): srcAxisElt = rel.toModelObject if isinstance(srcAxisElt, ModelEuAxisCoord): visited.add(srcAxisElt) newAxisElt = etree.SubElement( newLinkElt, "{http://xbrl.org/2011/table}ruleAxis") copyAttrs(srcAxisElt, newAxisElt, ("id", "{http://www.w3.org/1999/xlink}type", "{http://www.w3.org/1999/xlink}label")) newAxisElt.set("abstract", "true") # always true on root element newArcElt = etree.SubElement( newLinkElt, "{http://xbrl.org/2011/table}axisArc") copyAttrs(rel, newArcElt, ("id", "{http://www.w3.org/1999/xlink}type", "{http://www.w3.org/1999/xlink}from", "{http://www.w3.org/1999/xlink}to", "order")) newArcElt.set("{http://www.w3.org/1999/xlink}arcrole", XbrlConst.tableBreakdown) newArcElt.set("axisDisposition", rel.axisDisposition) generateAxis(newLinkElt, newAxisElt, srcAxisElt, axisMbrRelSet, visited) visited.discard(srcAxisElt) def generateAxis(newLinkElt, newAxisParentElt, srcAxisElt, axisMbrRelSet, visited): for rel in axisMbrRelSet.fromModelObject(srcAxisElt): tgtAxisElt = rel.toModelObject if isinstance(tgtAxisElt, ModelEuAxisCoord) and tgtAxisElt not in visited: visited.add(tgtAxisElt) newAxisElt = etree.SubElement( newLinkElt, "{http://xbrl.org/2011/table}ruleAxis") copyAttrs( tgtAxisElt, newAxisElt, ("id", "abstract", "{http://www.w3.org/1999/xlink}type", "{http://www.w3.org/1999/xlink}label")) if tgtAxisElt.primaryItemQname: newRuleElt = etree.SubElement( newAxisElt, "{http://xbrl.org/2008/formula}concept") newQnameElt = etree.SubElement( newRuleElt, "{http://xbrl.org/2008/formula}qname") newQnameElt.text = XmlUtil.addQnameValue( docObj, tgtAxisElt.primaryItemQname) for dimQname, memQname in tgtAxisElt.explicitDims: newRuleElt = etree.SubElement( newAxisElt, "{http://xbrl.org/2008/formula}explicitDimension") newRuleElt.set("dimension", XmlUtil.addQnameValue(docObj, dimQname)) newMbrElt = etree.SubElement( newRuleElt, "{http://xbrl.org/2008/formula}member") newQnameElt = etree.SubElement( newMbrElt, "{http://xbrl.org/2008/formula}qname") newQnameElt.text = XmlUtil.addQnameValue(docObj, memQname) newArcElt = etree.SubElement( newLinkElt, "{http://xbrl.org/2011/table}axisArc") copyAttrs(rel, newArcElt, ("id", "{http://www.w3.org/1999/xlink}type", "{http://www.w3.org/1999/xlink}from", "{http://www.w3.org/1999/xlink}to", "order")) newArcElt.set("{http://www.w3.org/1999/xlink}arcrole", XbrlConst.tableAxisSubtree) generateAxis(newLinkElt, newAxisElt, tgtAxisElt, axisMbrRelSet, visited) visited.discard(tgtAxisElt) # sort URIs linkroleUris = sorted( [linkroleUri for linkroleUri in tblAxisRelSet.linkRoleUris]) firstNewLinkElt = None roleRefUris = set() for linkroleUri in linkroleUris: numELRs += 1 newLinkElt = etree.SubElement(lbElement, "{http://xbrl.org/2008/generic}link") newLinkElt.set("{http://www.w3.org/1999/xlink}type", "extended") newLinkElt.set("{http://www.w3.org/1999/xlink}role", linkroleUri) if firstNewLinkElt is None: firstNewLinkElt = newLinkElt # To do: add roleRef if needed tblAxisRelSet = dts.relationshipSet(XbrlConst.euTableAxis, linkroleUri) axisMbrRelSet = dts.relationshipSet(XbrlConst.euAxisMember, linkroleUri) for srcTblElt in tblAxisRelSet.rootConcepts: if srcTblElt.tag == "{http://www.eurofiling.info/2010/rendering}table": numTables += 1 newTblElt = etree.SubElement( newLinkElt, "{http://xbrl.org/2011/table}table") newTblElt.set("aspectModel", "dimensional") copyAttrs(srcTblElt, newTblElt, ("id", "{http://www.w3.org/1999/xlink}type", "{http://www.w3.org/1999/xlink}label")) generateTable(newLinkElt, newTblElt, srcTblElt, tblAxisRelSet, axisMbrRelSet, set()) if linkroleUri not in roleRefUris: srcRoleRefElt = XmlUtil.descendant(srcTblElt.getroottree(), XbrlConst.link, "roleRef", "roleURI", linkroleUri) if srcRoleRefElt is not None: roleRefUris.add(linkroleUri) newRoleRefElt = etree.Element( "{http://www.xbrl.org/2003/linkbase}roleRef") copyAttrs( srcRoleRefElt, newRoleRefElt, ("roleURI", "{http://www.w3.org/1999/xlink}type", "{http://www.w3.org/1999/xlink}href")) firstNewLinkElt.addprevious(newRoleRefElt) fh = open(updatedTableLinkbaseFile, "w", encoding="utf-8") XmlUtil.writexml(fh, xmlDocument, encoding="utf-8") fh.close() dts.info( "info:updateTableLinkbase", _("Updated Table Linkbase of %(entryFile)s has %(numberOfLinkroles)s linkroles, %(numberOfTables)s tables in file %(tableLBOutputFile)s." ), modelObject=dts, entryFile=dts.uri, numberOfLinkroles=numELRs, numberOfTables=numTables, tableLBOutputFile=updatedTableLinkbaseFile)
def sphinxTestcaseVariationReadMeFirstUris(modelTestcaseVariation): xbrlElement = XmlUtil.descendant(modelTestcaseVariation, 'http://www.corefiling.com/sphinx-conformance-harness/2.0', "xbrl") if xbrlElement is not None: modelTestcaseVariation._readMeFirstUris.append(xbrlElement.elementText) return True # found it return False # not a sphinx test case variation
def fileNumber(self): return XmlUtil.text(XmlUtil.descendant(self, edgr, "fileNumber"))
def versioningReportUri(self): return XmlUtil.text(XmlUtil.descendant(self, None, "versioningReport"))
def sphinxTestcaseVariationExpectedSeverity(modelTestcaseVariation): issueElement = XmlUtil.descendant(modelTestcaseVariation, 'http://www.corefiling.com/sphinx-conformance-harness/2.0', "issue") if issueElement is not None: return issueElement.get("severity") return None # no issue or not a sphinx test case variation
def resultIsInfoset(self): if self.modelDocument.outpath: result = XmlUtil.descendant(self, None, "result") if result is not None: return XmlUtil.child(result, None, "file") is not None or XmlUtil.text(result).endswith(".xml") return False
def checkFilingDTS(val, modelDocument, isEFM, isGFM, visited): global targetNamespaceDatePattern, efmFilenamePattern, htmlFileNamePattern, roleTypePattern, arcroleTypePattern, \ arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet, \ namespacesConflictPattern if targetNamespaceDatePattern is None: targetNamespaceDatePattern = re.compile( r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|" r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|") efmFilenamePattern = re.compile( r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml|\.htm)$") htmlFileNamePattern = re.compile( r"^[a-zA-Z0-9][._a-zA-Z0-9-]*(\.htm)$") roleTypePattern = re.compile(r"^.*/role/[^/\s]+$") arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$") arcroleDefinitionPattern = re.compile( r"^.*[^\\s]+.*$") # at least one non-whitespace character namePattern = re.compile( "[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]" ) # u20ac=Euro, u00a3=pound sterling linkroleDefinitionBalanceIncomeSheet = re.compile( r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)", re.IGNORECASE) namespacesConflictPattern = re.compile( r"http://(xbrl\.us|fasb\.org|xbrl\.sec\.gov)/(dei|us-types|us-roles|rr)/([0-9]{4}-[0-9]{2}-[0-9]{2})$" ) nonDomainItemNameProblemPattern = re.compile( r"({0})|(FirstQuarter|SecondQuarter|ThirdQuarter|FourthQuarter|[1-4]Qtr|Qtr[1-4]|ytd|YTD|HalfYear)(?:$|[A-Z\W])" .format(re.sub(r"\W", "", (val.entityRegistrantName or "").title()))) visited.append(modelDocument) for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items( ): #6.07.01 no includes if modelDocumentReference.referenceType == "include": val.modelXbrl.error( ("EFM.6.07.01", "GFM.1.03.01"), _("Taxonomy schema %(schema)s includes %(include)s, only import is allowed" ), modelObject=modelDocumentReference.referringModelObject, schema=os.path.basename(modelDocument.uri), include=os.path.basename(referencedDocument.uri)) if referencedDocument not in visited and referencedDocument.inDTS: # ignore EdgarRenderer added non-DTS documents checkFilingDTS(val, referencedDocument, isEFM, isGFM, visited) if val.disclosureSystem.standardTaxonomiesDict is None: pass if isEFM: if modelDocument.uri in val.disclosureSystem.standardTaxonomiesDict: if modelDocument.targetNamespace: # check for duplicates of us-types, dei, and rr taxonomies match = namespacesConflictPattern.match( modelDocument.targetNamespace) if match is not None: val.standardNamespaceConflicts[match.group(2)].add( modelDocument) else: if len(modelDocument.basename) > 32: val.modelXbrl.error( "EFM.5.01.01.tooManyCharacters", _("Document file name %(filename)s must not exceed 32 characters." ), modelObject=modelDocument, filename=modelDocument.basename) if modelDocument.type == ModelDocument.Type.INLINEXBRL: if not htmlFileNamePattern.match(modelDocument.basename): val.modelXbrl.error( "EFM.5.01.01", _("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with .htm." ), modelObject=modelDocument, filename=modelDocument.basename) elif not efmFilenamePattern.match(modelDocument.basename): val.modelXbrl.error( "EFM.5.01.01", _("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with .xsd or .xml." ), modelObject=modelDocument, filename=modelDocument.basename) if (modelDocument.type == ModelDocument.Type.SCHEMA and modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and modelDocument.uri.startswith(val.modelXbrl.uriDir)): val.hasExtensionSchema = True # check schema contents types # 6.7.3 check namespace for standard authority targetNamespaceAuthority = UrlUtil.authority( modelDocument.targetNamespace) if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities: val.modelXbrl.error( ("EFM.6.07.03", "GFM.1.03.03"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s is a disallowed authority" ), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, targetNamespaceAuthority=UrlUtil.authority( modelDocument.targetNamespace, includeScheme=False)) # 6.7.4 check namespace format if modelDocument.targetNamespace is None or not modelDocument.targetNamespace.startswith( "http://"): match = None else: targetNamespaceDate = modelDocument.targetNamespace[ len(targetNamespaceAuthority):] match = targetNamespaceDatePattern.match(targetNamespaceDate) if match is not None: try: if match.lastindex == 3: date = datetime.date(int(match.group(1)), int(match.group(2)), int(match.group(3))) elif match.lastindex == 6: date = datetime.date(int(match.group(4)), int(match.group(5)), int(match.group(6))) else: match = None except ValueError: match = None if match is None: val.modelXbrl.error( ("EFM.6.07.04", "GFM.1.03.04"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must have format http://{authority}/{versionDate}" ), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) elif val.fileNameDate and date > val.fileNameDate: val.modelXbrl.info( ("EFM.6.07.06", "GFM.1.03.06"), _("Warning: Taxonomy schema %(schema)s namespace %(targetNamespace)s has date later than document name date %(docNameDate)s" ), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, docNameDate=val.fileNameDate) if modelDocument.targetNamespace is not None: # 6.7.5 check prefix for _ authority = UrlUtil.authority(modelDocument.targetNamespace) if not re.match(r"(http://|https://|ftp://|urn:)\w+", authority): val.modelXbrl.error( ("EFM.6.07.05", "GFM.1.03.05"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must be a valid URL with a valid authority for the namespace." ), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement, modelDocument.targetNamespace) if not prefix: val.modelXbrl.error( ("EFM.6.07.07", "GFM.1.03.07"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s missing prefix for the namespace." ), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) elif "_" in prefix: val.modelXbrl.error( ("EFM.6.07.07", "GFM.1.03.07"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s prefix %(prefix)s must not have an '_'" ), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, prefix=prefix) for modelConcept in modelDocument.xmlRootElement.iterdescendants( tag="{http://www.w3.org/2001/XMLSchema}element"): if isinstance(modelConcept, ModelConcept): # 6.7.16 name not duplicated in standard taxonomies name = modelConcept.get("name") if name is None: name = "" if modelConcept.get("ref") is not None: continue # don't validate ref's here for c in val.modelXbrl.nameConcepts.get(name, []): if c.modelDocument != modelDocument: if not c.modelDocument.uri.startswith( val.modelXbrl.uriDir): val.modelXbrl.error( ("EFM.6.07.16", "GFM.1.03.18"), _("Concept %(concept)s is also defined in standard taxonomy schema schema %(standardSchema)s" ), modelObject=(modelConcept, c), concept=modelConcept.qname, standardSchema=os.path.basename( c.modelDocument.uri), standardConcept=c.qname) # 6.7.17 id properly formed _id = modelConcept.id requiredId = (prefix if prefix is not None else "") + "_" + name if _id != requiredId: val.modelXbrl.error( ("EFM.6.07.17", "GFM.1.03.19"), _("Concept %(concept)s id %(id)s should be %(requiredId)s" ), modelObject=modelConcept, concept=modelConcept.qname, id=_id, requiredId=requiredId) # 6.7.18 nillable is true nillable = modelConcept.get("nillable") if nillable != "true" and modelConcept.isItem: val.modelXbrl.error( ("EFM.6.07.18", "GFM.1.03.20"), _("Taxonomy schema %(schema)s element %(concept)s nillable %(nillable)s should be 'true'" ), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=name, nillable=nillable) # 6.7.19 not tuple if modelConcept.isTuple: val.modelXbrl.error( ("EFM.6.07.19", "GFM.1.03.21"), _("Concept %(concept)s is a tuple"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.20 no typed domain ref if modelConcept.isTypedDimension: val.modelXbrl.error( ("EFM.6.07.20", "GFM.1.03.22"), _("Concept %(concept)s has typedDomainRef %(typedDomainRef)s" ), modelObject=modelConcept, concept=modelConcept.qname, typedDomainRef=modelConcept.typedDomainElement. qname if modelConcept.typedDomainElement is not None else modelConcept.typedDomainRef) # 6.7.21 abstract must be duration isDuration = modelConcept.periodType == "duration" if modelConcept.isAbstract and not isDuration: val.modelXbrl.error( ("EFM.6.07.21", "GFM.1.03.23"), _("Taxonomy schema %(schema)s element %(concept)s is abstract but period type is not duration" ), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname) # 6.7.22 abstract must be stringItemType ''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08 if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType: val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"), _("Concept %(concept)s is abstract but type is not xbrli:stringItemType"), modelObject=modelConcept, concept=modelConcept.qname) ''' substitutionGroupQname = modelConcept.substitutionGroupQname # 6.7.23 Axis must be subs group dimension if name.endswith("Axis") ^ ( substitutionGroupQname == XbrlConst.qnXbrldtDimensionItem): val.modelXbrl.error( ("EFM.6.07.23", "GFM.1.03.25"), _("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.24 Table must be subs group hypercube if name.endswith("Table") ^ ( substitutionGroupQname == XbrlConst.qnXbrldtHypercubeItem): val.modelXbrl.error( ("EFM.6.07.24", "GFM.1.03.26"), _("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group" ), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname) # 6.7.25 if neither hypercube or dimension, substitution group must be item if substitutionGroupQname not in ( None, XbrlConst.qnXbrldtDimensionItem, XbrlConst.qnXbrldtHypercubeItem, XbrlConst.qnXbrliItem): val.modelXbrl.error( ("EFM.6.07.25", "GFM.1.03.27"), _("Concept %(concept)s has disallowed substitution group %(substitutionGroup)s" ), modelObject=modelConcept, concept=modelConcept.qname, substitutionGroup=modelConcept. substitutionGroupQname) # 6.7.26 Table must be subs group hypercube if name.endswith( "LineItems") and modelConcept.abstract != "true": val.modelXbrl.error( ("EFM.6.07.26", "GFM.1.03.28"), _("Concept %(concept)s is a LineItems but not abstract" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.27 type domainMember must end with Domain or Member conceptType = modelConcept.type isDomainItemType = conceptType is not None and conceptType.isDomainItemType endsWithDomainOrMember = name.endswith( "Domain") or name.endswith("Member") if isDomainItemType != endsWithDomainOrMember: val.modelXbrl.error( ("EFM.6.07.27", "GFM.1.03.29"), _("Concept %(concept)s must end with Domain or Member for type of domainItemType" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.28 domainItemType must be duration if isDomainItemType and not isDuration: val.modelXbrl.error( ("EFM.6.07.28", "GFM.1.03.30"), _("Concept %(concept)s is a domainItemType and must be periodType duration" ), modelObject=modelConcept, concept=modelConcept.qname) #6.7.31 (version 27) fractions if modelConcept.isFraction: val.modelXbrl.error( "EFM.6.07.31", _("Concept %(concept)s is a fraction"), modelObject=modelConcept, concept=modelConcept.qname) #6.7.32 (version 27) instant non numeric if modelConcept.isItem and (not modelConcept.isNumeric and not isDuration and not modelConcept.isAbstract and not isDomainItemType): val.modelXbrl.error( "EFM.6.07.32", _("Taxonomy schema %(schema)s element %(concept)s is non-numeric but period type is not duration" ), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname) # 6.8.5 semantic check, check LC3 name if name: if not name[0].isupper(): val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.05.firstLetter", "GFM.2.03.05.firstLetter"), _("Concept %(concept)s name must start with a capital letter" ), modelObject=modelConcept, concept=modelConcept.qname) if namePattern.search(name): val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.05.disallowedCharacter", "GFM.2.03.05.disallowedCharacter"), _("Concept %(concept)s has disallowed name character" ), modelObject=modelConcept, concept=modelConcept.qname) if len(name) > 200: val.modelXbrl.log( "ERROR-SEMANTIC", "EFM.6.08.05.nameLength", _("Concept %(concept)s name length %(namelength)s exceeds 200 characters" ), modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name)) if isEFM: label = modelConcept.label(lang="en-US", fallbackToQname=False) if label: # allow Joe's Bar, N.A. to be JoesBarNA -- remove ', allow A. as not article "a" lc3name = ''.join( re.sub(r"['.-]", "", ( w[0] or w[2] or w[3] or w[4])).title() for w in re.findall( r"((\w+')+\w+)|(A[.-])|([.-]A(?=\W|$))|(\w+)", label ) # EFM implies this should allow - and . re.findall(r"[\w\-\.]+", label) if w[4].lower() not in ("the", "a", "an")) if not (name == lc3name or (name and lc3name and lc3name[0].isdigit() and name[1:] == lc3name and (name[0].isalpha() or name[0] == '_'))): val.modelXbrl.log( "WARNING-SEMANTIC", "EFM.6.08.05.LC3", _("Concept %(concept)s should match expected LC3 composition %(lc3name)s" ), modelObject=modelConcept, concept=modelConcept.qname, lc3name=lc3name) if conceptType is not None: # 6.8.6 semantic check if not isDomainItemType and conceptType.qname != XbrlConst.qnXbrliDurationItemType: nameProblems = nonDomainItemNameProblemPattern.findall( name) if any( any(t) for t in nameProblems ): # list of tuples with possibly nonempty strings val.modelXbrl.log( "WARNING-SEMANTIC", ("EFM.6.08.06", "GFM.2.03.06"), _("Concept %(concept)s should not contain company or period information, found: %(matches)s" ), modelObject=modelConcept, concept=modelConcept.qname, matches=", ".join(''.join(t) for t in nameProblems)) if conceptType.qname == XbrlConst.qnXbrliMonetaryItemType: if not modelConcept.balance: # 6.8.11 may not appear on a income or balance statement if any( linkroleDefinitionBalanceIncomeSheet. match(roleType.definition) for rel in val.modelXbrl. relationshipSet(XbrlConst.parentChild). toModelObject(modelConcept) for roleType in val.modelXbrl. roleTypes.get(rel.linkrole, ())): val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.11", "GFM.2.03.11"), _("Concept %(concept)s must have a balance because it appears in a statement of income or balance sheet" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.11.5 semantic check, must have a documentation label stdLabel = modelConcept.label( lang="en-US", fallbackToQname=False) defLabel = modelConcept.label( preferredLabel=XbrlConst. documentationLabel, lang="en-US", fallbackToQname=False) if not defLabel or ( # want different words than std label stdLabel and re.findall(r"\w+", stdLabel) == re.findall(r"\w+", defLabel)): val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.11.05", "GFM.2.04.04"), _("Concept %(concept)s is monetary without a balance and must have a documentation label that disambiguates its sign" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.8.16 semantic check if conceptType.qname == XbrlConst.qnXbrliDateItemType and modelConcept.periodType != "duration": val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.16", "GFM.2.03.16"), _("Concept %(concept)s of type xbrli:dateItemType must have periodType duration" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.8.17 semantic check if conceptType.qname == XbrlConst.qnXbrliStringItemType and modelConcept.periodType != "duration": val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.17", "GFM.2.03.17"), _("Concept %(concept)s of type xbrli:stringItemType must have periodType duration" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.8 check for embedded linkbase for e in modelDocument.xmlRootElement.iterdescendants( tag="{http://www.xbrl.org/2003/linkbase}linkbase"): if isinstance(e, ModelObject): val.modelXbrl.error( ("EFM.6.07.08", "GFM.1.03.08"), _("Taxonomy schema %(schema)s contains an embedded linkbase" ), modelObject=e, schema=modelDocument.basename) break requiredUsedOns = { XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink, XbrlConst.qnLinkDefinitionLink } standardUsedOns = { XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference, XbrlConst.qnLinkDefinitionArc, XbrlConst.qnLinkCalculationArc, XbrlConst.qnLinkPresentationArc, XbrlConst.qnLinkLabelArc, XbrlConst.qnLinkReferenceArc, # per WH, private footnote arc and footnore resource roles are not allowed XbrlConst.qnLinkFootnoteArc, XbrlConst.qnLinkFootnote, } # 6.7.9 role types authority for e in modelDocument.xmlRootElement.iterdescendants( tag="{http://www.xbrl.org/2003/linkbase}roleType"): if isinstance(e, ModelObject): roleURI = e.get("roleURI") if targetNamespaceAuthority != UrlUtil.authority(roleURI): val.modelXbrl.error( ("EFM.6.07.09", "GFM.1.03.09"), _("RoleType %(roleType)s does not match authority %(targetNamespaceAuthority)s" ), modelObject=e, roleType=roleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace) # 6.7.9 end with .../role/lc3 name if not roleTypePattern.match(roleURI): val.modelXbrl.warning( ("EFM.6.07.09.roleEnding", "GFM.1.03.09"), "RoleType %(roleType)s should end with /role/{LC3name}", modelObject=e, roleType=roleURI) # 6.7.10 only one role type declaration in DTS modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI) if modelRoleTypes is not None: modelRoleType = modelRoleTypes[0] definition = modelRoleType.definitionNotStripped usedOns = modelRoleType.usedOns if len(modelRoleTypes) == 1: # 6.7.11 used on's for pre, cal, def if any has a used on if not usedOns.isdisjoint(requiredUsedOns) and len( requiredUsedOns - usedOns) > 0: val.modelXbrl.error( ("EFM.6.07.11", "GFM.1.03.11"), _("RoleType %(roleType)s missing used on %(usedOn)s" ), modelObject=e, roleType=roleURI, usedOn=requiredUsedOns - usedOns) # 6.7.12 definition match pattern if (val.disclosureSystem.roleDefinitionPattern is not None and (definition is None or not val.disclosureSystem. roleDefinitionPattern.match(definition))): val.modelXbrl.error( ("EFM.6.07.12", "GFM.1.03.12-14"), _("RoleType %(roleType)s definition \"%(definition)s\" must match {Sortcode} - {Type} - {Title}" ), modelObject=e, roleType=roleURI, definition=(definition or "")) if usedOns & standardUsedOns: # semantics check val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"), _("RoleType %(roleuri)s is defined using role types already defined by standard roles for: %(qnames)s" ), modelObject=e, roleuri=roleURI, qnames=', '.join( str(qn) for qn in usedOns & standardUsedOns)) # 6.7.13 arcrole types authority for e in modelDocument.xmlRootElement.iterdescendants( tag="{http://www.xbrl.org/2003/linkbase}arcroleType"): if isinstance(e, ModelObject): arcroleURI = e.get("arcroleURI") if targetNamespaceAuthority != UrlUtil.authority(arcroleURI): val.modelXbrl.error( ("EFM.6.07.13", "GFM.1.03.15"), _("ArcroleType %(arcroleType)s does not match authority %(targetNamespaceAuthority)s" ), modelObject=e, arcroleType=arcroleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace) # 6.7.13 end with .../arcrole/lc3 name if not arcroleTypePattern.match(arcroleURI): val.modelXbrl.warning( ("EFM.6.07.13.arcroleEnding", "GFM.1.03.15"), _("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}" ), modelObject=e, arcroleType=arcroleURI) # 6.7.15 definition match pattern modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI] definition = modelRoleTypes[0].definition if definition is None or not arcroleDefinitionPattern.match( definition): val.modelXbrl.error( ("EFM.6.07.15", "GFM.1.03.17"), _("ArcroleType %(arcroleType)s definition must be non-empty" ), modelObject=e, arcroleType=arcroleURI) # semantic checks usedOns = modelRoleTypes[0].usedOns if usedOns & standardUsedOns: # semantics check val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"), _("ArcroleType %(arcroleuri)s is defined using role types already defined by standard arcroles for: %(qnames)s" ), modelObject=e, arcroleuri=arcroleURI, qnames=', '.join( str(qn) for qn in usedOns & standardUsedOns)) #6.3.3 filename check m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9]).xsd$", modelDocument.basename) if m: try: # check date value datetime.datetime.strptime(m.group(1), "%Y%m%d").date() # date and format are ok, check "should" part of 6.3.3 if val.fileNameBasePart: expectedFilename = "{0}-{1}.xsd".format( val.fileNameBasePart, val.fileNameDatePart) if modelDocument.basename != expectedFilename: val.modelXbrl.log( "WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"), _('Schema file name warning: %(filename)s, should match %(expectedFilename)s' ), modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename) except ValueError: val.modelXbrl.error( (val.EFM60303, "GFM.1.01.01"), _('Invalid schema file base name part (date) in "{base}-{yyyymmdd}.xsd": %(filename)s' ), modelObject=modelDocument, filename=modelDocument.basename, messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01")) else: val.modelXbrl.error( (val.EFM60303, "GFM.1.01.01"), _('Invalid schema file name, must match "{base}-{yyyymmdd}.xsd": %(filename)s' ), modelObject=modelDocument, filename=modelDocument.basename, messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01")) elif modelDocument.type == ModelDocument.Type.LINKBASE: # if it is part of the submission (in same directory) check name labelRels = None if modelDocument.filepath.startswith( val.modelXbrl.modelDocument.filepathdir): #6.3.3 filename check extLinkElt = XmlUtil.descendant( modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended") if extLinkElt is None: # no ext link element val.modelXbrl.error( (val.EFM60303 + ".noLinkElement", "GFM.1.01.01.noLinkElement"), _('Invalid linkbase file name: %(filename)s, has no extended link element, cannot determine link type.' ), modelObject=modelDocument, filename=modelDocument.basename, messageCodes=("EFM.6.03.03.noLinkElement", "EFM.6.23.01.noLinkElement", "GFM.1.01.01.noLinkElement")) elif extLinkElt.localName not in extLinkEltFileNameEnding: val.modelXbrl.error( "EFM.6.03.02", _('Invalid linkbase link element %(linkElement)s in %(filename)s' ), modelObject=modelDocument, linkElement=extLinkElt.localName, filename=modelDocument.basename) else: m = re.match( r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9])(_[a-z]{3}).xml$", modelDocument.basename) expectedSuffix = extLinkEltFileNameEnding[extLinkElt.localName] if m and m.group(2) == expectedSuffix: try: # check date value datetime.datetime.strptime(m.group(1), "%Y%m%d").date() # date and format are ok, check "should" part of 6.3.3 if val.fileNameBasePart: expectedFilename = "{0}-{1}{2}.xml".format( val.fileNameBasePart, val.fileNameDatePart, expectedSuffix) if modelDocument.basename != expectedFilename: val.modelXbrl.log( "WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"), _('Linkbase name warning: %(filename)s should match %(expectedFilename)s' ), modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename) except ValueError: val.modelXbrl.error( (val.EFM60303, "GFM.1.01.01"), _('Invalid linkbase base file name part (date) in "{base}-{yyyymmdd}_{suffix}.xml": %(filename)s' ), modelObject=modelDocument, filename=modelDocument.basename, messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01")) else: val.modelXbrl.error( (val.EFM60303, "GFM.1.01.01"), _('Invalid linkbase name, must match "{base}-{yyyymmdd}%(expectedSuffix)s.xml": %(filename)s' ), modelObject=modelDocument, filename=modelDocument.basename, expectedSuffix=expectedSuffix, messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01")) if extLinkElt.localName == "labelLink": if labelRels is None: labelRels = val.modelXbrl.relationshipSet( XbrlConst.conceptLabel) for labelElt in XmlUtil.children(extLinkElt, XbrlConst.link, "label"): # 6.10.9 if XbrlConst.isNumericRole(labelElt.role): for rel in labelRels.toModelObject(labelElt): if rel.fromModelObject is not None and not rel.fromModelObject.isNumeric: val.modelXbrl.error( "EFM.6.10.09", _("Label of non-numeric concept %(concept)s has a numeric role: %(role)s" ), modelObject=(labelElt, rel.fromModelObject), concept=rel.fromModelObject.qname, role=labelElt.role)
def expectedVersioningReport(self): XmlUtil.text(XmlUtil.text(XmlUtil.descendant(XmlUtil.descendant(self, None, "result"), None, "versioningReport")))
def sphinxTestcaseVariationReadMeFirstUris(modelTestcaseVariation, *args, **kwargs): xbrlElement = XmlUtil.descendant(modelTestcaseVariation, 'http://www.corefiling.com/sphinx-conformance-harness/2.0', "xbrl") if xbrlElement is not None: modelTestcaseVariation._readMeFirstUris.append(xbrlElement.textValue) return True # found it return False # not a sphinx test case variation
def checkDTSdocument(val, modelDocument, *args, **kwargs): modelXbrl = val.modelXbrl if modelDocument.type in (ModelDocument.Type.SCHEMA, ModelDocument.Type.LINKBASE): isSchema = modelDocument.type == ModelDocument.Type.SCHEMA docinfo = modelDocument.xmlDocument.docinfo if docinfo and docinfo.xml_version != "1.0": modelXbrl.error("SBR.NL.2.2.0.02" if isSchema else "SBR.NL.2.3.0.02", _('%(docType)s xml version must be "1.0" but is "%(xmlVersion)s"'), modelObject=modelDocument, docType=modelDocument.gettype().title(), xmlVersion=docinfo.xml_version) if modelDocument.documentEncoding.lower() != "utf-8": modelXbrl.error("SBR.NL.2.2.0.03" if isSchema else "SBR.NL.2.3.0.03", _('%(docType)s encoding must be "utf-8" but is "%(xmlEncoding)s"'), modelObject=modelDocument, docType=modelDocument.gettype().title(), xmlEncoding=modelDocument.documentEncoding) lookingForPrecedingComment = True for commentNode in modelDocument.xmlRootElement.itersiblings(preceding=True): if isinstance(commentNode, etree._Comment): if lookingForPrecedingComment: lookingForPrecedingComment = False else: modelXbrl.error("SBR.NL.2.2.0.05" if isSchema else "SBR.NL.2.3.0.05", _('%(docType)s must have only one comment node before schema element'), modelObject=modelDocument, docType=modelDocument.gettype().title()) if lookingForPrecedingComment: modelXbrl.error("SBR.NL.2.2.0.04" if isSchema else "SBR.NL.2.3.0.04", _('%(docType)s must have comment node only on line 2'), modelObject=modelDocument, docType=modelDocument.gettype().title()) # check namespaces are used for prefix, ns in modelDocument.xmlRootElement.nsmap.items(): if ((prefix not in val.valUsedPrefixes) and (modelDocument.type != ModelDocument.Type.SCHEMA or ns != modelDocument.targetNamespace)): modelXbrl.error("SBR.NL.2.2.0.11" if modelDocument.type == ModelDocument.Type.SCHEMA else "SBR.NL.2.3.0.08", _('%(docType)s namespace declaration "%(declaration)s" is not used'), modelObject=modelDocument, docType=modelDocument.gettype().title(), declaration=("xmlns" + (":" + prefix if prefix else "") + "=" + ns)) if isSchema and val.annotationsCount > 1: modelXbrl.error("SBR.NL.2.2.0.22", _('Schema has %(annotationsCount)s xs:annotation elements, only 1 allowed'), modelObject=modelDocument, annotationsCount=val.annotationsCount) if modelDocument.type == ModelDocument.Type.LINKBASE: if not val.containsRelationship: modelXbrl.error("SBR.NL.2.3.0.12", "Linkbase has no relationships", modelObject=modelDocument) # check file name suffixes extLinkElt = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended") if extLinkElt is not None: expectedSuffix = None if extLinkElt.localName == "labelLink": anyLabel = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/XML/1998/namespace}lang", "*") if anyLabel is not None: xmlLang = anyLabel.get("{http://www.w3.org/XML/1998/namespace}lang") expectedSuffix = "-lab-{0}.xml".format(xmlLang) else: expectedSuffix = {"referenceLink": "-ref.xml", "presentationLink": "-pre.xml", "definitionLink": "-def.xml"}.get(extLinkElt.localName, None) if expectedSuffix: if not modelDocument.uri.endswith(expectedSuffix): modelXbrl.error("SBR.NL.3.2.1.09", "Linkbase filename expected to end with %(expectedSuffix)s: %(filename)s", modelObject=modelDocument, expectedSuffix=expectedSuffix, filename=modelDocument.uri) elif extLinkElt.qname == XbrlConst.qnGenLink: anyLabel = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/XML/1998/namespace}lang", "*") if anyLabel is not None: xmlLang = anyLabel.get("{http://www.w3.org/XML/1998/namespace}lang") expectedSuffix = "-generic-lab-{0}.xml".format(xmlLang) elif XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "reference") is not None: expectedSuffix = "-generic-ref.xml" if expectedSuffix and not modelDocument.uri.endswith(expectedSuffix): modelXbrl.error("SBR.NL.3.2.1.10", "Generic linkbase filename expected to end with %(expectedSuffix)s: %(filename)s", modelObject=modelDocument, expectedSuffix=expectedSuffix, filename=modelDocument.uri) # label checks for qnLabel in (XbrlConst.qnLinkLabel, XbrlConst.qnGenLabel): for modelLabel in modelDocument.xmlRootElement.iterdescendants(tag=qnLabel.clarkNotation): if isinstance(modelLabel, ModelResource): if not modelLabel.text or not modelLabel.text[:1].isupper(): modelXbrl.error("SBR.NL.3.2.7.05", _("Labels MUST have a capital first letter, label %(label)s: %(text)s"), modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64]) if modelLabel.role in (XbrlConst.standardLabel, XbrlConst.genStandardLabel): if len(modelLabel.text) > 255: modelXbrl.error("SBR.NL.3.2.7.06", _("Labels with the 'standard' role MUST NOT exceed 255 characters, label %(label)s: %(text)s"), modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64]) if modelLabel.role in (XbrlConst.standardLabel, XbrlConst.genStandardLabel): if len(modelLabel.text) > 255: modelXbrl.error("SBR.NL.3.2.7.06", _("Labels with the 'standard' role MUST NOT exceed 255 characters, label %(label)s: %(text)s"), modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64]) for modelResource in modelDocument.xmlRootElement.iter(): # locator checks if isinstance(modelResource, ModelLocator): hrefModelObject = modelResource.dereference() if isinstance(hrefModelObject, ModelObject): expectedLocLabel = hrefModelObject.id + "_loc" if modelResource.xlinkLabel != expectedLocLabel: modelXbrl.error("SBR.NL.3.2.11.01", _("Locator @xlink:label names MUST be concatenated from: @id from the XML node, underscore, 'loc', expected %(expectedLocLabel)s, found %(foundLocLabel)s"), modelObject=modelResource, expectedLocLabel=expectedLocLabel, foundLocLabel=modelResource.xlinkLabel) # xlinkLabel checks if isinstance(modelResource, ModelResource): if re.match(r"[^a-zA-Z0-9_-]", modelResource.xlinkLabel): modelXbrl.error("SBR.NL.3.2.11.03", _("@xlink:label names MUST use a-zA-Z0-9_- characters only: %(xlinkLabel)s"), modelObject=modelResource, xlinkLabel=modelResource.xlinkLabel) elif modelDocument.targetNamespace: # SCHEMA with targetNamespace # check for unused imports for referencedDocument in modelDocument.referencesDocument.keys(): if (referencedDocument.type == ModelDocument.Type.SCHEMA and referencedDocument.targetNamespace not in {XbrlConst.xbrli, XbrlConst.link} and referencedDocument.targetNamespace not in val.referencedNamespaces): modelXbrl.error("SBR.NL.2.2.0.15", _("A schema import schemas of which no content is being addressed: %(importedFile)s"), modelObject=modelDocument, importedFile=referencedDocument.basename) if modelDocument.targetNamespace != modelDocument.targetNamespace.lower(): modelXbrl.error("SBR.NL.3.2.3.02", _("Namespace URI's MUST be lower case: %(namespaceURI)s"), modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace) if len(modelDocument.targetNamespace) > 255: modelXbrl.error("SBR.NL.3.2.3.03", _("Namespace URI's MUST NOT be longer than 255 characters: %(namespaceURI)s"), modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace) if re.match(r"[^a-z0-9_/-]", modelDocument.targetNamespace): modelXbrl.error("SBR.NL.3.2.3.04", _("Namespace URI's MUST use only signs from a-z0-9_-/: %(namespaceURI)s"), modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace) if not modelDocument.targetNamespace.startswith('http://www.nltaxonomie.nl'): modelXbrl.error("SBR.NL.3.2.3.05", _("Namespace URI's MUST start with 'http://www.nltaxonomie.nl': %(namespaceURI)s"), modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace) namespacePrefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement, modelDocument.targetNamespace) if not namespacePrefix: modelXbrl.error("SBR.NL.3.2.4.01", _("TargetNamespaces MUST have a prefix: %(namespaceURI)s"), modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace) elif namespacePrefix in val.prefixNamespace: modelXbrl.error("SBR.NL.3.2.4.02", _("Namespace prefix MUST be unique within the NT but prefix '%(prefix)s' is used by both %(namespaceURI)s and %(namespaceURI2)s."), modelObject=modelDocument, prefix=namespacePrefix, namespaceURI=modelDocument.targetNamespace, namespaceURI2=val.prefixNamespace[namespacePrefix]) else: val.prefixNamespace[namespacePrefix] = modelDocument.targetNamespace val.namespacePrefix[modelDocument.targetNamespace] = namespacePrefix if namespacePrefix in {"xsi", "xsd", "xs", "xbrli", "link", "xlink", "xbrldt", "xbrldi", "gen", "xl"}: modelXbrl.error("SBR.NL.3.2.4.03", _("Namespace prefix '%(prefix)s' reserved by organizations for international specifications is used %(namespaceURI)s."), modelObject=modelDocument, prefix=namespacePrefix, namespaceURI=modelDocument.targetNamespace) if len(namespacePrefix) > 20: modelXbrl.warning("SBR.NL.3.2.4.06", _("Namespace prefix '%(prefix)s' SHOULD not exceed 20 characters %(namespaceURI)s."), modelObject=modelDocument, prefix=namespacePrefix, namespaceURI=modelDocument.targetNamespace) # check every non-targetnamespace prefix against its schema requiredLinkrole = None # only set for extension taxonomies # check folder names if modelDocument.filepathdir.startswith(modelXbrl.uriDir): partnerPrefix = None pathDir = modelDocument.filepathdir[len(modelXbrl.uriDir) + 1:].replace("\\", "/") lastPathSegment = None for pathSegment in pathDir.split("/"): if pathSegment.lower() != pathSegment: modelXbrl.error("SBR.NL.3.2.1.02", _("Folder names must be in lower case: %(folder)s"), modelObject=modelDocument, folder=pathSegment) if len(pathSegment) >= 15 : modelXbrl.error("SBR.NL.3.2.1.03", _("Folder names must be less than 15 characters: %(folder)s"), modelObject=modelDocument, folder=pathSegment) if pathSegment in ("bd", "kvk", "cbs"): partnerPrefix = pathSegment + '-' lastPathSegment = pathSegment if modelDocument.basename.lower() != modelDocument.basename: modelXbrl.error("SBR.NL.3.2.1.05", _("File names must be in lower case: %(file)s"), modelObject=modelDocument, file=modelDocument.basename) if partnerPrefix and not modelDocument.basename.startswith(partnerPrefix): modelXbrl.error("SBR.NL.3.2.1.14", "NT Partner DTS files MUST start with %(partnerPrefix)s consistently: %(filename)s", modelObject=modelDocument, partnerPrefix=partnerPrefix, filename=modelDocument.uri) if modelDocument.type == ModelDocument.Type.SCHEMA: if modelDocument.targetNamespace: nsParts = modelDocument.targetNamespace.split("/") # [0] = https, [1] = // [2] = nl.taxonomie [3] = year or version nsYrOrVer = nsParts[3] requiredNamespace = "http://www.nltaxonomie.nl/" + nsYrOrVer + "/" + pathDir + "/" + modelDocument.basename[:-4] requiredLinkrole = "http://www.nltaxonomie.nl/" + nsYrOrVer + "/" + pathDir + "/" if modelDocument == modelXbrl.modelDocument: # entry point nsYr = "{year}" if '2009' <= nsParts[3] < '2020': # must be a year, use as year nsYr = nsParts[3] else: # look for year in parts of basename of required namespace for nsPart in nsParts: for baseNamePart in nsPart.split('-'): if '2009' <= baseNamePart < '2020': nsYr = baseNamePart break if not requiredNamespace.endswith('-' + nsYr): requiredNamespace += '-' + nsYr if not modelDocument.targetNamespace.startswith(requiredNamespace): modelXbrl.error("SBR.NL.3.2.3.06", _("Namespace URI's MUST be constructed like %(requiredNamespace)s: %(namespaceURI)s"), modelObject=modelDocument, requiredNamespace=requiredNamespace, namespaceURI=modelDocument.targetNamespace) else: requiredLinkrole = '' # concept checks for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"): if isinstance(modelConcept, ModelConcept): # 6.7.16 name not duplicated in standard taxonomies name = modelConcept.get("name") if name: ''' removed per RH 2013-03-25 substititutionGroupQname = modelConcept.substitutionGroupQname if substititutionGroupQname: if name.endswith("Member") ^ (substititutionGroupQname.localName == "domainMemberItem" and substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")): modelXbrl.error("SBR.NL.3.2.5.11", _("Concept %(concept)s must end in Member to be in sbr:domainMemberItem substitution group"), modelObject=modelConcept, concept=modelConcept.qname) if name.endswith("Domain") ^ (substititutionGroupQname.localName == "domainItem" and substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")): modelXbrl.error("SBR.NL.3.2.5.12", _("Concept %(concept)s must end in Domain to be in sbr:domainItem substitution group"), modelObject=modelConcept, concept=modelConcept.qname) if name.endswith("TypedAxis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem and modelConcept.isTypedDimension): modelXbrl.error("SBR.NL.3.2.5.14", _("Concept %(concept)s must end in TypedAxis to be in xbrldt:dimensionItem substitution group if they represent a typed dimension"), modelObject=modelConcept, concept=modelConcept.qname) if (name.endswith("Axis") and not name.endswith("TypedAxis")) ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem and modelConcept.isExplicitDimension): modelXbrl.error("SBR.NL.3.2.5.13", _("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group if they represent an explicit dimension"), modelObject=modelConcept, concept=modelConcept.qname) if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem): modelXbrl.error("SBR.NL.3.2.5.15", _("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"), modelObject=modelConcept, concept=modelConcept.qname) if name.endswith("Title") ^ (substititutionGroupQname.localName == "presentationItem" and substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")): modelXbrl.error("SBR.NL.3.2.5.16", _("Concept %(concept)s must end in Title to be in sbr:presentationItem substitution group"), modelObject=modelConcept, concept=modelConcept.qname) ''' if len(name) > 200: modelXbrl.error("SBR.NL.3.2.12.02" if modelConcept.isLinkPart else "SBR.NL.3.2.5.21" if (modelConcept.isItem or modelConcept.isTuple) else "SBR.NL.3.2.14.01", _("Concept %(concept)s name length %(namelength)s exceeds 200 characters"), modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name)) # type checks for typeType in ("simpleType", "complexType"): for modelType in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}" + typeType): if isinstance(modelType, ModelType): name = modelType.get("name") if name is None: name = "" if modelType.get("ref") is not None: continue # don't validate ref's here if len(name) > 200: modelXbrl.error("SBR.NL.3.2.5.21", _("Type %(type)s name length %(namelength)s exceeds 200 characters"), modelObject=modelType, type=modelType.qname, namelength=len(name)) if modelType.qnameDerivedFrom and modelType.qnameDerivedFrom.namespaceURI != XbrlConst.xbrli: modelXbrl.error("SBR.NL.3.2.8.01", _("Custom datatypes MUST be a restriction from XII defined datatypes: %(type)s"), modelObject=modelType, type=modelType.qname) if re.match(r"[^a-zA-Z0-9_-]", name): modelXbrl.error("SBR.NL.3.2.8.02", _("Datatype names MUST use characters a-zA-Z0-9_- only: %(type)s"), modelObject=modelDocument, type=modelType.qname) if modelType.facets and "enumeration" in modelType.facets: if not modelType.qnameDerivedFrom == XbrlConst.qnXbrliStringItemType: modelXbrl.error("SBR.NL.3.2.13.01", _("Enumerations MUST use a restriction on xbrli:stringItemType: %(type)s"), modelObject=modelDocument, type=modelType.qname) if lastPathSegment == "entrypoints": if not modelDocument.xmlRootElement.id: modelXbrl.error("SBR.NL.2.2.0.23", _("xs:schema/@id MUST be present in schema files in the reports/{NT partner}/entrypoints/ folder"), modelObject=modelDocument) # check for idObject conflicts for id, modelObject in modelDocument.idObjects.items(): if id in val.idObjects: modelXbrl.error("SBR.NL.3.2.6.01", _("ID %(id)s must be unique in the DTS but is present on two elements."), modelObject=(modelObject, val.idObjects[id]), id=id) else: val.idObjects[id] = modelObject for roleURI, modelRoleTypes in modelXbrl.roleTypes.items(): if not roleURI.startswith("http://www.xbrl.org"): usedOns = set.union(*[modelRoleType.usedOns for modelRoleType in modelRoleTypes]) # check roletypes for linkroles (only) if usedOns & {XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink, XbrlConst.qnLinkDefinitionLink, XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference, XbrlConst.qnLinkFootnote}: if len(modelRoleTypes) > 1: modelXbrl.error("SBR.NL.3.2.9.01", _("Linkrole URI's MUST be unique in the NT: %(linkrole)s"), modelObject=modelRoleTypes, linkrole=roleURI) if roleURI.lower() != roleURI: modelXbrl.error("SBR.NL.3.2.9.02", _("Linkrole URI's MUST be in lowercase: %(linkrole)s"), modelObject=modelRoleTypes, linkrole=roleURI) if re.match(r"[^a-z0-9_/-]", roleURI): modelXbrl.error("SBR.NL.3.2.9.03", _("Linkrole URI's MUST use characters a-z0-9_-/ only: %(linkrole)s"), modelObject=modelRoleTypes, linkrole=roleURI) if len(roleURI) > 255: modelXbrl.error("SBR.NL.3.2.9.04", _("Linkrole URI's MUST NOT be longer than 255 characters, length is %(len)s: %(linkrole)s"), modelObject=modelRoleTypes, len=len(roleURI), linkrole=roleURI) ''' removed per RH 2013-03-13 e-mail if not roleURI.startswith('http://www.nltaxonomie.nl'): modelXbrl.error("SBR.NL.3.2.9.05", _("Linkrole URI's MUST start with 'http://www.nltaxonomie.nl': %(linkrole)s"), modelObject=modelRoleTypes, linkrole=roleURI) if (requiredLinkrole and not roleURI.startswith(requiredLinkrole) and re.match(r".*(domain$|axis$|table$|lineitem$)", roleURI)): modelXbrl.error("SBR.NL.3.2.9.06", _("Linkrole URI's MUST have the following construct: http://www.nltaxonomie.nl / {folder path} / {functional name} - {domain or axis or table or lineitem}: %(linkrole)s"), modelObject=modelRoleTypes, linkrole=roleURI) ''' for modelRoleType in modelRoleTypes: if len(modelRoleType.id) > 255: modelXbrl.error("SBR.NL.3.2.10.02", _("Linkrole @id MUST NOT exceed 255 characters, length is %(length)s: %(linkroleID)s"), modelObject=modelRoleType, length=len(modelRoleType.id), linkroleID=modelRoleType.id) partnerPrefix = modelRoleTypes[0].modelDocument.basename.split('-') if partnerPrefix: # first element before dash is prefix urnPartnerLinkroleStart = "urn:{0}:linkrole:".format(partnerPrefix[0]) if not roleURI.startswith(urnPartnerLinkroleStart): modelXbrl.error("SBR.NL.3.2.9.10", _("Linkrole MUST start with urn:{NT partner code}:linkrole:, \nexpecting: %(expectedStart)s..., \nfound: %(linkrole)s"), modelObject=modelRoleType, expectedStart=urnPartnerLinkroleStart, linkrole=roleURI)
def dir(self): self.open() if not self.isOpen: return None elif self.filesDir is not None: return self.filesDir elif self.isZip: files = [] for zipinfo in self.fs.infolist(): files.append(zipinfo.filename) self.filesDir = files elif self.isTarGz: self.filesDir = self.fs.getnames() elif self.isEis: files = [] for docElt in self.eisDocument.iter( tag="{http://www.sec.gov/edgar/common}document"): outfn = docElt.findtext( "{http://www.sec.gov/edgar/common}conformedName") if outfn: files.append(outfn) self.filesDir = files elif self.isXfd: files = [] for data in self.xfdDocument.iter(tag="data"): outfn = data.findtext("filename") if outfn: if len(outfn) > 2 and outfn[0].isalpha() and \ outfn[1] == ':' and outfn[2] == '\\': continue files.append(outfn) self.filesDir = files elif self.isRss: files = [] # return title, descr, pubdate, linst doc edgr = "http://www.sec.gov/Archives/edgar" try: for dsElt in XmlUtil.descendants(self.rssDocument, None, "item"): instDoc = None for instDocElt in XmlUtil.descendants( dsElt, edgr, "xbrlFile"): if instDocElt.get( "(http://www.sec.gov/Archives/edgar}description" ).endswith("INSTANCE DOCUMENT"): instDoc = instDocElt.get( "(http://www.sec.gov/Archives/edgar}url") break if not instDoc: continue files.append(( XmlUtil.text(XmlUtil.descendant(dsElt, None, "title")), # tooltip "{0}\n {1}\n {2}\n {3}\n {4}".format( XmlUtil.text( XmlUtil.descendant(dsElt, edgr, "companyName")), XmlUtil.text( XmlUtil.descendant(dsElt, edgr, "formType")), XmlUtil.text( XmlUtil.descendant(dsElt, edgr, "filingDate")), XmlUtil.text( XmlUtil.descendant(dsElt, edgr, "cikNumber")), XmlUtil.text( XmlUtil.descendant(dsElt, edgr, "period"))), XmlUtil.text( XmlUtil.descendant(dsElt, None, "description")), XmlUtil.text(XmlUtil.descendant( dsElt, None, "pubDate")), instDoc)) self.filesDir = files except (EnvironmentError, etree.LxmlError) as err: pass elif self.isInstalledTaxonomyPackage: files = [] baseurlPathLen = len(os.path.dirname(self.baseurl)) + 1 def packageDirsFiles(dir): for file in os.listdir(dir): path = dir + "/" + file # must have / and not \\ even on windows files.append(path[baseurlPathLen:]) if os.path.isdir(path): packageDirsFiles(path) packageDirsFiles(self.baseurl[0:baseurlPathLen - 1]) self.filesDir = files return self.filesDir
def period(self): per = XmlUtil.text(XmlUtil.descendant(self, edgr, "period")) if per and len(per) == 8: return "{0}-{1}-{2}".format(per[0:4],per[4:6],per[6:8]) return None
def accessionNumber(self): return XmlUtil.text(XmlUtil.descendant(self, edgr, "accessionNumber"))
def resultXbrlInstanceUri(self): resultInstance = XmlUtil.descendant(XmlUtil.descendant(self, None, "result"), None, "instance") if resultInstance is not None: return XmlUtil.text(resultInstance) return None
def formType(self): return XmlUtil.text(XmlUtil.descendant(self, edgr, "formType"))
def sphinxTestcaseVariationExpectedSeverity(modelTestcaseVariation, *args, **kwargs): issueElement = XmlUtil.descendant(modelTestcaseVariation, 'http://www.corefiling.com/sphinx-conformance-harness/2.0', "issue") if issueElement is not None: return issueElement.get("severity") return None # no issue or not a sphinx test case variation
def resultIsTable(self): result = XmlUtil.descendant(self, None, "result") if result is not None : child = XmlUtil.child(result, None, "table") return child is not None and XmlUtil.text(child).endswith(".xml") return False
def dir(self): self.open() if not self.isOpen: return None elif self.filesDir is not None: return self.filesDir elif self.isZip: files = [] for zipinfo in self.fs.infolist(): files.append(zipinfo.filename) self.filesDir = files elif self.isTarGz: self.filesDir = self.fs.getnames() elif self.isEis: files = [] for docElt in self.eisDocument.iter(tag="{http://www.sec.gov/edgar/common}document"): outfn = docElt.findtext("{http://www.sec.gov/edgar/common}conformedName") if outfn: files.append(outfn); self.filesDir = files elif self.isXfd: files = [] for data in self.xfdDocument.iter(tag="data"): outfn = data.findtext("filename") if outfn: if len(outfn) > 2 and outfn[0].isalpha() and \ outfn[1] == ':' and outfn[2] == '\\': continue files.append(outfn); self.filesDir = files elif self.isRss: files = [] # return title, descr, pubdate, linst doc edgr = "http://www.sec.gov/Archives/edgar" try: for dsElt in XmlUtil.descendants(self.rssDocument, None, "item"): instDoc = None for instDocElt in XmlUtil.descendants(dsElt, edgr, "xbrlFile"): if instDocElt.get("(http://www.sec.gov/Archives/edgar}description").endswith("INSTANCE DOCUMENT"): instDoc = instDocElt.get("(http://www.sec.gov/Archives/edgar}url") break if not instDoc: continue files.append(( XmlUtil.text(XmlUtil.descendant(dsElt, None, "title")), # tooltip "{0}\n {1}\n {2}\n {3}\n {4}".format( XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "companyName")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "formType")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "filingDate")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "cikNumber")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "period"))), XmlUtil.text(XmlUtil.descendant(dsElt, None, "description")), XmlUtil.text(XmlUtil.descendant(dsElt, None, "pubDate")), instDoc)) self.filesDir = files except (EnvironmentError, etree.LxmlError) as err: pass elif self.isInstalledTaxonomyPackage: files = [] baseurlPathLen = len(os.path.dirname(self.baseurl)) + 1 def packageDirsFiles(dir): for file in os.listdir(dir): path = dir + "/" + file # must have / and not \\ even on windows files.append(path[baseurlPathLen:]) if os.path.isdir(path): packageDirsFiles(path) packageDirsFiles(self.baseurl[0:baseurlPathLen - 1]) self.filesDir = files return self.filesDir
def cikNumber(self): return XmlUtil.text(XmlUtil.descendant(self, edgr, "cikNumber"))