def expected(self):
if self.localName == "testcase":
return self.document.basename[:4] #starts with PASS or FAIL
errorElement = XmlUtil.descendant(self, None, "error")
if errorElement is not None:
return ModelValue.qname(errorElement, XmlUtil.text(errorElement))
resultElement = XmlUtil.descendant(self, None, "result")
if resultElement is not None:
expected = resultElement.get("expected")
if expected:
return expected
for assertElement in XmlUtil.children(resultElement, None, "assert"):
num = assertElement.get("num")
if len(num) == 5:
return "EFM.{0}.{1}.{2}".format(num[0],num[1:3],num[3:6])
asserTests = {}
for atElt in XmlUtil.children(resultElement, None, "assertionTests"):
try:
asserTests[atElt.get("assertionID")] = (_INT(atElt.get("countSatisfied")),_INT(atElt.get("countNotSatisfied")))
except ValueError:
pass
if asserTests:
return asserTests
elif self.get("result"):
return self.get("result")
return None
def reference(self):
efmNameElts = XmlUtil.children(self.getparent(), None, "name")
for efmNameElt in efmNameElts:
if efmNameElt is not None and efmNameElt.text.startswith("EDGAR"):
return efmNameElt.text
referenceElement = XmlUtil.descendant(self, None, "reference")
if referenceElement is not None: # formula test suite
return "{0}#{1}".format(
referenceElement.get("specification"),
referenceElement.get("id"))
referenceElement = XmlUtil.descendant(self, None,
"documentationReference")
if referenceElement is not None: # w3c test suite
return referenceElement.get("{http://www.w3.org/1999/xlink}href")
descriptionElement = XmlUtil.descendant(self, None, "description")
if descriptionElement is not None and descriptionElement.get(
"reference"):
return descriptionElement.get("reference") # xdt test suite
if self.getparent().get("description"):
return self.getparent().get(
"description") # base spec 2.1 test suite
functRegistryRefElt = XmlUtil.descendant(self.getparent(), None,
"reference")
if functRegistryRefElt is not None: # function registry
return functRegistryRefElt.get(
"{http://www.w3.org/1999/xlink}href")
return None
def cfcnTest(self):
# tuple of (expression, element holding the expression)
try:
return self._cfcnTest
except AttributeError:
self._cfcnTest = None
if self.localName == "test-case": # xpath testcase
outputFileElement = XmlUtil.descendant(self, None, "output-file")
if outputFileElement is not None and outputFileElement.get("compare") == "Text":
filepath = (
self.modelDocument.filepathdir
+ "/"
+ "ExpectedTestResults/"
+ self.get("FilePath")
+ outputFileElement.text
)
if os.sep != "/":
filepath = filepath.replace("/", os.sep)
with io.open(filepath, "rt", encoding="utf-8") as f:
self._cfcnTest = ("xs:string($result) eq '{0}'".format(f.read()), self)
else:
testElement = XmlUtil.descendant(self, XbrlConst.cfcn, "test")
if testElement is not None:
self._cfcnTest = (XmlUtil.innerText(testElement), testElement)
elif self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms":
output = self.get("output")
if output:
self._cfcnTest = ("$result eq '{0}'".format(output.replace("'", "''")), self)
return self._cfcnTest
def readMeFirstUris(self):
try:
return self._readMeFirstUris
except AttributeError:
self._readMeFirstUris = []
# first look if any plugin method to get readme first URIs
if not any(
pluginXbrlMethod(self)
for pluginXbrlMethod in pluginClassMethods(
"ModelTestcaseVariation.ReadMeFirstUris")):
if self.localName == "testGroup": #w3c testcase
instanceTestElement = XmlUtil.descendant(
self, None, "instanceTest")
if instanceTestElement is not None: # take instance first
self._readMeFirstUris.append(
XmlUtil.descendantAttr(
instanceTestElement, None, "instanceDocument",
"{http://www.w3.org/1999/xlink}href"))
else:
schemaTestElement = XmlUtil.descendant(
self, None, "schemaTest")
if schemaTestElement is not None:
self._readMeFirstUris.append(
XmlUtil.descendantAttr(
schemaTestElement, None, "schemaDocument",
"{http://www.w3.org/1999/xlink}href"))
elif self.localName == "test-case": #xpath testcase
inputFileElement = XmlUtil.descendant(
self, None, "input-file")
if inputFileElement is not None: # take instance first
self._readMeFirstUris.append(
"TestSources/" + inputFileElement.text + ".xml")
else:
# default built-in method for readme first uris
for anElement in self.iterdescendants():
if isinstance(anElement,
ModelObject) and anElement.get(
"readMeFirst") == "true":
if anElement.get(
"{http://www.w3.org/1999/xlink}href"):
uri = anElement.get(
"{http://www.w3.org/1999/xlink}href")
else:
uri = XmlUtil.innerText(anElement)
if anElement.get("name"):
self._readMeFirstUris.append((ModelValue.qname(
anElement, anElement.get("name")), uri))
elif anElement.get("dts"):
self._readMeFirstUris.append(
(anElement.get("dts"), uri))
else:
self._readMeFirstUris.append(uri)
if not self._readMeFirstUris: # provide a dummy empty instance document
self._readMeFirstUris.append(
os.path.join(self.modelXbrl.modelManager.cntlr.configDir,
"empty-instance.xml"))
return self._readMeFirstUris
def measures(self):
try:
return self._measures
except AttributeError:
if self.isDivide:
self._measures = (measuresOf(XmlUtil.descendant(self, XbrlConst.xbrli, "unitNumerator")),
measuresOf(XmlUtil.descendant(self, XbrlConst.xbrli, "unitDenominator")))
else:
self._measures = (measuresOf(self),[])
return self._measures
def resultXbrlInstanceUri(self):
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ResultXbrlInstanceUri"):
resultInstanceUri = pluginXbrlMethod(self)
if resultInstanceUri is not None:
return resultInstanceUri or None # (empty string returns None)
resultInstance = XmlUtil.descendant(XmlUtil.descendant(self, None, "result"), None, "instance")
if resultInstance is not None:
return XmlUtil.text(resultInstance)
return None
def severityLevel(self):
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ExpectedSeverity"):
severityLevelName = pluginXbrlMethod(self)
if severityLevelName: # ignore plug in if not a plug-in-recognized test case
return logging._checkLevel(severityLevelName)
# default behavior without plugins
# SEC error cases have <assert severity={err|wrn}>...
if (XmlUtil.descendant(self, None, "assert", attrName="severity", attrValue="wrn") is not None or
XmlUtil.descendant(self, None, "result", attrName="severity", attrValue="warning") is not None):
return logging._checkLevel("WARNING")
return logging._checkLevel("INCONSISTENCY")
def measures(self):
"""([QName],[Qname]) -- Returns a tuple of multiply measures list and divide members list
(empty if not a divide element). Each list of QNames is in prefixed-name order."""
try:
return self._measures
except AttributeError:
if self.isDivide:
self._measures = (measuresOf(XmlUtil.descendant(self, XbrlConst.xbrli, "unitNumerator")),
measuresOf(XmlUtil.descendant(self, XbrlConst.xbrli, "unitDenominator")))
else:
self._measures = (measuresOf(self),[])
return self._measures
def expected(self):
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ExpectedResult"):
expected = pluginXbrlMethod(self)
if expected:
return expected
# default behavior without plugins
if self.localName == "testcase":
return self.document.basename[:4] #starts with PASS or FAIL
elif self.localName == "testGroup": #w3c testcase
instanceTestElement = XmlUtil.descendant(self, None, "instanceTest")
if instanceTestElement is not None: # take instance first
return XmlUtil.descendantAttr(instanceTestElement, None, "expected", "validity")
else:
schemaTestElement = XmlUtil.descendant(self, None, "schemaTest")
if schemaTestElement is not None:
return XmlUtil.descendantAttr(schemaTestElement, None, "expected", "validity")
resultElement = XmlUtil.descendant(self, None, "result")
if resultElement is not None:
expected = resultElement.get("expected")
if expected and resultElement.get("nonStandardErrorCodes") == "true":
# if @expected and @nonStandardErrorCodes then use expected instead of error codes
return expected
errorElement = XmlUtil.descendant(self, None, "error")
resultElement = XmlUtil.descendant(self, None, "result")
if errorElement is not None and not errorElement.get("nonStandardErrorCodes"):
_errorText = XmlUtil.text(errorElement)
if ' ' in _errorText: # list of tokens
return _errorText
return ModelValue.qname(errorElement, _errorText) # turn into a QName
if resultElement is not None:
if expected:
return expected
for assertElement in XmlUtil.children(resultElement, None, "assert"):
num = assertElement.get("num")
if num == "99999": # inline test, use name as expected
return assertElement.get("name")
if len(num) == 5:
return "EFM.{0}.{1}.{2}".format(num[0],num[1:3],num[3:6])
asserTests = {}
for atElt in XmlUtil.children(resultElement, None, "assertionTests"):
try:
asserTests[atElt.get("assertionID")] = (_INT(atElt.get("countSatisfied")),_INT(atElt.get("countNotSatisfied")))
except ValueError:
pass
if asserTests:
return asserTests
elif self.get("result"):
return self.get("result")
return None
def dir(self):
self.open()
if not self.isOpen:
return None
elif self.filesDir is not None:
return self.filesDir
elif self.isZip:
files = []
for zipinfo in self.fs.infolist():
files.append(zipinfo.filename)
self.filesDir = files
elif self.isTarGz:
self.filesDir = self.fs.getnames()
elif self.isEis:
files = []
for docElt in self.eisDocument.iter(tag=u"{http://www.sec.gov/edgar/common}document"):
outfn = docElt.findtext(u"{http://www.sec.gov/edgar/common}conformedName")
if outfn:
files.append(outfn);
self.filesDir = files
elif self.isXfd:
files = []
for data in self.xfdDocument.iter(tag=u"data"):
outfn = data.findtext(u"filename")
if outfn:
if len(outfn) > 2 and outfn[0].isalpha() and \
outfn[1] == u':' and outfn[2] == u'\\':
continue
files.append(outfn);
self.filesDir = files
elif self.isRss:
files = [] # return title, descr, pubdate, linst doc
edgr = u"http://www.sec.gov/Archives/edgar"
try:
for dsElt in XmlUtil.descendants(self.rssDocument, None, u"item"):
instDoc = None
for instDocElt in XmlUtil.descendants(dsElt, edgr, u"xbrlFile"):
if instDocElt.get(u"(http://www.sec.gov/Archives/edgar}description").endswith(u"INSTANCE DOCUMENT"):
instDoc = instDocElt.get(u"(http://www.sec.gov/Archives/edgar}url")
break
if not instDoc:
continue
files.append((
XmlUtil.text(XmlUtil.descendant(dsElt, None, u"title")),
# tooltip
u"{0}\n {1}\n {2}\n {3}\n {4}".format(
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, u"companyName")),
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, u"formType")),
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, u"filingDate")),
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, u"cikNumber")),
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, u"period"))),
XmlUtil.text(XmlUtil.descendant(dsElt, None, u"description")),
XmlUtil.text(XmlUtil.descendant(dsElt, None, u"pubDate")),
instDoc))
self.filesDir = files
except (EnvironmentError,
etree.LxmlError), err:
pass
def pubDate(self):
try:
return self._pubDate
except AttributeError:
from arelle.UrlUtil import parseRfcDatetime
self._pubDate = parseRfcDatetime(XmlUtil.text(XmlUtil.descendant(self, None, "pubDate")))
return self._pubDate
def resultIsTable(self):
result = XmlUtil.descendant(self, None, "result")
if result is not None :
child = XmlUtil.child(result, None, "table")
if child is not None and XmlUtil.text(child).endswith(".xml"):
return True
return False
def cfcnCall(self):
# tuple of (expression, element holding the expression)
try:
return self._cfcnCall
except AttributeError:
self._cfcnCall = None
if self.localName == "test-case": # xpath testcase
queryElement = XmlUtil.descendant(self, None, "query")
if queryElement is not None:
filepath = (
self.modelDocument.filepathdir
+ "/"
+ "Queries/XQuery/"
+ self.get("FilePath")
+ queryElement.get("name")
+ ".xq"
)
if os.sep != "/":
filepath = filepath.replace("/", os.sep)
with io.open(filepath, "rt", encoding="utf-8") as f:
self._cfcnCall = (f.read(), self)
else:
for callElement in XmlUtil.descendants(self, XbrlConst.cfcn, "call"):
self._cfcnCall = (XmlUtil.innerText(callElement), callElement)
break
if self._cfcnCall is None and self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms":
name = self.getparent().get("name")
input = self.get("input")
if name and input:
self._cfcnCall = ("{0}('{1}')".format(name, input.replace("'", "''")), self)
return self._cfcnCall
def resultTableUri(self):
result = XmlUtil.descendant(self, None, "result")
if result is not None:
child = XmlUtil.child(result, None, "table")
if child is not None:
return os.path.join(self.modelDocument.outpath, XmlUtil.text(child))
return None
def resultInfosetUri(self):
result = XmlUtil.descendant(self, None, "result")
if result is not None:
child = XmlUtil.child(result, None, "file")
return os.path.join(
self.modelDocument.outpath,
XmlUtil.text(child if child is not None else result))
return None
def resultIsInfoset(self):
if self.modelDocument.outpath:
result = XmlUtil.descendant(self, None, "result")
if result is not None:
return XmlUtil.child(result, None,
"file") is not None or XmlUtil.text(
result).endswith(".xml")
return False
def acceptanceDatetime(self):
try:
return self._acceptanceDatetime
except AttributeError:
import datetime
self._acceptanceDatetime = None
date = XmlUtil.text(XmlUtil.descendant(self, edgr, u"acceptanceDatetime"))
if date and len(date) == 14:
self._acceptanceDatetime = datetime.datetime(_INT(date[0:4]),_INT(date[4:6]),_INT(date[6:8]),_INT(date[8:10]),_INT(date[10:12]),_INT(date[12:14]))
return self._acceptanceDatetime
def unit_denominator(xc, p, args):
if len(args) != 1: raise XPathContext.FunctionNumArgs()
if len(args[0]) != 1: raise XPathContext.FunctionArgType(1,"xbrl:unit")
unit = args[0][0]
if isinstance(unit,ModelObject) and \
unit.localName == "unit" and unit.namespaceURI == XbrlConst.xbrli:
measuresParent = XmlUtil.descendant(unit, XbrlConst.xbrli, "unitDenominator")
if measuresParent is None: return []
return XmlUtil.descendants(measuresParent, XbrlConst.xbrli, "measure")
raise XPathContext.FunctionArgType(1,"xbrl:unit")
def unit_numerator(xc, p, args):
if len(args) != 1: raise XPathContext.FunctionNumArgs()
if len(args[0]) != 1: raise XPathContext.FunctionArgType(1,"xbrl:unit")
unit = args[0][0]
if isinstance(unit,ModelObject.ModelObject): unit = unit.element
if isinstance(unit,xml.dom.Node) and unit.nodeType == 1 and \
unit.localName == "unit" and unit.namespaceURI == XbrlConst.xbrli:
measuresParent = XmlUtil.descendant(unit, XbrlConst.xbrli, "unitNumerator")
if measuresParent is None: measuresParent = unit
return XmlUtil.descendants(measuresParent, XbrlConst.xbrli, "measure")
raise XPathContext.FunctionArgType(1,"xbrl:unit")
def filingDate(self):
try:
return self._filingDate
except AttributeError:
import datetime
self._filingDate = None
date = XmlUtil.text(XmlUtil.descendant(self, edgr, "filingDate"))
d = date.split("/")
if d and len(d) == 3:
self._filingDate = datetime.date(_INT(d[2]),_INT(d[0]),_INT(d[1]))
return self._filingDate
def dir(self):
self.open()
if not self.isOpen:
return None
elif self.filesDir is not None:
return self.filesDir
elif self.isZip:
files = []
for zipinfo in self.fs.infolist():
files.append(zipinfo.filename)
self.filesDir = files
elif self.isXfd:
files = []
for data in self.xfdDocument.getElementsByTagName("data"):
outfn = XmlUtil.text(data.getElementsByTagName("filename")[0])
if len(outfn) > 1:
if len(outfn) > 2 and outfn[0].isalpha() and \
outfn[1] == ':' and outfn[2] == '\\':
continue
files.append(outfn);
self.filesDir = files
elif self.isRss:
files = [] # return title, descr, pubdate, linst doc
edgr = "http://www.sec.gov/Archives/edgar"
try:
for dsElt in self.rssDocument.getElementsByTagName("item"):
instDoc = None
for instDocElt in XmlUtil.descendants(dsElt, edgr, "xbrlFile"):
if instDocElt.getAttributeNS(edgr,"description").endswith("INSTANCE DOCUMENT"):
instDoc = instDocElt.getAttributeNS(edgr,"url")
break
if not instDoc:
continue
files.append((
XmlUtil.text(XmlUtil.descendant(dsElt, None, "title")),
# tooltip
"{0}\n {1}\n {2}\n {3}\n {4}".format(
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "companyName")),
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "formType")),
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "filingDate")),
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "cikNumber")),
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "period"))),
XmlUtil.text(XmlUtil.descendant(dsElt, None, "description")),
XmlUtil.text(XmlUtil.descendant(dsElt, None, "pubDate")),
instDoc))
self.filesDir = files
except (EnvironmentError,
xml.parsers.expat.ExpatError,
xml.dom.DOMException) as err:
pass
return self.filesDir
def expected(self):
for pluginXbrlMethod in pluginClassMethods(u"ModelTestcaseVariation.ExpectedResult"):
expected = pluginXbrlMethod(self)
if expected:
return expected
# default behavior without plugins
if self.localName == u"testcase":
return self.document.basename[:4] #starts with PASS or FAIL
elif self.localName == u"testGroup": #w3c testcase
instanceTestElement = XmlUtil.descendant(self, None, u"instanceTest")
if instanceTestElement is not None: # take instance first
return XmlUtil.descendantAttr(instanceTestElement, None, u"expected", u"validity")
else:
schemaTestElement = XmlUtil.descendant(self, None, u"schemaTest")
if schemaTestElement is not None:
return XmlUtil.descendantAttr(schemaTestElement, None, u"expected", u"validity")
errorElement = XmlUtil.descendant(self, None, u"error")
if errorElement is not None:
return ModelValue.qname(errorElement, XmlUtil.text(errorElement))
resultElement = XmlUtil.descendant(self, None, u"result")
if resultElement is not None:
expected = resultElement.get(u"expected")
if expected:
return expected
for assertElement in XmlUtil.children(resultElement, None, u"assert"):
num = assertElement.get(u"num")
if len(num) == 5:
return u"EFM.{0}.{1}.{2}".format(num[0],num[1:3],num[3:6])
asserTests = {}
for atElt in XmlUtil.children(resultElement, None, u"assertionTests"):
try:
asserTests[atElt.get(u"assertionID")] = (_INT(atElt.get(u"countSatisfied")),_INT(atElt.get(u"countNotSatisfied")))
except ValueError:
pass
if asserTests:
return asserTests
elif self.get(u"result"):
return self.get(u"result")
return None
def name(self):
try:
return self._name
except AttributeError:
if self.get("name"):
self._name = self.get("name")
else:
nameElement = XmlUtil.descendant(self, None, "name" if self.localName != "testcase" else "number")
if nameElement is not None:
self._name = XmlUtil.innerText(nameElement)
else:
self._name = None
return self._name
def cfcnTest(self):
# tuple of (expression, element holding the expression)
try:
return self._cfcnTest
except AttributeError:
self._cfcnTest = None
testElement = XmlUtil.descendant(self, XbrlConst.cfcn, "test")
if testElement is not None:
self._cfcnTest = (XmlUtil.innerText(testElement), testElement)
elif self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms":
output = self.get("output")
if output:
self._cfcnTest = ("$result eq '{0}'".format(output.replace("'","''")), self)
return self._cfcnTest
def parent_child(args, parentName, childName, findDescendant=False):
if len(args) != 1: raise XPathContext.FunctionNumArgs()
if len(args[0]) != 1: raise XPathContext.FunctionArgType(1,"xbrl:" + parentName)
parent = args[0][0]
if isinstance(parent,ModelObject) and \
parent.localName == parentName and parent.namespaceURI == XbrlConst.xbrli:
if childName.startswith('@'):
return parent.get(childName[1:])
elif childName == 'text()':
return XmlUtil.textNotStripped(parent)
elif childName == 'strip-text()':
return XmlUtil.text(parent)
elif findDescendant:
return XmlUtil.descendant(parent, XbrlConst.xbrli, childName)
else:
return XmlUtil.child(parent, XbrlConst.xbrli, childName)
raise XPathContext.FunctionArgType(1,"xbrl:" + parentName)
def cfcnCall(self):
# tuple of (expression, element holding the expression)
try:
return self._cfcnCall
except AttributeError:
self._cfcnCall = None
if self.localName == "test-case": #xpath testcase
queryElement = XmlUtil.descendant(self, None, "query")
if queryElement is not None:
filepath = (self.modelDocument.filepathdir + "/" + "Queries/XQuery/" +
self.get("FilePath") + queryElement.get("name") + '.xq')
if os.sep != "/": filepath = filepath.replace("/", os.sep)
with io.open(filepath, 'rt', encoding='utf-8') as f:
self._cfcnCall = (f.read(), self)
else:
for callElement in XmlUtil.descendants(self, XbrlConst.cfcn, "call"):
self._cfcnCall = (XmlUtil.innerText(callElement), callElement)
break
if self._cfcnCall is None and self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms":
name = self.getparent().get("name")
input = self.get("input")
if name and input:
self._cfcnCall = ("{0}('{1}')".format(name, input.replace("'","''")), self)
return self._cfcnCall
def period(self):
per = XmlUtil.text(XmlUtil.descendant(self, edgr, "period"))
if per and len(per) == 8:
return "{0}-{1}-{2}".format(per[0:4], per[4:6], per[6:8])
return None
def dir(self):
self.open()
if not self.isOpen:
return None
elif self.filesDir is not None:
return self.filesDir
elif self.isZip:
files = []
for zipinfo in self.fs.infolist():
files.append(zipinfo.filename)
self.filesDir = files
elif self.isXfd:
files = []
for data in self.xfdDocument.getElementsByTagName("data"):
outfn = XmlUtil.text(data.getElementsByTagName("filename")[0])
if len(outfn) > 1:
if len(outfn) > 2 and outfn[0].isalpha() and \
outfn[1] == ':' and outfn[2] == '\\':
continue
files.append(outfn)
self.filesDir = files
elif self.isRss:
files = [] # return title, descr, pubdate, linst doc
edgr = "http://www.sec.gov/Archives/edgar"
try:
for dsElt in self.rssDocument.getElementsByTagName("item"):
instDoc = None
for instDocElt in XmlUtil.descendants(
dsElt, edgr, "xbrlFile"):
if instDocElt.getAttributeNS(
edgr,
"description").endswith("INSTANCE DOCUMENT"):
instDoc = instDocElt.getAttributeNS(edgr, "url")
break
if not instDoc:
continue
files.append((
XmlUtil.text(XmlUtil.descendant(dsElt, None, "title")),
# tooltip
"{0}\n {1}\n {2}\n {3}\n {4}".format(
XmlUtil.text(
XmlUtil.descendant(dsElt, edgr,
"companyName")),
XmlUtil.text(
XmlUtil.descendant(dsElt, edgr, "formType")),
XmlUtil.text(
XmlUtil.descendant(dsElt, edgr, "filingDate")),
XmlUtil.text(
XmlUtil.descendant(dsElt, edgr, "cikNumber")),
XmlUtil.text(
XmlUtil.descendant(dsElt, edgr, "period"))),
XmlUtil.text(
XmlUtil.descendant(dsElt, None, "description")),
XmlUtil.text(XmlUtil.descendant(
dsElt, None, "pubDate")),
instDoc))
self.filesDir = files
except (EnvironmentError, xml.parsers.expat.ExpatError,
xml.dom.DOMException) as err:
pass
return self.filesDir
def fiscalYearEnd(self):
yrEnd = XmlUtil.text(XmlUtil.descendant(self, edgr, "fiscalYearEnd"))
if yrEnd and len(yrEnd) == 4:
return "{0}-{1}".format(yrEnd[0:2],yrEnd[2:4])
return None
def resultInfosetUri(self):
result = XmlUtil.descendant(self, None, "result")
if result is not None:
child = XmlUtil.child(result, None, "file")
return os.path.join(self.modelDocument.outpath, XmlUtil.text(child if child is not None else result))
return None
def description(self):
nameElement = XmlUtil.descendant(self, None, ("description", "documentation"))
if nameElement is not None:
return XmlUtil.innerText(nameElement)
return None
def resultIsVersioningReport(self):
return XmlUtil.descendant(XmlUtil.descendant(self, None, "result"), None, "versioningReport") is not None
def resultIsXbrlInstance(self):
return XmlUtil.descendant(XmlUtil.descendant(self, None, "result"), None, "instance") is not None
def checkFilingDTS(val, modelDocument, visited):
global targetNamespaceDatePattern, efmFilenamePattern, roleTypePattern, arcroleTypePattern, \
arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet, \
namespacesConflictPattern
if targetNamespaceDatePattern is None:
targetNamespaceDatePattern = re.compile(
r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|"
r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|")
efmFilenamePattern = re.compile(
r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml)$")
roleTypePattern = re.compile(r"^.*/role/[^/\s]+$")
arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$")
arcroleDefinitionPattern = re.compile(
r"^.*[^\\s]+.*$") # at least one non-whitespace character
namePattern = re.compile(
"[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]"
) # u20ac=Euro, u00a3=pound sterling
linkroleDefinitionBalanceIncomeSheet = re.compile(
r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)",
re.IGNORECASE)
namespacesConflictPattern = re.compile(
r"http://(xbrl\.us|fasb\.org|xbrl\.sec\.gov)/(dei|us-types|us-roles|rr)/([0-9]{4}-[0-9]{2}-[0-9]{2})$"
)
visited.append(modelDocument)
for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items(
):
#6.07.01 no includes
if "include" in modelDocumentReference.referenceTypes:
val.modelXbrl.error(
"SBR.NL.2.2.0.18",
_("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"
),
modelObject=modelDocumentReference.referringModelObject,
schema=os.path.basename(modelDocument.uri),
include=os.path.basename(referencedDocument.uri))
if referencedDocument not in visited:
checkFilingDTS(val, referencedDocument, visited)
if val.disclosureSystem.standardTaxonomiesDict is None:
pass
if (modelDocument.type == ModelDocument.Type.SCHEMA
and modelDocument.targetNamespace
not in val.disclosureSystem.baseTaxonomyNamespaces
and modelDocument.uri.startswith(val.modelXbrl.uriDir)):
# check schema contents types
definesLinkroles = False
definesArcroles = False
definesLinkParts = False
definesAbstractItems = False
definesNonabstractItems = False
definesConcepts = False
definesTuples = False
definesPresentationTuples = False
definesSpecificationTuples = False
definesTypes = False
definesEnumerations = False
definesDimensions = False
definesDomains = False
definesHypercubes = False
genrlSpeclRelSet = val.modelXbrl.relationshipSet(
XbrlConst.generalSpecial)
for modelConcept in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.w3.org/2001/XMLSchema}element"):
if isinstance(modelConcept, ModelConcept):
# 6.7.16 name not duplicated in standard taxonomies
name = modelConcept.get("name")
if name is None:
name = ""
if modelConcept.get("ref") is not None:
continue # don't validate ref's here
for c in val.modelXbrl.nameConcepts.get(name, []):
if c.modelDocument != modelDocument:
if not (genrlSpeclRelSet.isRelated(
modelConcept, "child", c)
or genrlSpeclRelSet.isRelated(
c, "child", modelConcept)):
val.modelXbrl.error(
"SBR.NL.2.2.2.02",
_("Concept %(concept)s is also defined in standard taxonomy schema %(standardSchema)s without a general-special relationship"
),
modelObject=c,
concept=modelConcept.qname,
standardSchema=os.path.basename(
c.modelDocument.uri))
''' removed RH 2011-12-23 corresponding set up of table in ValidateFiling
if val.validateSBRNL and name in val.nameWordsTable:
if not any( any( genrlSpeclRelSet.isRelated(c, "child", modelConcept)
for c in val.modelXbrl.nameConcepts.get(partialWordName, []))
for partialWordName in val.nameWordsTable[name]):
val.modelXbrl.error("SBR.NL.2.3.2.01",
_("Concept %(specialName)s is appears to be missing a general-special relationship to %(generalNames)s"),
modelObject=c, specialName=modelConcept.qname, generalNames=', or to '.join(val.nameWordsTable[name]))
'''
if modelConcept.isTuple:
if modelConcept.substitutionGroupQname.localName == "presentationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith(
"/basis/sbr/xbrl/xbrl-syntax-extension"
): # namespace may change each year
definesPresentationTuples = True
elif modelConcept.substitutionGroupQname.localName == "specificationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith(
"/basis/sbr/xbrl/xbrl-syntax-extension"
): # namespace may change each year
definesSpecificationTuples = True
else:
definesTuples = True
definesConcepts = True
if modelConcept.isAbstract:
val.modelXbrl.error(
"SBR.NL.2.2.2.03",
_("Concept %(concept)s is an abstract tuple"),
modelObject=modelConcept,
concept=modelConcept.qname)
if tupleCycle(val, modelConcept):
val.modelXbrl.error(
"SBR.NL.2.2.2.07",
_("Tuple %(concept)s has a tuple cycle"),
modelObject=modelConcept,
concept=modelConcept.qname)
if modelConcept.get(
"nillable") != "false" and modelConcept.isRoot:
val.modelXbrl.error(
"SBR.NL.2.2.2.17", #don't want default, just what was really there
_("Tuple %(concept)s must have nillable='false'"),
modelObject=modelConcept,
concept=modelConcept.qname)
elif modelConcept.isItem:
definesConcepts = True
if modelConcept.abstract == "true":
if modelConcept.isRoot:
if modelConcept.get(
"nillable"
) != "false": #don't want default, just what was really there
val.modelXbrl.error(
"SBR.NL.2.2.2.16",
_("Abstract root concept %(concept)s must have nillable='false'"
),
modelObject=modelConcept,
concept=modelConcept.qname)
if modelConcept.typeQname != XbrlConst.qnXbrliStringItemType:
val.modelXbrl.error(
"SBR.NL.2.2.2.21",
_("Abstract root concept %(concept)s must have type='xbrli:stringItemType'"
),
modelObject=modelConcept,
concept=modelConcept.qname)
if modelConcept.balance:
val.modelXbrl.error(
"SBR.NL.2.2.2.22",
_("Abstract concept %(concept)s must not have a balance attribute"
),
modelObject=modelConcept,
concept=modelConcept.qname)
if modelConcept.isHypercubeItem:
definesHypercubes = True
elif modelConcept.isDimensionItem:
definesDimensions = True
elif modelConcept.substitutionGroupQname and modelConcept.substitutionGroupQname.localName in (
"domainItem", "domainMemberItem"):
definesDomains = True
elif modelConcept.isItem:
definesAbstractItems = True
else: # not abstract
if modelConcept.isItem:
definesNonabstractItems = True
if not (modelConcept.label(
preferredLabel=XbrlConst.documentationLabel,
fallbackToQname=False,
lang="nl") or val.modelXbrl.relationshipSet(
XbrlConst.conceptReference
).fromModelObject(c) or modelConcept.genLabel(
role=XbrlConst.genDocumentationLabel,
lang="nl")
or val.modelXbrl.relationshipSet(
XbrlConst.elementReference).
fromModelObject(c)):
val.modelXbrl.error(
"SBR.NL.2.2.2.28",
_("Concept %(concept)s must have a documentation label or reference"
),
modelObject=modelConcept,
concept=modelConcept.qname)
if modelConcept.balance and not modelConcept.instanceOfType(
XbrlConst.qnXbrliMonetaryItemType):
val.modelXbrl.error(
"SBR.NL.2.2.2.24",
_("Non-monetary concept %(concept)s must not have a balance attribute"
),
modelObject=modelConcept,
concept=modelConcept.qname)
if modelConcept.isLinkPart:
definesLinkParts = True
val.modelXbrl.error(
"SBR.NL.2.2.5.01",
_("Link:part concept %(concept)s is not allowed"),
modelObject=modelConcept,
concept=modelConcept.qname)
if not modelConcept.genLabel(fallbackToQname=False,
lang="nl"):
val.modelXbrl.error(
"SBR.NL.2.2.5.02",
_("Link part definition %(concept)s must have a generic label in language 'nl'"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.7.9 role types authority
for e in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.xbrl.org/2003/linkbase}roleType"):
if isinstance(e, ModelObject):
roleURI = e.get("roleURI")
# 6.7.10 only one role type declaration in DTS
modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI)
if modelRoleTypes is not None:
modelRoleType = modelRoleTypes[0]
definition = modelRoleType.definitionNotStripped
usedOns = modelRoleType.usedOns
if usedOns & XbrlConst.standardExtLinkQnames or XbrlConst.qnGenLink in usedOns:
definesLinkroles = True
if not e.genLabel():
val.modelXbrl.error(
"SBR.NL.2.2.3.03",
_("Link RoleType %(roleType)s missing a generic standard label"
),
modelObject=e,
roleType=roleURI)
nlLabel = e.genLabel(lang="nl")
if definition != nlLabel:
val.modelXbrl.error(
"SBR.NL.2.2.3.04",
_("Link RoleType %(roleType)s definition does not match NL standard generic label, \ndefinition: %(definition)s \nNL label: %(label)s"
),
modelObject=e,
roleType=roleURI,
definition=definition,
label=nlLabel)
if definition and (definition[0].isspace()
or definition[-1].isspace()):
val.modelXbrl.error(
"SBR.NL.2.2.3.07",
_('Link RoleType %(roleType)s definition has leading or trailing spaces: "%(definition)s"'
),
modelObject=e,
roleType=roleURI,
definition=definition)
# 6.7.13 arcrole types authority
for e in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.xbrl.org/2003/linkbase}arcroleType"):
if isinstance(e, ModelObject):
arcroleURI = e.get("arcroleURI")
definesArcroles = True
val.modelXbrl.error(
"SBR.NL.2.2.4.01",
_("Arcrole type definition is not allowed: %(arcroleURI)s"
),
modelObject=e,
arcroleURI=arcroleURI)
for appinfoElt in modelDocument.xmlRootElement.iter(
tag="{http://www.w3.org/2001/XMLSchema}appinfo"):
for nonLinkElt in appinfoElt.iterdescendants():
if isinstance(nonLinkElt, ModelObject
) and nonLinkElt.namespaceURI != XbrlConst.link:
val.modelXbrl.error(
"SBR.NL.2.2.11.05",
_("Appinfo contains disallowed non-link element %(element)s"
),
modelObject=nonLinkElt,
element=nonLinkElt.qname)
for cplxTypeElt in modelDocument.xmlRootElement.iter(
tag="{http://www.w3.org/2001/XMLSchema}complexType"):
choiceElt = cplxTypeElt.find(
"{http://www.w3.org/2001/XMLSchema}choice")
if choiceElt is not None:
val.modelXbrl.error(
"SBR.NL.2.2.11.09",
_("ComplexType contains disallowed xs:choice element"),
modelObject=choiceElt)
for cplxContentElt in modelDocument.xmlRootElement.iter(
tag="{http://www.w3.org/2001/XMLSchema}complexContent"):
if XmlUtil.descendantAttr(
cplxContentElt, "http://www.w3.org/2001/XMLSchema",
("extension", "restriction"), "base") != "sbr:placeholder":
val.modelXbrl.error("SBR.NL.2.2.11.10",
_("ComplexContent is disallowed"),
modelObject=cplxContentElt)
for typeEltTag in ("{http://www.w3.org/2001/XMLSchema}complexType",
"{http://www.w3.org/2001/XMLSchema}simpleType"):
for typeElt in modelDocument.xmlRootElement.iter(tag=typeEltTag):
definesTypes = True
name = typeElt.get("name")
if name:
if not name[0].islower() or not name.isalnum():
val.modelXbrl.error(
"SBR.NL.3.2.8.09",
_("Type name attribute must be lower camelcase: %(name)s."
),
modelObject=typeElt,
name=name)
for enumElt in modelDocument.xmlRootElement.iter(
tag="{http://www.w3.org/2001/XMLSchema}enumeration"):
definesEnumerations = True
if any(not valueElt.genLabel(lang="nl")
for valueElt in enumElt.iter(
tag="{http://www.w3.org/2001/XMLSchema}value")):
val.modelXbrl.error(
"SBR.NL.2.2.7.05",
_("Enumeration element has value(s) without generic label."
),
modelObject=enumElt)
if (definesLinkroles + definesArcroles + definesLinkParts +
definesAbstractItems + definesNonabstractItems + definesTuples
+ definesPresentationTuples + definesSpecificationTuples +
definesTypes + definesEnumerations + definesDimensions +
definesDomains + definesHypercubes) != 1:
schemaContents = []
if definesLinkroles: schemaContents.append(_("linkroles"))
if definesArcroles: schemaContents.append(_("arcroles"))
if definesLinkParts: schemaContents.append(_("link parts"))
if definesAbstractItems: schemaContents.append(_("abstract items"))
if definesNonabstractItems:
schemaContents.append(_("nonabstract items"))
if definesTuples: schemaContents.append(_("tuples"))
if definesPresentationTuples:
schemaContents.append(_("sbrPresentationTuples"))
if definesSpecificationTuples:
schemaContents.append(_("sbrSpecificationTuples"))
if definesTypes: schemaContents.append(_("types"))
if definesEnumerations: schemaContents.append(_("enumerations"))
if definesDimensions: schemaContents.append(_("dimensions"))
if definesDomains: schemaContents.append(_("domains"))
if definesHypercubes: schemaContents.append(_("hypercubes"))
if schemaContents:
if not ((definesTuples or definesPresentationTuples
or definesSpecificationTuples)
and not (definesLinkroles or definesArcroles
or definesLinkParts or definesAbstractItems
or definesTypes or definesDimensions
or definesDomains or definesHypercubes)):
val.modelXbrl.error(
"SBR.NL.2.2.1.01",
_("Taxonomy schema may only define one of these: %(contents)s"
),
modelObject=modelDocument,
contents=', '.join(schemaContents))
elif not any(refDoc.inDTS and refDoc.targetNamespace not in
val.disclosureSystem.baseTaxonomyNamespaces
for refDoc in modelDocument.referencesDocument.keys(
)): # no linkbase ref or includes
val.modelXbrl.error(
"SBR.NL.2.2.1.01",
_("Taxonomy schema must be a DTS entrypoint OR define linkroles OR arcroles OR link:parts OR context fragments OR abstract items OR tuples OR non-abstract elements OR types OR enumerations OR dimensions OR domains OR hypercubes"
),
modelObject=modelDocument)
if definesConcepts ^ any( # xor so either concepts and no label LB or no concepts and has label LB
(refDoc.type == ModelDocument.Type.LINKBASE
and XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link,
"labelLink") is not None) for refDoc in
modelDocument.referencesDocument.keys()): # no label linkbase
val.modelXbrl.error(
"SBR.NL.2.2.1.02",
_("A schema that defines concepts MUST have a linked 2.1 label linkbase"
),
modelObject=modelDocument)
if (
definesNonabstractItems or definesTuples
) and not any( # was xor but changed to and not per RH 1/11/12
(refDoc.type == ModelDocument.Type.LINKBASE and
(XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link,
"referenceLink") is not None
or XmlUtil.descendant(
refDoc.xmlRootElement, XbrlConst.link, "label",
"{http://www.w3.org/1999/xlink}role",
"http://www.xbrl.org/2003/role/documentation") is not None))
for refDoc in modelDocument.referencesDocument.keys()):
val.modelXbrl.error(
"SBR.NL.2.2.1.03",
_("A schema that defines non-abstract items MUST have a linked (2.1) reference linkbase AND/OR a label linkbase with @xlink:role=documentation"
),
modelObject=modelDocument)
elif modelDocument.type == ModelDocument.Type.LINKBASE:
pass
visited.remove(modelDocument)
def companyName(self):
return XmlUtil.text(XmlUtil.descendant(self, edgr, "companyName"))
def readMeFirstUris(self):
try:
return self._readMeFirstUris
except AttributeError:
self._readMeFirstUris = []
self.readMeFirstElements = []
# first look if any plugin method to get readme first URIs
if not any(
pluginXbrlMethod(self)
for pluginXbrlMethod in pluginClassMethods(
"ModelTestcaseVariation.ReadMeFirstUris")):
if self.localName == "testGroup": #w3c testcase
instanceTestElement = XmlUtil.descendant(
self, None, "instanceTest")
if instanceTestElement is not None: # take instance first
self._readMeFirstUris.append(
XmlUtil.descendantAttr(
instanceTestElement, None, "instanceDocument",
"{http://www.w3.org/1999/xlink}href"))
self.readMeFirstElements.append(instanceTestElement)
else:
schemaTestElement = XmlUtil.descendant(
self, None, "schemaTest")
if schemaTestElement is not None:
self._readMeFirstUris.append(
XmlUtil.descendantAttr(
schemaTestElement, None, "schemaDocument",
"{http://www.w3.org/1999/xlink}href"))
self.readMeFirstElements.append(schemaTestElement)
elif self.localName == "test-case": #xpath testcase
inputFileElement = XmlUtil.descendant(
self, None, "input-file")
if inputFileElement is not None: # take instance first
self._readMeFirstUris.append("TestSources/" +
inputFileElement.text +
".xml")
self.readMeFirstElements.append(inputFileElement)
elif self.resultIsTaxonomyPackage:
self._readMeFirstUris.append(
os.path.join(self.modelDocument.filepathdir, "tests",
self.get("name") + ".zip"))
self.readMeFirstElements.append(self)
else:
# default built-in method for readme first uris
for anElement in self.iterdescendants():
if isinstance(anElement,
ModelObject) and anElement.get(
"readMeFirst") == "true":
if anElement.get(
"{http://www.w3.org/1999/xlink}href"):
uri = anElement.get(
"{http://www.w3.org/1999/xlink}href")
else:
uri = XmlUtil.innerText(anElement)
if anElement.get("name"):
self._readMeFirstUris.append(
(ModelValue.qname(anElement,
anElement.get("name")),
uri))
self.readMeFirstElements.append(anElement)
elif anElement.get("dts"):
self._readMeFirstUris.append(
(anElement.get("dts"), uri))
self.readMeFirstElements.append(anElement)
else:
self._readMeFirstUris.append(uri)
self.readMeFirstElements.append(anElement)
if not self._readMeFirstUris: # provide a dummy empty instance document
self._readMeFirstUris.append(
os.path.join(self.modelXbrl.modelManager.cntlr.configDir,
"empty-instance.xml"))
self.readMeFirstElements.append(None)
return self._readMeFirstUris
def expected(self):
for pluginXbrlMethod in pluginClassMethods(
"ModelTestcaseVariation.ExpectedResult"):
expected = pluginXbrlMethod(self)
if expected:
return expected
# default behavior without plugins
if self.localName == "testcase":
return self.document.basename[:4] #starts with PASS or FAIL
elif self.localName == "testGroup": #w3c testcase
instanceTestElement = XmlUtil.descendant(self, None,
"instanceTest")
if instanceTestElement is not None: # take instance first
return XmlUtil.descendantAttr(instanceTestElement, None,
"expected", "validity")
else:
schemaTestElement = XmlUtil.descendant(self, None,
"schemaTest")
if schemaTestElement is not None:
return XmlUtil.descendantAttr(schemaTestElement, None,
"expected", "validity")
resultElement = XmlUtil.descendant(self, None, "result")
if resultElement is not None:
expected = resultElement.get("expected")
if expected and resultElement.get(
"nonStandardErrorCodes") == "true":
# if @expected and @nonStandardErrorCodes then use expected instead of error codes
return expected
errorElements = XmlUtil.descendants(self, None, "error")
resultElement = XmlUtil.descendant(self, None, "result")
if isinstance(errorElements, list) and len(errorElements) > 0:
return [
ModelValue.qname(e, e.stringValue)
if not e.get("nonStandardErrorCodes") else e.stringValue
for e in errorElements
]
#else:
# errorElement = errorElements[1]
# if errorElement is not None and not errorElement.get("nonStandardErrorCodes"):
# _errorText = XmlUtil.text(errorElement)
# if ' ' in _errorText: # list of tokens
# return _errorText
# return ModelValue.qname(errorElement, _errorText) # turn into a QName
if resultElement is not None:
if expected:
return expected
for assertElement in XmlUtil.children(resultElement, None,
"assert"):
num = assertElement.get("num")
if num == "99999": # inline test, use name as expected
return assertElement.get("name")
if len(num) == 5:
return "EFM.{0}.{1}.{2}".format(num[0], num[1:3], num[3:6])
asserTests = {}
for atElt in XmlUtil.children(resultElement, None,
"assertionTests"):
try:
asserTests[atElt.get("assertionID")] = (
_INT(atElt.get("countSatisfied")),
_INT(atElt.get("countNotSatisfied")))
except ValueError:
pass
if asserTests:
return asserTests
elif self.get("result"):
return self.get("result")
return None
def assignedSic(self):
return XmlUtil.text(XmlUtil.descendant(self, edgr, "assignedSic"))
def fiscalYearEnd(self):
yrEnd = XmlUtil.text(XmlUtil.descendant(self, edgr, "fiscalYearEnd"))
if yrEnd and len(yrEnd) == 4:
return "{0}-{1}".format(yrEnd[0:2], yrEnd[2:4])
return None
def checkFilingDTS(val, modelDocument, isEFM, isGFM, visited):
global targetNamespaceDatePattern, efmFilenamePattern, htmlFileNamePattern, roleTypePattern, arcroleTypePattern, \
arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet
if targetNamespaceDatePattern is None:
targetNamespaceDatePattern = re.compile(
r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|"
r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|")
efmFilenamePattern = re.compile(
r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml|\.htm)$")
htmlFileNamePattern = re.compile(
r"^[a-zA-Z0-9][._a-zA-Z0-9-]*(\.htm)$")
roleTypePattern = re.compile(r"^.*/role/[^/\s]+$")
arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$")
arcroleDefinitionPattern = re.compile(
r"^.*[^\\s]+.*$") # at least one non-whitespace character
namePattern = re.compile(
"[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]"
) # u20ac=Euro, u00a3=pound sterling
linkroleDefinitionBalanceIncomeSheet = re.compile(
r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)",
re.IGNORECASE)
nonDomainItemNameProblemPattern = re.compile(
r"({0})|(FirstQuarter|SecondQuarter|ThirdQuarter|FourthQuarter|[1-4]Qtr|Qtr[1-4]|ytd|YTD|HalfYear)(?:$|[A-Z\W])"
.format(re.sub(r"\W", "", (val.entityRegistrantName or "").title())))
visited.append(modelDocument)
for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items(
):
#6.07.01 no includes
if modelDocumentReference.referenceType == "include":
val.modelXbrl.error(
("EFM.6.07.01", "GFM.1.03.01"),
_("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"
),
modelObject=modelDocumentReference.referringModelObject,
schema=modelDocument.basename,
include=referencedDocument.basename)
if referencedDocument not in visited and (
referencedDocument.inDTS or referencedDocument.type
== ModelDocument.Type.INLINEXBRLDOCUMENTSET
): # ignore EdgarRenderer added non-DTS documents
checkFilingDTS(val, referencedDocument, isEFM, isGFM, visited)
if modelDocument.type == ModelDocument.Type.INLINEXBRLDOCUMENTSET:
return # nothing to check in inline document set surrogate parent
if val.disclosureSystem.standardTaxonomiesDict is None:
pass
if isEFM:
if modelDocument.uri in val.disclosureSystem.standardTaxonomiesDict:
if modelDocument.targetNamespace:
# check for duplicates of us-types, dei, and rr taxonomies
match = standardNamespacesPattern.match(
modelDocument.targetNamespace)
if match is not None:
conflictClass = match.group(2) or match.group(5)
if (conflictClass == 'us-gaap' and
match.group(3) < '2018') or conflictClass == 'srt':
val.standardNamespaceConflicts['srt+us-gaap'].add(
modelDocument
) # ignore non-srt multi-usgaap in Filing.py
if conflictClass == 'us-gaap' or conflictClass == 'ifrs-full':
val.standardNamespaceConflicts['ifrs+us-gaap'].add(
modelDocument)
if conflictClass not in ('us-gaap', 'srt'):
val.standardNamespaceConflicts[conflictClass].add(
modelDocument)
else:
if len(modelDocument.basename) > 32:
val.modelXbrl.error(
"EFM.5.01.01.tooManyCharacters",
_("Document file name %(filename)s must not exceed 32 characters."
),
edgarCode="cp-0501-File-Name-Length",
modelObject=modelDocument,
filename=modelDocument.basename)
if modelDocument.type != ModelDocument.Type.INLINEXBRLDOCUMENTSET:
if modelDocument.type == ModelDocument.Type.INLINEXBRL:
_pattern = htmlFileNamePattern
_suffix = ".htm"
else:
_pattern = efmFilenamePattern
_suffix = ".xsd or .xml"
if not _pattern.match(modelDocument.basename):
val.modelXbrl.error(
"EFM.5.01.01",
_("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with %(suffix)s."
),
edgarCode="cp-0501-File-Name",
modelObject=modelDocument,
filename=modelDocument.basename,
suffix=_suffix)
if (modelDocument.type == ModelDocument.Type.SCHEMA
and modelDocument.targetNamespace
not in val.disclosureSystem.baseTaxonomyNamespaces
and modelDocument.uri.startswith(val.modelXbrl.uriDir)):
val.hasExtensionSchema = True
# check schema contents types
# 6.7.3 check namespace for standard authority
targetNamespaceAuthority = UrlUtil.authority(
modelDocument.targetNamespace)
if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities:
val.modelXbrl.error(
("EFM.6.07.03", "GFM.1.03.03"),
_("The target namespace, %(targetNamespace)s cannot have the same authority (%(targetNamespaceAuthority)s) as a standard "
"taxonomy, in %(schema)s. Please change your target namespace."
),
edgarCode="fs-0703-Extension-Has-Standard-Namespace-Authority",
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace,
targetNamespaceAuthority=UrlUtil.authority(
modelDocument.targetNamespace, includeScheme=False))
# 6.7.4 check namespace format
if modelDocument.targetNamespace is None or not modelDocument.targetNamespace.startswith(
"http://"):
match = None
else:
targetNamespaceDate = modelDocument.targetNamespace[
len(targetNamespaceAuthority):]
match = targetNamespaceDatePattern.match(targetNamespaceDate)
if match is not None:
try:
if match.lastindex == 3:
date = datetime.date(int(match.group(1)),
int(match.group(2)),
int(match.group(3)))
elif match.lastindex == 6:
date = datetime.date(int(match.group(4)),
int(match.group(5)),
int(match.group(6)))
else:
match = None
except ValueError:
match = None
if match is None:
val.modelXbrl.error(
("EFM.6.07.04", "GFM.1.03.04"),
_("You did not adhere to the requirements for target namespace, for %(targetNamespace)s in %(schema)s. "
"Please recheck your submission and adhere to the target namespace requirements."
),
edgarCode="cp-0704-Taxonomy-Valid-Target-Namespace",
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace)
elif val.fileNameDate and date > val.fileNameDate:
val.modelXbrl.info(
("EFM.6.07.06", "GFM.1.03.06"),
_("Warning: Taxonomy schema %(schema)s namespace %(targetNamespace)s has date later than document name date %(docNameDate)s"
),
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace,
docNameDate=val.fileNameDate)
if modelDocument.targetNamespace is not None:
# 6.7.5 check prefix for _
authority = UrlUtil.authority(modelDocument.targetNamespace)
if not re.match(r"(http://|https://|ftp://|urn:)\w+", authority):
val.modelXbrl.error(
("EFM.6.07.05", "GFM.1.03.05"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s must be a valid URI with a valid authority for the namespace."
),
edgarCode="du-0705-Namespace-Authority",
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace)
# may be multiple prefixes for namespace
prefixes = [
(prefix if prefix is not None else "")
for prefix, NS in modelDocument.xmlRootElement.nsmap.items()
if NS == modelDocument.targetNamespace
]
if not prefixes:
prefix = None
val.modelXbrl.error(
("EFM.6.07.07", "GFM.1.03.07"),
_("The schema does not supply a prefix for %(targetNamespace)s without an underscore character, in file %(schema)s. "
"Please provide or change a prefix"),
edgarCode="du-0707-Recommended-Prefix-Disallowed",
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace)
else:
prefix = prefixes[0]
if len(prefixes) > 1:
val.modelXbrl.error(
("EFM.6.07.07", "GFM.1.03.07"),
_("The schema does not supply a prefix for %(targetNamespace)s without an underscore character, in file %(schema)s. "
"Please provide or change a prefix"),
edgarCode="du-0707-Recommended-Prefix-Disallowed",
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace,
prefix=", ".join(prefixes))
elif "_" in prefix:
val.modelXbrl.error(
("EFM.6.07.07", "GFM.1.03.07"),
_("The schema does not supply a prefix for %(targetNamespace)s without an underscore character, in file %(schema)s. "
"Please provide or change a prefix"),
edgarCode="du-0707-Recommended-Prefix-Disallowed",
modelObject=modelDocument,
schema=modelDocument.basename,
targetNamespace=modelDocument.targetNamespace,
prefix=prefix)
for modelConcept in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.w3.org/2001/XMLSchema}element"):
if isinstance(modelConcept, ModelConcept):
# 6.7.16 name not duplicated in standard taxonomies
name = modelConcept.get("name")
if name is None:
name = ""
if modelConcept.get("ref") is not None:
continue # don't validate ref's here
for c in val.modelXbrl.nameConcepts.get(name, []):
if c.modelDocument != modelDocument:
if not c.modelDocument.uri.startswith(
val.modelXbrl.uriDir):
val.modelXbrl.error(
("EFM.6.07.16", "GFM.1.03.18"),
_("Your extension taxonomy contains an element, %(concept)s, which has the same name as an element in the base taxonomy, "
"%(standardConcept)s. Please ensure that this extension is appropriate and if so, please change the extension concept name."
),
edgarCode=
"cp-0716-Element-Name-Same-As-Base",
modelObject=(modelConcept, c),
concept=modelConcept.qname,
standardSchema=c.modelDocument.basename,
standardConcept=c.qname)
# 6.7.17 id properly formed
_id = modelConcept.id
requiredId = (prefix
if prefix is not None else "") + "_" + name
if _id != requiredId:
val.modelXbrl.error(
("EFM.6.07.17", "GFM.1.03.19"),
_("You did not adhere to the declarations for concepts by containing an 'id' attribute whose value begins with the recommended "
"namespace prefix of the taxonomy, followed by an underscore, followed by an element name, for the concept %(concept)s. "
"Please recheck your submission."),
edgarCode="cp-0717-Element-Id",
modelObject=modelConcept,
concept=modelConcept.qname,
id=_id,
requiredId=requiredId)
# 6.7.18 nillable is true
nillable = modelConcept.get("nillable")
if nillable != "true" and modelConcept.isItem:
val.modelXbrl.error(
("EFM.6.07.18", "GFM.1.03.20"),
_("Element %(concept)s is declared without a 'true' value for the nillable attribute. Please set the value to 'true'."
),
edgarCode="du-0718-Nillable-Not-True",
modelObject=modelConcept,
schema=modelDocument.basename,
concept=name,
nillable=nillable)
# 6.7.19 not tuple
if modelConcept.isTuple:
val.modelXbrl.error(
("EFM.6.07.19", "GFM.1.03.21"),
_("You provided an extension concept which is a tuple, %(concept)s. Please remove tuples and check your submission."
),
edgarCode="cp-0719-No-Tuple-Element",
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.7.20 no typed domain ref
if modelConcept.isTypedDimension:
val.modelXbrl.error(
("EFM.6.07.20", "GFM.1.03.22"),
_("There is an xbrldt:typedDomainRef attribute on %(concept)s (%(typedDomainRef)s). Please remove it."
),
edgarCode="du-0720-Typed-Domain-Ref-Disallowed",
modelObject=modelConcept,
concept=modelConcept.qname,
typedDomainRef=modelConcept.typedDomainElement.
qname if modelConcept.typedDomainElement
is not None else modelConcept.typedDomainRef)
# 6.7.21 abstract must be duration
isDuration = modelConcept.periodType == "duration"
if modelConcept.isAbstract and not isDuration:
val.modelXbrl.error(
("EFM.6.07.21", "GFM.1.03.23"),
_("Element %(concept)s is declared as an abstract item with period type 'instant'. Please change it to 'duration' or "
"make the element not abstract."),
edgarCode="du-0721-Abstract-Is-Instant",
modelObject=modelConcept,
schema=modelDocument.basename,
concept=modelConcept.qname)
# 6.7.22 abstract must be stringItemType
''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08
if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType:
val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"),
_("Concept %(concept)s is abstract but type is not xbrli:stringItemType"),
modelObject=modelConcept, concept=modelConcept.qname)
'''
substitutionGroupQname = modelConcept.substitutionGroupQname
# 6.7.23 Axis must be subs group dimension
if name.endswith("Axis") ^ (
substitutionGroupQname
== XbrlConst.qnXbrldtDimensionItem):
val.modelXbrl.error(
("EFM.6.07.23", "GFM.1.03.25"),
_("The substitution group 'xbrldt:dimensionItem' is only consistent with an element name that ends with 'Axis'. "
"Please change %(conceptLocalName)s or change the substitutionGroup."
),
edgarCode="du-0723-Axis-Dimension-Name-Mismatch",
modelObject=modelConcept,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName)
# 6.7.24 Table must be subs group hypercube
if name.endswith("Table") ^ (
substitutionGroupQname
== XbrlConst.qnXbrldtHypercubeItem):
val.modelXbrl.error(
("EFM.6.07.24", "GFM.1.03.26"),
_("The substitution group 'xbrldt:hypercubeItem' is only allowed with an element name that ends with 'Table'. "
"Please change %(conceptLocalName)s or change the substitutionGroup."
),
edgarCode="du-0724-Table-Hypercube-Name-Mismatch",
modelObject=modelConcept,
schema=modelDocument.basename,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName)
# 6.7.25 if neither hypercube or dimension, substitution group must be item
if substitutionGroupQname not in (
None, XbrlConst.qnXbrldtDimensionItem,
XbrlConst.qnXbrldtHypercubeItem,
XbrlConst.qnXbrliItem):
val.modelXbrl.error(
("EFM.6.07.25", "GFM.1.03.27"),
_("The substitution group attribute value %(substitutionGroup)s of element %(conceptLocalName)s is not allowed. "
"Please change it to one of 'xbrli:item', 'xbrldt:dimensionItem' or 'xbrldt:hypercubeItem'."
),
edgarCode="du-0725-Substitution-Group-Custom",
modelObject=modelConcept,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName,
substitutionGroup=modelConcept.
substitutionGroupQname)
# 6.7.26 Table must be subs group hypercube
if name.endswith(
"LineItems") and modelConcept.abstract != "true":
val.modelXbrl.error(
("EFM.6.07.26", "GFM.1.03.28"),
_("The element %(conceptLocalName)s ends with 'LineItems' but is not abstract. Please change %(conceptLocalName)s or "
"the value of the 'abstract' attribute."),
edgarCode=
"du-0726-LineItems-Abstract-Name-Mismatch",
modelObject=modelConcept,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName)
# 6.7.27 type domainMember must end with Domain or Member
conceptType = modelConcept.type
isDomainItemType = conceptType is not None and conceptType.isDomainItemType
endsWithDomainOrMember = name.endswith(
"Domain") or name.endswith("Member")
if isDomainItemType != endsWithDomainOrMember:
val.modelXbrl.error(
("EFM.6.07.27", "GFM.1.03.29"),
_("The type 'us-types:domainItemType' is only allowed with an element name that ends with 'Domain' or 'Member'. "
"Please change %(conceptLocalName)s or change the type."
),
edgarCode="du-0727-Domain-Type-Name-Mismatch",
modelObject=modelConcept,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName)
# 6.7.28 domainItemType must be duration
if isDomainItemType and not isDuration:
val.modelXbrl.error(
("EFM.6.07.28", "GFM.1.03.30"),
_("Element %(conceptLocalName)s is declared as a us-types:domainItemType with period type 'instant'. "
"Please change it to 'duration' or change the item type."
),
edgarCode="du-0728-Domain-Member-Is-Instant",
modelObject=modelConcept,
concept=modelConcept.qname,
conceptLocalName=modelConcept.qname.localName)
#6.7.31 (version 27) fractions
if modelConcept.isFraction:
val.modelXbrl.error(
"EFM.6.07.31",
_("Element %(concept)s is declared as a fraction item type. Change or remove the declaration."
),
edgarCode="du-0731-Fraction-Item-Type",
modelObject=modelConcept,
concept=modelConcept.qname)
#6.7.32 (version 27) instant non numeric
if modelConcept.isItem and (not modelConcept.isNumeric
and not isDuration
and not modelConcept.isAbstract
and not isDomainItemType):
val.modelXbrl.error(
"EFM.6.07.32",
_("Declaration of element %(concept)s in %(schema)s must have xbrli:periodType of 'duration' because its base type is not numeric."
),
edgarCode="rq-0732-Nonnnumeric-Must-Be-Duration",
modelObject=modelConcept,
schema=modelDocument.basename,
concept=modelConcept.qname)
# 6.8.5 semantic check, check LC3 name
if name:
if not name[0].isupper():
val.modelXbrl.log(
"ERROR-SEMANTIC", ("EFM.6.08.05.firstLetter",
"GFM.2.03.05.firstLetter"),
_("Concept %(concept)s name must start with a capital letter"
),
modelObject=modelConcept,
concept=modelConcept.qname)
if namePattern.search(name):
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.05.disallowedCharacter",
"GFM.2.03.05.disallowedCharacter"),
_("Concept %(concept)s has disallowed name character"
),
modelObject=modelConcept,
concept=modelConcept.qname)
if len(name) > 200:
val.modelXbrl.log(
"ERROR-SEMANTIC",
"EFM.6.08.05.nameLength",
_("Concept %(concept)s name length %(namelength)s exceeds 200 characters"
),
modelObject=modelConcept,
concept=modelConcept.qname,
namelength=len(name))
if isEFM:
label = modelConcept.label(lang="en-US",
fallbackToQname=False)
if label:
# allow Joe's Bar, N.A. to be JoesBarNA -- remove ', allow A. as not article "a"
lc3name = ''.join(
re.sub(r"['.-]", "", (
w[0] or w[2] or w[3] or w[4])).title()
for w in re.findall(
r"((\w+')+\w+)|(A[.-])|([.-]A(?=\W|$))|(\w+)",
label
) # EFM implies this should allow - and . re.findall(r"[\w\-\.]+", label)
if w[4].lower() not in ("the", "a", "an"))
if not (name == lc3name or
(name and lc3name and lc3name[0].isdigit()
and name[1:] == lc3name and
(name[0].isalpha() or name[0] == '_'))):
val.modelXbrl.log(
"WARNING-SEMANTIC",
"EFM.6.08.05.LC3",
_("Concept %(concept)s should match expected LC3 composition %(lc3name)s"
),
modelObject=modelConcept,
concept=modelConcept.qname,
lc3name=lc3name)
if conceptType is not None:
# 6.8.6 semantic check
if not isDomainItemType and conceptType.qname != XbrlConst.qnXbrliDurationItemType:
nameProblems = nonDomainItemNameProblemPattern.findall(
name)
if any(
any(t) for t in nameProblems
): # list of tuples with possibly nonempty strings
val.modelXbrl.log(
"WARNING-SEMANTIC",
("EFM.6.08.06", "GFM.2.03.06"),
_("Concept %(concept)s should not contain company or period information, found: %(matches)s"
),
modelObject=modelConcept,
concept=modelConcept.qname,
matches=", ".join(''.join(t)
for t in nameProblems))
if conceptType.qname == XbrlConst.qnXbrliMonetaryItemType:
if not modelConcept.balance:
# 6.8.11 may not appear on a income or balance statement
if any(
linkroleDefinitionBalanceIncomeSheet.
match(roleType.definition)
for rel in val.modelXbrl.
relationshipSet(XbrlConst.parentChild).
toModelObject(modelConcept)
for roleType in val.modelXbrl.
roleTypes.get(rel.linkrole, ())):
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.11", "GFM.2.03.11"),
_("Concept %(concept)s must have a balance because it appears in a statement of income or balance sheet"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.11.5 semantic check, must have a documentation label
stdLabel = modelConcept.label(
lang="en-US", fallbackToQname=False)
defLabel = modelConcept.label(
preferredLabel=XbrlConst.
documentationLabel,
lang="en-US",
fallbackToQname=False)
if not defLabel or ( # want different words than std label
stdLabel
and re.findall(r"\w+", stdLabel)
== re.findall(r"\w+", defLabel)):
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.11.05", "GFM.2.04.04"),
_("Concept %(concept)s is monetary without a balance and must have a documentation label that disambiguates its sign"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.8.16 semantic check
if conceptType.qname == XbrlConst.qnXbrliDateItemType and modelConcept.periodType != "duration":
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.16", "GFM.2.03.16"),
_("Concept %(concept)s of type xbrli:dateItemType must have periodType duration"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.8.17 semantic check
if conceptType.qname == XbrlConst.qnXbrliStringItemType and modelConcept.periodType != "duration":
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.17", "GFM.2.03.17"),
_("Concept %(concept)s of type xbrli:stringItemType must have periodType duration"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.7.8 check for embedded linkbase
for e in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
if isinstance(e, ModelObject):
val.modelXbrl.error(
("EFM.6.07.08", "GFM.1.03.08"),
_("Your filing contained embedded linkbases in %(schema)s. Please recheck your submission and remove all embedded linkbases."
),
edgarCode="cp-0708-No-Embedded-Linkbases",
modelObject=e,
schema=modelDocument.basename)
break
requiredUsedOns = {
XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink,
XbrlConst.qnLinkDefinitionLink
}
standardUsedOns = {
XbrlConst.qnLinkLabel,
XbrlConst.qnLinkReference,
XbrlConst.qnLinkDefinitionArc,
XbrlConst.qnLinkCalculationArc,
XbrlConst.qnLinkPresentationArc,
XbrlConst.qnLinkLabelArc,
XbrlConst.qnLinkReferenceArc,
# per WH, private footnote arc and footnore resource roles are not allowed
XbrlConst.qnLinkFootnoteArc,
XbrlConst.qnLinkFootnote,
}
# 6.7.9 role types authority
for e in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.xbrl.org/2003/linkbase}roleType"):
if isinstance(e, ModelObject):
roleURI = e.get("roleURI")
if targetNamespaceAuthority != UrlUtil.authority(roleURI):
val.modelXbrl.error(
("EFM.6.07.09", "GFM.1.03.09"),
_("Role %(roleType)s does not begin with %(targetNamespace)s's scheme and authority. "
"Please change the role URI or target namespace URI."
),
edgarCode="du-0709-Role-Namespace-Mismatch",
modelObject=e,
roleType=roleURI,
targetNamespaceAuthority=targetNamespaceAuthority,
targetNamespace=modelDocument.targetNamespace)
# 6.7.9 end with .../role/lc3 name
if not roleTypePattern.match(roleURI):
val.modelXbrl.warning(
("EFM.6.07.09.roleEnding", "GFM.1.03.09"),
"RoleType %(roleType)s should end with /role/{LC3name}",
modelObject=e,
roleType=roleURI)
# 6.7.10 only one role type declaration in DTS
modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI)
if modelRoleTypes is not None:
modelRoleType = modelRoleTypes[0]
definition = modelRoleType.definitionNotStripped
usedOns = modelRoleType.usedOns
if len(modelRoleTypes) == 1:
# 6.7.11 used on's for pre, cal, def if any has a used on
if not usedOns.isdisjoint(requiredUsedOns) and len(
requiredUsedOns - usedOns) > 0:
val.modelXbrl.error(
("EFM.6.07.11", "GFM.1.03.11"),
_("The role %(roleType)s did not provide a usedOn element for all three link types (presentation, "
"calculation and definition), missing %(usedOn)s. Change the declaration to be for all three types of link, and resubmit."
),
edgarCode=
"du-0711-Role-Type-Declaration-Incomplete",
modelObject=e,
roleType=roleURI,
usedOn=requiredUsedOns - usedOns)
# 6.7.12 definition match pattern
if (val.disclosureSystem.roleDefinitionPattern
is not None and
(definition is None or not val.disclosureSystem.
roleDefinitionPattern.match(definition))):
val.modelXbrl.error(
("EFM.6.07.12", "GFM.1.03.12-14"),
_("The definition '%(definition)s' of role %(roleType)s does not match the expected format. "
"Please check that the definition matches {number} - {type} - {text}."
),
edgarCode="rq-0712-Role-Definition-Mismatch",
modelObject=e,
roleType=roleURI,
definition=(definition or ""))
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log(
"ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("RoleType %(roleuri)s is defined using role types already defined by standard roles for: %(qnames)s"
),
modelObject=e,
roleuri=roleURI,
qnames=', '.join(
str(qn) for qn in usedOns & standardUsedOns))
# 6.7.13 arcrole types authority
for e in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.xbrl.org/2003/linkbase}arcroleType"):
if isinstance(e, ModelObject):
arcroleURI = e.get("arcroleURI")
if targetNamespaceAuthority != UrlUtil.authority(arcroleURI):
val.modelXbrl.error(
("EFM.6.07.13", "GFM.1.03.15"),
_("Relationship role %(arcroleType)s does not begin with %(targetNamespace)s's scheme and authority. "
"Please change the relationship role URI or target namespace URI."
),
edgarCode="du-0713-Arcrole-Namespace-Mismatch",
modelObject=e,
arcroleType=arcroleURI,
targetNamespaceAuthority=targetNamespaceAuthority,
targetNamespace=modelDocument.targetNamespace)
# 6.7.13 end with .../arcrole/lc3 name
if not arcroleTypePattern.match(arcroleURI):
val.modelXbrl.warning(
("EFM.6.07.13.arcroleEnding", "GFM.1.03.15"),
_("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"
),
modelObject=e,
arcroleType=arcroleURI)
# 6.7.15 definition match pattern
modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI]
definition = modelRoleTypes[0].definition
if definition is None or not arcroleDefinitionPattern.match(
definition):
val.modelXbrl.error(
("EFM.6.07.15", "GFM.1.03.17"),
_("Relationship role declaration %(arcroleType)s is missing a definition. Please provide a definition."
),
edgarCode="du-0715-Arcrole-Definition-Missing",
modelObject=e,
arcroleType=arcroleURI)
# semantic checks
usedOns = modelRoleTypes[0].usedOns
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log(
"ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("ArcroleType %(arcroleuri)s is defined using role types already defined by standard arcroles for: %(qnames)s"
),
modelObject=e,
arcroleuri=arcroleURI,
qnames=', '.join(
str(qn) for qn in usedOns & standardUsedOns))
#6.3.3 filename check
m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9]).xsd$",
modelDocument.basename)
if m:
try: # check date value
datetime.datetime.strptime(m.group(1), "%Y%m%d").date()
# date and format are ok, check "should" part of 6.3.3
if val.fileNameBasePart:
expectedFilename = "{0}-{1}.xsd".format(
val.fileNameBasePart, val.fileNameDatePart)
if modelDocument.basename != expectedFilename:
val.modelXbrl.log(
"WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance",
"GFM.1.01.01.matchInstance"),
_('Schema file name warning: %(filename)s, should match %(expectedFilename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
expectedFilename=expectedFilename)
except ValueError:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid schema file base name part (date) in "{base}-{yyyymmdd}.xsd": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
else:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid schema file name, must match "{base}-{yyyymmdd}.xsd": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
elif modelDocument.type == ModelDocument.Type.LINKBASE:
# if it is part of the submission (in same directory) check name
labelRels = None
if modelDocument.filepath.startswith(
val.modelXbrl.modelDocument.filepathdir):
#6.3.3 filename check
extLinkElt = XmlUtil.descendant(
modelDocument.xmlRootElement, XbrlConst.link, "*",
"{http://www.w3.org/1999/xlink}type", "extended")
if extLinkElt is None: # no ext link element
val.modelXbrl.error(
(val.EFM60303 + ".noLinkElement",
"GFM.1.01.01.noLinkElement"),
_('Invalid linkbase file name: %(filename)s, has no extended link element, cannot determine link type.'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03.noLinkElement",
"EFM.6.23.01.noLinkElement",
"GFM.1.01.01.noLinkElement"))
elif extLinkElt.localName not in extLinkEltFileNameEnding:
val.modelXbrl.error(
"EFM.6.03.02",
_('Invalid linkbase link element %(linkElement)s in %(filename)s'
),
modelObject=modelDocument,
linkElement=extLinkElt.localName,
filename=modelDocument.basename)
else:
m = re.match(
r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9])(_[a-z]{3}).xml$",
modelDocument.basename)
expectedSuffix = extLinkEltFileNameEnding[extLinkElt.localName]
if m and m.group(2) == expectedSuffix:
try: # check date value
datetime.datetime.strptime(m.group(1), "%Y%m%d").date()
# date and format are ok, check "should" part of 6.3.3
if val.fileNameBasePart:
expectedFilename = "{0}-{1}{2}.xml".format(
val.fileNameBasePart, val.fileNameDatePart,
expectedSuffix)
if modelDocument.basename != expectedFilename:
val.modelXbrl.log(
"WARNING-SEMANTIC",
("EFM.6.03.03.matchInstance",
"GFM.1.01.01.matchInstance"),
_('Linkbase name warning: %(filename)s should match %(expectedFilename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
expectedFilename=expectedFilename)
except ValueError:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid linkbase base file name part (date) in "{base}-{yyyymmdd}_{suffix}.xml": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01",
"GFM.1.01.01"))
else:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid linkbase name, must match "{base}-{yyyymmdd}%(expectedSuffix)s.xml": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
expectedSuffix=expectedSuffix,
messageCodes=("EFM.6.03.03", "EFM.6.23.01",
"GFM.1.01.01"))
if extLinkElt.localName == "labelLink":
if labelRels is None:
labelRels = val.modelXbrl.relationshipSet(
XbrlConst.conceptLabel)
for labelElt in XmlUtil.children(extLinkElt,
XbrlConst.link, "label"):
# 6.10.9
if XbrlConst.isNumericRole(labelElt.role):
for rel in labelRels.toModelObject(labelElt):
if rel.fromModelObject is not None and not rel.fromModelObject.isNumeric:
val.modelXbrl.error(
"EFM.6.10.09",
_("Non-numeric element %(concept)s has a label role for numeric elements: %(role)s. "
"Please change the role attribute."),
edgarCode="du-1009-Numeric-Label-Role",
modelObject=(labelElt,
rel.fromModelObject),
concept=rel.fromModelObject.qname,
role=labelElt.role)
def loadFromExcel(cntlr, excelFile):
from arelle import xlrd
from arelle.xlrd.sheet import empty_cell
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
startedAt = time.time()
importExcelBook = xlrd.open_workbook(excelFile)
controlSheet = importExcelBook.sheet_by_index(1)
imports = {"xbrli": ( ("namespace", XbrlConst.xbrli),
("schemaLocation", "http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd") )} # xml of imports
importXmlns = {}
linkbaseRefs = []
labelLinkbases = []
hasPreLB = hasCalLB = hasDefLB = False
# xxxLB structure [ (elr1, def1, "_ELR_", [roots]), (elr2, def2, "_ELR_", [rootw]) ...]
# roots = (rootHref, None, "_root_", [children])
# children = (childPrefix, childName, arcrole, [grandChildren])
preLB = []
defLB = []
calLB = []
def lbDepthList(lbStruct, depth, parentList=None):
if depth == 0:
return lbStruct[-1].childStruct
return lbDepthList(lbStruct[-1].childStruct, depth-1, list)
extensionElements = {}
extensionRoles = {} # key is roleURI, value is role definition
extensionLabels = {} # key = (prefix, name, lang, role), value = label text
def extensionHref(prefix, name):
if prefix == extensionSchemaPrefix:
filename = extensionSchemaFilename
elif prefix in imports:
filename = imports[prefix][1][1]
else:
return None
return "{0}#{1}_{2}".format(filename, prefix, name)
isUSGAAP = False
for iRow in range(1, controlSheet.nrows):
try:
row = controlSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY): # skip if col 1 is empty
continue
action = row[0].value
filetype = row[1].value
prefix = row[2].value
filename = row[3].value
namespaceURI = row[4].value
lbType = lang = None
if action == "import":
imports[prefix] = ( ("namespace", namespaceURI), ("schemaLocation", filename) )
importXmlns[prefix] = namespaceURI
if re.match(r"http://[^/]+/us-gaap/", namespaceURI):
isUSGAAP = True
elif action == "extension":
if filetype == "schema":
extensionSchemaPrefix = prefix
extensionSchemaFilename = filename
extensionSchemaNamespaceURI = namespaceURI
elif filetype == "linkbase":
typeLang = prefix.split()
if len(typeLang) > 0:
lbType = typeLang[0]
else:
lbType = "unknown"
if len(typeLang) > 1:
lang = typeLang[1]
else:
lang = "en"
if lbType == "label":
labelLinkbases.append((lang, filename))
elif lbType == "presentation":
hasPreLB = True
elif lbType == "definition":
hasDefLB = True
elif lbType == "calculation":
hasCalLB = True
linkbaseRefs.append( (lbType, filename) )
elif filetype == "role" and namespaceURI:
extensionRoles[namespaceURI] = filename
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:exception")
importExcelSheet = importExcelBook.sheet_by_index(0)
# find column headers row
headerCols = {}
headerRows = set()
# find out which rows are header rows
for iRow in range(0, importExcelSheet.nrows):
row = importExcelSheet.row(iRow)
for iCol, colCell in enumerate(row):
v = colCell.value
if v in importColumnHeaders:
headerCols[importColumnHeaders[v]] = iCol
if all(colName in headerCols
for colName in ("name", "type", "depth")): # must have these to be a header col
# it's a header col
headerRows.add(iRow)
headerCols.clear()
def cellValue(row, header):
if header in headerCols:
iCol = headerCols[header]
if iCol < len(row):
return row[iCol].value
return ''
def checkImport(qname):
prefix, sep, localName = qname.partition(":")
if sep:
if prefix not in imports:
if prefix == "xbrldt":
imports["xbrldt"] = ("namespace", XbrlConst.xbrldt), ("schemaLocation", "http://www.xbrl.org/2005/xbrldt-2005.xsd")
elif prefix == "nonnum":
imports["nonnum"] = ("namespace", "http://www.xbrl.org/dtr/type/non-numeric"), ("schemaLocation", "http://www.xbrl.org/dtr/type/nonNumeric-2009-12-16.xsd")
else:
cntlr.addToLog("Warning: prefix schema file is not imported for: {qname}"
.format(qname=qname),
messageCode="importExcel:warning")
# find header rows
currentELR = currentELRdefinition = None
for iRow in range(0, importExcelSheet.nrows):
useLabels = False
try:
row = importExcelSheet.row(iRow)
isHeaderRow = iRow in headerRows
isELRrow = (iRow + 1) in headerRows
if isHeaderRow:
headerCols.clear()
for iCol, colCell in enumerate(row):
v = colCell.value
if v in importColumnHeaders:
headerCols[importColumnHeaders[v]] = iCol
elif isELRrow:
currentELR = currentELRdefinition = None
for colCell in row:
v = colCell.value
if v.startswith("http://"):
currentELR = v
elif not currentELRdefinition and v.endswith(" 科目一覧"):
currentELRdefinition = v[0:-5]
elif not currentELRdefinition:
currentELRdefinition = v
if currentELR or currentELRdefinition:
if hasPreLB:
preLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasDefLB:
defLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasCalLB:
calLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
elif headerCols:
prefix = cellValue(row, 'prefix').strip()
name = cellValue(row, 'name').strip()
if "depth" in headerCols:
try:
depth = int(cellValue(row, 'depth'))
except ValueError:
depth = None
else:
depth = None
if prefix == extensionSchemaPrefix and name not in extensionElements:
# elements row
eltType = cellValue(row, 'type')
subsGrp = cellValue(row, 'substitutionGroup')
abstract = cellValue(row, 'abstract')
nillable = cellValue(row, 'nillable')
balance = cellValue(row, 'balance')
periodType = cellValue(row, 'periodType')
newElt = [ ("name", name), ("id", prefix + "_" + name) ]
if eltType:
newElt.append( ("type", eltType) )
checkImport(eltType)
if subsGrp:
newElt.append( ("substitutionGroup", subsGrp) )
checkImport(subsGrp)
if abstract:
newElt.append( ("abstract", abstract) )
if nillable:
newElt.append( ("nillable", nillable) )
if balance:
newElt.append( ("{http://www.xbrl.org/2003/instance}balance", balance) )
if periodType:
newElt.append( ("{http://www.xbrl.org/2003/instance}periodType", periodType) )
extensionElements[name] = newElt
useLabels = True
if depth is not None:
if hasPreLB:
entryList = lbDepthList(preLB, depth)
preferredLabel = cellValue(row, 'preferredLabel')
if preferredLabel and not preferredLabel.startswith("http://"):
preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel
if entryList is not None:
if depth == 0:
entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) )
else:
entryList.append( LBentry(prefix=prefix, name=name, arcrole=XbrlConst.parentChild,
role=preferredLabel) )
if hasDefLB:
entryList = lbDepthList(defLB, depth)
if entryList is not None:
if depth == 0:
entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) )
else:
entryList.append( LBentry(prefix=prefix, name=name, arcrole="_dimensions_") )
if hasCalLB:
calcParent = cellValue(row, 'calculationParent')
calcWeight = cellValue(row, 'calculationWeight')
if calcParent and calcWeight:
calcParentPrefix, sep, calcParentName = calcParent.partition(":")
entryList = lbDepthList(calLB, 0)
if entryList is not None:
entryList.append( LBentry(prefix=calcParentPrefix, name=calcParentName, isRoot=True, childStruct=
[LBentry(prefix=prefix, name=name, arcrole=XbrlConst.summationItem, weight=calcWeight )]) )
# accumulate extension labels
if useLabels:
prefix = cellValue(row, 'prefix').strip()
name = cellValue(row, 'name').strip()
preferredLabel = cellValue(row, 'preferredLabel')
if preferredLabel and not preferredLabel.startswith("http://"):
preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel
for colItem, iCol in headerCols.items():
if isinstance(colItem, tuple):
colItemType, role, lang = colItem
cell = row[iCol]
if cell.ctype == xlrd.XL_CELL_EMPTY:
values = ()
elif colItemType == "label":
values = (cell.value,)
elif colItemType == "labels":
values = cell.value.split('\n')
else:
values = ()
if preferredLabel: # first label column sets preferredLabel if any
role = preferredLabel
preferredLabel = None
for value in values:
extensionLabels[prefix, name, lang, role] = value.strip()
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:exception")
if isUSGAAP and hasDefLB:
# move line items above table
def fixUsggapTableDims(lvl1Struct):
foundLineItems = False
for i1, lvl1Entry in enumerate(lvl1Struct):
for i2, lvl2Entry in enumerate(lvl1Entry.childStruct):
for i3, lvl3Entry in enumerate(lvl2Entry.childStruct):
if lvl3Entry.name.endswith("LineItems") and lvl2Entry.name.endswith("Table"):
foundLineItems = True
break
if foundLineItems:
break
else:
fixUsggapTableDims(lvl1Entry.childStruct)
if foundLineItems:
lvl1Struct.insert(i1 + 1, LBentry(prefix=lvl3Entry.prefix, name=lvl3Entry.name, arcrole=lvl1Entry.arcrole, childStruct=lvl3Entry.childStruct)) # must keep lvl1Rel if it is __root__
lvl3Entry.childStruct.insert(0, lvl2Entry)
if lvl1Entry.name.endswith("Abstract"):
del lvl1Struct[i1]
del lvl2Entry.childStruct[i3]
pass
fixUsggapTableDims(defLB)
dts = cntlr.modelManager.create(newDocumentType=ModelDocument.Type.SCHEMA,
url=extensionSchemaFilename,
isEntry=True,
base='', # block pathname from becomming absolute
initialXml='''
<schema xmlns="http://www.w3.org/2001/XMLSchema"
targetNamespace="{targetNamespace}"
attributeFormDefault="unqualified"
elementFormDefault="qualified"
xmlns:xsd="http://www.w3.org/2001/XMLSchema"
xmlns:{extensionPrefix}="{targetNamespace}"
{importXmlns}
xmlns:iod="http://disclosure.edinet-fsa.go.jp/taxonomy/common/2013-03-31/iod"
xmlns:nonnum="http://www.xbrl.org/dtr/type/non-numeric"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xbrli="http://www.xbrl.org/2003/instance"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrldt="http://xbrl.org/2005/xbrldt"/>
'''.format(targetNamespace=extensionSchemaNamespaceURI,
extensionPrefix=extensionSchemaPrefix,
importXmlns=''.join('xmlns:{0}="{1}"\n'.format(prefix, namespaceURI)
for prefix, namespaceURI in importXmlns.items())
)
)
dtsSchemaDocument = dts.modelDocument
dtsSchemaDocument.inDTS = True # entry document always in DTS
dtsSchemaDocument.targetNamespace = extensionSchemaNamespaceURI # not set until schemaDiscover too late otherwise
schemaElt = dtsSchemaDocument.xmlRootElement
#foreach linkbase
annotationElt = XmlUtil.addChild(schemaElt, XbrlConst.xsd, "annotation")
appinfoElt = XmlUtil.addChild(annotationElt, XbrlConst.xsd, "appinfo")
for iRow in range(0, importExcelSheet.nrows):
try:
row = importExcelSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY): # skip if col 1 is empty
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="loadFromExcel:exception")
# add linkbaseRefs
appinfoElt = XmlUtil.descendant(schemaElt, XbrlConst.xsd, "appinfo")
# don't yet add linkbase refs, want to process imports first to get roleType definitions
# add imports
for importAttributes in sorted(imports.values()):
XmlUtil.addChild(schemaElt,
XbrlConst.xsd, "import",
attributes=importAttributes)
# add elements
for eltName, eltAttrs in sorted(extensionElements.items(), key=lambda item: item[0]):
XmlUtil.addChild(schemaElt,
XbrlConst.xsd, "element",
attributes=eltAttrs)
# add role definitions (for discovery)
for roleURI, roleDefinition in extensionRoles.items():
roleElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "roleType",
attributes=(("roleURI", roleURI),
("id", "roleType_" + roleURI.rpartition("/")[2])))
if roleDefinition:
XmlUtil.addChild(roleElt, XbrlConst.link, "definition", text=roleDefinition)
if hasPreLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:presentationLink")
if hasDefLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:definitionLink")
if hasCalLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:calculationLink")
dtsSchemaDocument.schemaDiscover(schemaElt, False, extensionSchemaNamespaceURI)
def addLinkbaseRef(lbType, lbFilename, lbDoc):
role = "http://www.xbrl.org/2003/role/{0}LinkbaseRef".format(lbType)
lbRefElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "linkbaseRef",
attributes=(("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", lbFilename),
("{http://www.w3.org/1999/xlink}role", role),
("{http://www.w3.org/1999/xlink}arcrole", "http://www.w3.org/1999/xlink/properties/linkbase"),
))
dtsSchemaDocument.referencesDocument[lbDoc] = ModelDocumentReference("href", lbRefElt)
# label linkbase
for lang, filename in labelLinkbases:
lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base="", initialXml="""
<link:linkbase
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase
http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance"/>
""")
lbDoc.inDTS = True
addLinkbaseRef("label", filename, lbDoc)
lbElt = lbDoc.xmlRootElement
linkElt = XmlUtil.addChild(lbElt,
XbrlConst.link, "labelLink",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/link")))
locs = set()
for labelKey, text in extensionLabels.items():
prefix, name, labelLang, role = labelKey
if lang == labelLang:
locLabel = prefix + "_" + name
if locLabel not in locs:
locs.add(locLabel)
XmlUtil.addChild(linkElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", extensionHref(prefix, name)),
("{http://www.w3.org/1999/xlink}label", locLabel)))
XmlUtil.addChild(linkElt,
XbrlConst.link, "labelArc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", "http://www.xbrl.org/2003/arcrole/concept-label"),
("{http://www.w3.org/1999/xlink}from", locLabel),
("{http://www.w3.org/1999/xlink}to", "label_" + locLabel),
("order", 1.0)))
XmlUtil.addChild(linkElt,
XbrlConst.link, "label",
attributes=(("{http://www.w3.org/1999/xlink}type", "resource"),
("{http://www.w3.org/1999/xlink}label", "label_" + locLabel),
("{http://www.w3.org/1999/xlink}role", role),
("{http://www.w3.org/XML/1998/namespace}lang", lang)),
text=text)
lbDoc.linkbaseDiscover(lbElt)
def hrefConcept(prefix, name):
qn = schemaElt.prefixedNameQname(prefix + ":" + name)
if qn in dts.qnameConcepts:
return dts.qnameConcepts[qn]
return None
def lbTreeWalk(lbType, parentElt, lbStruct, roleRefs, locs=None, fromPrefix=None, fromName=None):
order = 1.0
for lbEntry in lbStruct:
if lbEntry.isELR:
role = "unspecified"
if lbEntry.role and lbEntry.role.startswith("http://"): # have a role specified
role = lbEntry.role
elif lbEntry.name: #may be a definition
for linkroleUri, modelRoleTypes in dts.roleTypes.items():
definition = modelRoleTypes[0].definition
if lbEntry.name == definition:
role = linkroleUri
break
if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef
roleType = modelRoleTypes[0]
roleRef = ("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id)
roleRefs.add(roleRef)
linkElt = XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Link",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", role)))
locs = set()
lbTreeWalk(lbType, linkElt, lbEntry.childStruct, roleRefs, locs)
else:
toPrefix = lbEntry.prefix
toName = lbEntry.name
toHref = extensionHref(toPrefix, toName)
toLabel = toPrefix + "_" + toName
if toHref not in locs:
XmlUtil.addChild(parentElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", toHref),
("{http://www.w3.org/1999/xlink}label", toLabel)))
locs.add(toHref)
if not lbEntry.isRoot:
fromLabel = fromPrefix + "_" + fromName
if lbType == "calculation" and lbEntry.weight is not None:
otherAttrs = ( ("weight", lbEntry.weight), )
elif lbType == "presentation" and lbEntry.role is not None:
otherAttrs = ( ("preferredLabel", lbEntry.role), )
else:
otherAttrs = ( )
if lbEntry.arcrole == "_dimensions_": # pick proper consecutive arcrole
fromConcept = hrefConcept(fromPrefix, fromName)
toConcept = hrefConcept(toPrefix, toName)
if toConcept is not None and toConcept.isHypercubeItem:
arcrole = XbrlConst.all
elif toConcept is not None and toConcept.isDimensionItem:
arcrole = XbrlConst.hypercubeDimension
elif fromConcept is not None and fromConcept.isDimensionItem:
arcrole = XbrlConst.dimensionDomain
else:
arcrole = XbrlConst.domainMember
else:
arcrole = lbEntry.arcrole
XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Arc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", arcrole),
("{http://www.w3.org/1999/xlink}from", fromLabel),
("{http://www.w3.org/1999/xlink}to", toLabel),
("order", order)) + otherAttrs )
order += 1.0
if lbType != "calculation" or lbEntry.isRoot:
lbTreeWalk(lbType, parentElt, lbEntry.childStruct, roleRefs, locs, toPrefix, toName)
for hasLB, lbType, lbLB in ((hasPreLB, "presentation", preLB),
(hasDefLB, "definition", defLB),
(hasCalLB, "calculation", calLB)):
if hasLB:
for lbRefType, filename in linkbaseRefs:
if lbType == lbRefType:
# output presentation linkbase
lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base='', initialXml="""
<link:linkbase
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase
http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance"/>
""")
lbDoc.inDTS = True
addLinkbaseRef(lbRefType, filename, lbDoc)
lbElt = lbDoc.xmlRootElement
roleRefs = set()
if lbType == "definition":
roleRefs.update((("arcroleRef", XbrlConst.all, "http://www.xbrl.org/2005/xbrldt-2005.xsd#all"),
("arcroleRef", XbrlConst.dimensionDefault, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-default"),
("arcroleRef", XbrlConst.dimensionDomain, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-domain"),
("arcroleRef", XbrlConst.domainMember, "http://www.xbrl.org/2005/xbrldt-2005.xsd#domain-member"),
("arcroleRef", XbrlConst.hypercubeDimension, "http://www.xbrl.org/2005/xbrldt-2005.xsd#hypercube-dimension")))
lbTreeWalk(lbType, lbElt, lbLB, roleRefs)
firstLinkElt = None
for firstLinkElt in lbElt.iterchildren():
break
# add arcrole references
for roleref, roleURI, href in roleRefs:
XmlUtil.addChild(lbElt,
XbrlConst.link, roleref,
attributes=(("arcroleURI" if roleref == "arcroleRef" else "roleURI", roleURI),
("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", href)),
beforeSibling=firstLinkElt)
lbDoc.linkbaseDiscover(lbElt)
break
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
return dts
def generateUpdatedTableLB(dts, updatedTableLinkbaseFile):
import os, io
from arelle import XmlUtil, XbrlConst
from arelle.ViewUtil import viewReferences, referenceURI
from arelle.ModelRenderingObject import ModelEuAxisCoord
if dts.fileSource.isArchive:
dts.error(
"genTblLB:outFileIsArchive",
_("Updated Table Linkbase file cannot be an archive: %(tableLBOutputFile)s."
),
modelObject=dts,
tableLBOutputFile=updatedTableLinkbaseFile)
return
tblAxisRelSet = dts.relationshipSet(XbrlConst.euTableAxis)
axisMbrRelSet = dts.relationshipSet(XbrlConst.euAxisMember)
if len(tblAxisRelSet.modelRelationships) == 0:
dts.error(
"genTblLB:noInputTables",
_("DTS does not contain Eurofiling 2010 tables and axes: %(entryFile)s."
),
modelObject=dts,
entryFile=dts.uri)
return
file = io.StringIO('''
<nsmap>
<link:linkbase
xmlns:label="http://xbrl.org/2008/label"
xmlns:gen="http://xbrl.org/2008/generic"
xmlns:df="http://xbrl.org/2008/filter/dimension"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:reference="http://xbrl.org/2008/reference"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:table="http://xbrl.org/2011/table"
xmlns:formula="http://xbrl.org/2008/formula"
xsi:schemaLocation="
http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd
http://xbrl.org/2008/generic http://www.xbrl.org/2008/generic-link.xsd
http://xbrl.org/2008/reference http://www.xbrl.org/2008/generic-reference.xsd
http://xbrl.org/2008/label http://www.xbrl.org/2008/generic-label.xsd
http://xbrl.org/2011/table http://www.xbrl.org/2011/table.xsd
http://xbrl.org/2008/filter/dimension http://www.xbrl.org/2008/dimension-filter.xsd">
<link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/table-filter" xlink:type="simple"
xlink:href="http://www.xbrl.org/2011/table.xsd#table-filter"/>
<link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/table-axis" xlink:type="simple"
xlink:href="http://www.xbrl.org/2011/table.xsd#table-axis"/>
<link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/axis-subtree" xlink:type="simple"
xlink:href="http://www.xbrl.org/2011/table.xsd#axis-subtree"/>
<link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/axis-filter" xlink:type="simple"
xlink:href="http://www.xbrl.org/2011/filter-axis.xsd#axis-filter"/>
</link:linkbase>
</nsmap>
<!-- Generated by Arelle(r) http://arelle.org -->
''')
from arelle.ModelObjectFactory import parser
parser, parserLookupName, parserLookupClass = parser(dts, None)
from lxml import etree
xmlDocument = etree.parse(file,
parser=parser,
base_url=updatedTableLinkbaseFile)
file.close()
nsmapElt = xmlDocument.getroot()
#xmlDocument.getroot().init(self) ## is this needed ??
for lbElement in xmlDocument.iter(
tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
break
class DocObj: # fake ModelDocument for namespaces
def __init__(self):
self.xmlRootElement = lbElement
self.xmlDocument = xmlDocument
docObj = DocObj()
numELRs = 0
numTables = 0
def copyAttrs(fromElt, toElt, attrTags):
for attr in attrTags:
if fromElt.get(attr):
toElt.set(attr, fromElt.get(attr))
def generateTable(newLinkElt, newTblElt, srcTblElt, tblAxisRelSet,
axisMbrRelSet, visited):
if srcTblElt is not None:
for rel in tblAxisRelSet.fromModelObject(srcTblElt):
srcAxisElt = rel.toModelObject
if isinstance(srcAxisElt, ModelEuAxisCoord):
visited.add(srcAxisElt)
newAxisElt = etree.SubElement(
newLinkElt, "{http://xbrl.org/2011/table}ruleAxis")
copyAttrs(srcAxisElt, newAxisElt,
("id", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}label"))
newAxisElt.set("abstract",
"true") # always true on root element
newArcElt = etree.SubElement(
newLinkElt, "{http://xbrl.org/2011/table}axisArc")
copyAttrs(rel, newArcElt,
("id", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}from",
"{http://www.w3.org/1999/xlink}to", "order"))
newArcElt.set("{http://www.w3.org/1999/xlink}arcrole",
XbrlConst.tableBreakdown)
newArcElt.set("axisDisposition", rel.axisDisposition)
generateAxis(newLinkElt, newAxisElt, srcAxisElt,
axisMbrRelSet, visited)
visited.discard(srcAxisElt)
def generateAxis(newLinkElt, newAxisParentElt, srcAxisElt, axisMbrRelSet,
visited):
for rel in axisMbrRelSet.fromModelObject(srcAxisElt):
tgtAxisElt = rel.toModelObject
if isinstance(tgtAxisElt,
ModelEuAxisCoord) and tgtAxisElt not in visited:
visited.add(tgtAxisElt)
newAxisElt = etree.SubElement(
newLinkElt, "{http://xbrl.org/2011/table}ruleAxis")
copyAttrs(
tgtAxisElt, newAxisElt,
("id", "abstract", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}label"))
if tgtAxisElt.primaryItemQname:
newRuleElt = etree.SubElement(
newAxisElt, "{http://xbrl.org/2008/formula}concept")
newQnameElt = etree.SubElement(
newRuleElt, "{http://xbrl.org/2008/formula}qname")
newQnameElt.text = XmlUtil.addQnameValue(
docObj, tgtAxisElt.primaryItemQname)
for dimQname, memQname in tgtAxisElt.explicitDims:
newRuleElt = etree.SubElement(
newAxisElt,
"{http://xbrl.org/2008/formula}explicitDimension")
newRuleElt.set("dimension",
XmlUtil.addQnameValue(docObj, dimQname))
newMbrElt = etree.SubElement(
newRuleElt, "{http://xbrl.org/2008/formula}member")
newQnameElt = etree.SubElement(
newMbrElt, "{http://xbrl.org/2008/formula}qname")
newQnameElt.text = XmlUtil.addQnameValue(docObj, memQname)
newArcElt = etree.SubElement(
newLinkElt, "{http://xbrl.org/2011/table}axisArc")
copyAttrs(rel, newArcElt,
("id", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}from",
"{http://www.w3.org/1999/xlink}to", "order"))
newArcElt.set("{http://www.w3.org/1999/xlink}arcrole",
XbrlConst.tableAxisSubtree)
generateAxis(newLinkElt, newAxisElt, tgtAxisElt, axisMbrRelSet,
visited)
visited.discard(tgtAxisElt)
# sort URIs
linkroleUris = sorted(
[linkroleUri for linkroleUri in tblAxisRelSet.linkRoleUris])
firstNewLinkElt = None
roleRefUris = set()
for linkroleUri in linkroleUris:
numELRs += 1
newLinkElt = etree.SubElement(lbElement,
"{http://xbrl.org/2008/generic}link")
newLinkElt.set("{http://www.w3.org/1999/xlink}type", "extended")
newLinkElt.set("{http://www.w3.org/1999/xlink}role", linkroleUri)
if firstNewLinkElt is None: firstNewLinkElt = newLinkElt
# To do: add roleRef if needed
tblAxisRelSet = dts.relationshipSet(XbrlConst.euTableAxis, linkroleUri)
axisMbrRelSet = dts.relationshipSet(XbrlConst.euAxisMember,
linkroleUri)
for srcTblElt in tblAxisRelSet.rootConcepts:
if srcTblElt.tag == "{http://www.eurofiling.info/2010/rendering}table":
numTables += 1
newTblElt = etree.SubElement(
newLinkElt, "{http://xbrl.org/2011/table}table")
newTblElt.set("aspectModel", "dimensional")
copyAttrs(srcTblElt, newTblElt,
("id", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}label"))
generateTable(newLinkElt, newTblElt, srcTblElt, tblAxisRelSet,
axisMbrRelSet, set())
if linkroleUri not in roleRefUris:
srcRoleRefElt = XmlUtil.descendant(srcTblElt.getroottree(),
XbrlConst.link,
"roleRef", "roleURI",
linkroleUri)
if srcRoleRefElt is not None:
roleRefUris.add(linkroleUri)
newRoleRefElt = etree.Element(
"{http://www.xbrl.org/2003/linkbase}roleRef")
copyAttrs(
srcRoleRefElt, newRoleRefElt,
("roleURI", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}href"))
firstNewLinkElt.addprevious(newRoleRefElt)
fh = open(updatedTableLinkbaseFile, "w", encoding="utf-8")
XmlUtil.writexml(fh, xmlDocument, encoding="utf-8")
fh.close()
dts.info(
"info:updateTableLinkbase",
_("Updated Table Linkbase of %(entryFile)s has %(numberOfLinkroles)s linkroles, %(numberOfTables)s tables in file %(tableLBOutputFile)s."
),
modelObject=dts,
entryFile=dts.uri,
numberOfLinkroles=numELRs,
numberOfTables=numTables,
tableLBOutputFile=updatedTableLinkbaseFile)
def sphinxTestcaseVariationReadMeFirstUris(modelTestcaseVariation):
xbrlElement = XmlUtil.descendant(modelTestcaseVariation, 'http://www.corefiling.com/sphinx-conformance-harness/2.0', "xbrl")
if xbrlElement is not None:
modelTestcaseVariation._readMeFirstUris.append(xbrlElement.elementText)
return True # found it
return False # not a sphinx test case variation
def fileNumber(self):
return XmlUtil.text(XmlUtil.descendant(self, edgr, "fileNumber"))
def versioningReportUri(self):
return XmlUtil.text(XmlUtil.descendant(self, None, "versioningReport"))
def sphinxTestcaseVariationExpectedSeverity(modelTestcaseVariation):
issueElement = XmlUtil.descendant(modelTestcaseVariation, 'http://www.corefiling.com/sphinx-conformance-harness/2.0', "issue")
if issueElement is not None:
return issueElement.get("severity")
return None # no issue or not a sphinx test case variation
def resultIsInfoset(self):
if self.modelDocument.outpath:
result = XmlUtil.descendant(self, None, "result")
if result is not None:
return XmlUtil.child(result, None, "file") is not None or XmlUtil.text(result).endswith(".xml")
return False
def checkFilingDTS(val, modelDocument, isEFM, isGFM, visited):
global targetNamespaceDatePattern, efmFilenamePattern, htmlFileNamePattern, roleTypePattern, arcroleTypePattern, \
arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet, \
namespacesConflictPattern
if targetNamespaceDatePattern is None:
targetNamespaceDatePattern = re.compile(
r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|"
r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|")
efmFilenamePattern = re.compile(
r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml|\.htm)$")
htmlFileNamePattern = re.compile(
r"^[a-zA-Z0-9][._a-zA-Z0-9-]*(\.htm)$")
roleTypePattern = re.compile(r"^.*/role/[^/\s]+$")
arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$")
arcroleDefinitionPattern = re.compile(
r"^.*[^\\s]+.*$") # at least one non-whitespace character
namePattern = re.compile(
"[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]"
) # u20ac=Euro, u00a3=pound sterling
linkroleDefinitionBalanceIncomeSheet = re.compile(
r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)",
re.IGNORECASE)
namespacesConflictPattern = re.compile(
r"http://(xbrl\.us|fasb\.org|xbrl\.sec\.gov)/(dei|us-types|us-roles|rr)/([0-9]{4}-[0-9]{2}-[0-9]{2})$"
)
nonDomainItemNameProblemPattern = re.compile(
r"({0})|(FirstQuarter|SecondQuarter|ThirdQuarter|FourthQuarter|[1-4]Qtr|Qtr[1-4]|ytd|YTD|HalfYear)(?:$|[A-Z\W])"
.format(re.sub(r"\W", "", (val.entityRegistrantName or "").title())))
visited.append(modelDocument)
for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items(
):
#6.07.01 no includes
if modelDocumentReference.referenceType == "include":
val.modelXbrl.error(
("EFM.6.07.01", "GFM.1.03.01"),
_("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"
),
modelObject=modelDocumentReference.referringModelObject,
schema=os.path.basename(modelDocument.uri),
include=os.path.basename(referencedDocument.uri))
if referencedDocument not in visited and referencedDocument.inDTS: # ignore EdgarRenderer added non-DTS documents
checkFilingDTS(val, referencedDocument, isEFM, isGFM, visited)
if val.disclosureSystem.standardTaxonomiesDict is None:
pass
if isEFM:
if modelDocument.uri in val.disclosureSystem.standardTaxonomiesDict:
if modelDocument.targetNamespace:
# check for duplicates of us-types, dei, and rr taxonomies
match = namespacesConflictPattern.match(
modelDocument.targetNamespace)
if match is not None:
val.standardNamespaceConflicts[match.group(2)].add(
modelDocument)
else:
if len(modelDocument.basename) > 32:
val.modelXbrl.error(
"EFM.5.01.01.tooManyCharacters",
_("Document file name %(filename)s must not exceed 32 characters."
),
modelObject=modelDocument,
filename=modelDocument.basename)
if modelDocument.type == ModelDocument.Type.INLINEXBRL:
if not htmlFileNamePattern.match(modelDocument.basename):
val.modelXbrl.error(
"EFM.5.01.01",
_("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with .htm."
),
modelObject=modelDocument,
filename=modelDocument.basename)
elif not efmFilenamePattern.match(modelDocument.basename):
val.modelXbrl.error(
"EFM.5.01.01",
_("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with .xsd or .xml."
),
modelObject=modelDocument,
filename=modelDocument.basename)
if (modelDocument.type == ModelDocument.Type.SCHEMA
and modelDocument.targetNamespace
not in val.disclosureSystem.baseTaxonomyNamespaces
and modelDocument.uri.startswith(val.modelXbrl.uriDir)):
val.hasExtensionSchema = True
# check schema contents types
# 6.7.3 check namespace for standard authority
targetNamespaceAuthority = UrlUtil.authority(
modelDocument.targetNamespace)
if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities:
val.modelXbrl.error(
("EFM.6.07.03", "GFM.1.03.03"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s is a disallowed authority"
),
modelObject=modelDocument,
schema=os.path.basename(modelDocument.uri),
targetNamespace=modelDocument.targetNamespace,
targetNamespaceAuthority=UrlUtil.authority(
modelDocument.targetNamespace, includeScheme=False))
# 6.7.4 check namespace format
if modelDocument.targetNamespace is None or not modelDocument.targetNamespace.startswith(
"http://"):
match = None
else:
targetNamespaceDate = modelDocument.targetNamespace[
len(targetNamespaceAuthority):]
match = targetNamespaceDatePattern.match(targetNamespaceDate)
if match is not None:
try:
if match.lastindex == 3:
date = datetime.date(int(match.group(1)),
int(match.group(2)),
int(match.group(3)))
elif match.lastindex == 6:
date = datetime.date(int(match.group(4)),
int(match.group(5)),
int(match.group(6)))
else:
match = None
except ValueError:
match = None
if match is None:
val.modelXbrl.error(
("EFM.6.07.04", "GFM.1.03.04"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s must have format http://{authority}/{versionDate}"
),
modelObject=modelDocument,
schema=os.path.basename(modelDocument.uri),
targetNamespace=modelDocument.targetNamespace)
elif val.fileNameDate and date > val.fileNameDate:
val.modelXbrl.info(
("EFM.6.07.06", "GFM.1.03.06"),
_("Warning: Taxonomy schema %(schema)s namespace %(targetNamespace)s has date later than document name date %(docNameDate)s"
),
modelObject=modelDocument,
schema=os.path.basename(modelDocument.uri),
targetNamespace=modelDocument.targetNamespace,
docNameDate=val.fileNameDate)
if modelDocument.targetNamespace is not None:
# 6.7.5 check prefix for _
authority = UrlUtil.authority(modelDocument.targetNamespace)
if not re.match(r"(http://|https://|ftp://|urn:)\w+", authority):
val.modelXbrl.error(
("EFM.6.07.05", "GFM.1.03.05"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s must be a valid URL with a valid authority for the namespace."
),
modelObject=modelDocument,
schema=os.path.basename(modelDocument.uri),
targetNamespace=modelDocument.targetNamespace)
prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement,
modelDocument.targetNamespace)
if not prefix:
val.modelXbrl.error(
("EFM.6.07.07", "GFM.1.03.07"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s missing prefix for the namespace."
),
modelObject=modelDocument,
schema=os.path.basename(modelDocument.uri),
targetNamespace=modelDocument.targetNamespace)
elif "_" in prefix:
val.modelXbrl.error(
("EFM.6.07.07", "GFM.1.03.07"),
_("Taxonomy schema %(schema)s namespace %(targetNamespace)s prefix %(prefix)s must not have an '_'"
),
modelObject=modelDocument,
schema=os.path.basename(modelDocument.uri),
targetNamespace=modelDocument.targetNamespace,
prefix=prefix)
for modelConcept in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.w3.org/2001/XMLSchema}element"):
if isinstance(modelConcept, ModelConcept):
# 6.7.16 name not duplicated in standard taxonomies
name = modelConcept.get("name")
if name is None:
name = ""
if modelConcept.get("ref") is not None:
continue # don't validate ref's here
for c in val.modelXbrl.nameConcepts.get(name, []):
if c.modelDocument != modelDocument:
if not c.modelDocument.uri.startswith(
val.modelXbrl.uriDir):
val.modelXbrl.error(
("EFM.6.07.16", "GFM.1.03.18"),
_("Concept %(concept)s is also defined in standard taxonomy schema schema %(standardSchema)s"
),
modelObject=(modelConcept, c),
concept=modelConcept.qname,
standardSchema=os.path.basename(
c.modelDocument.uri),
standardConcept=c.qname)
# 6.7.17 id properly formed
_id = modelConcept.id
requiredId = (prefix
if prefix is not None else "") + "_" + name
if _id != requiredId:
val.modelXbrl.error(
("EFM.6.07.17", "GFM.1.03.19"),
_("Concept %(concept)s id %(id)s should be %(requiredId)s"
),
modelObject=modelConcept,
concept=modelConcept.qname,
id=_id,
requiredId=requiredId)
# 6.7.18 nillable is true
nillable = modelConcept.get("nillable")
if nillable != "true" and modelConcept.isItem:
val.modelXbrl.error(
("EFM.6.07.18", "GFM.1.03.20"),
_("Taxonomy schema %(schema)s element %(concept)s nillable %(nillable)s should be 'true'"
),
modelObject=modelConcept,
schema=os.path.basename(modelDocument.uri),
concept=name,
nillable=nillable)
# 6.7.19 not tuple
if modelConcept.isTuple:
val.modelXbrl.error(
("EFM.6.07.19", "GFM.1.03.21"),
_("Concept %(concept)s is a tuple"),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.7.20 no typed domain ref
if modelConcept.isTypedDimension:
val.modelXbrl.error(
("EFM.6.07.20", "GFM.1.03.22"),
_("Concept %(concept)s has typedDomainRef %(typedDomainRef)s"
),
modelObject=modelConcept,
concept=modelConcept.qname,
typedDomainRef=modelConcept.typedDomainElement.
qname if modelConcept.typedDomainElement
is not None else modelConcept.typedDomainRef)
# 6.7.21 abstract must be duration
isDuration = modelConcept.periodType == "duration"
if modelConcept.isAbstract and not isDuration:
val.modelXbrl.error(
("EFM.6.07.21", "GFM.1.03.23"),
_("Taxonomy schema %(schema)s element %(concept)s is abstract but period type is not duration"
),
modelObject=modelConcept,
schema=os.path.basename(modelDocument.uri),
concept=modelConcept.qname)
# 6.7.22 abstract must be stringItemType
''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08
if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType:
val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"),
_("Concept %(concept)s is abstract but type is not xbrli:stringItemType"),
modelObject=modelConcept, concept=modelConcept.qname)
'''
substitutionGroupQname = modelConcept.substitutionGroupQname
# 6.7.23 Axis must be subs group dimension
if name.endswith("Axis") ^ (
substitutionGroupQname
== XbrlConst.qnXbrldtDimensionItem):
val.modelXbrl.error(
("EFM.6.07.23", "GFM.1.03.25"),
_("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.7.24 Table must be subs group hypercube
if name.endswith("Table") ^ (
substitutionGroupQname
== XbrlConst.qnXbrldtHypercubeItem):
val.modelXbrl.error(
("EFM.6.07.24", "GFM.1.03.26"),
_("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"
),
modelObject=modelConcept,
schema=os.path.basename(modelDocument.uri),
concept=modelConcept.qname)
# 6.7.25 if neither hypercube or dimension, substitution group must be item
if substitutionGroupQname not in (
None, XbrlConst.qnXbrldtDimensionItem,
XbrlConst.qnXbrldtHypercubeItem,
XbrlConst.qnXbrliItem):
val.modelXbrl.error(
("EFM.6.07.25", "GFM.1.03.27"),
_("Concept %(concept)s has disallowed substitution group %(substitutionGroup)s"
),
modelObject=modelConcept,
concept=modelConcept.qname,
substitutionGroup=modelConcept.
substitutionGroupQname)
# 6.7.26 Table must be subs group hypercube
if name.endswith(
"LineItems") and modelConcept.abstract != "true":
val.modelXbrl.error(
("EFM.6.07.26", "GFM.1.03.28"),
_("Concept %(concept)s is a LineItems but not abstract"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.7.27 type domainMember must end with Domain or Member
conceptType = modelConcept.type
isDomainItemType = conceptType is not None and conceptType.isDomainItemType
endsWithDomainOrMember = name.endswith(
"Domain") or name.endswith("Member")
if isDomainItemType != endsWithDomainOrMember:
val.modelXbrl.error(
("EFM.6.07.27", "GFM.1.03.29"),
_("Concept %(concept)s must end with Domain or Member for type of domainItemType"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.7.28 domainItemType must be duration
if isDomainItemType and not isDuration:
val.modelXbrl.error(
("EFM.6.07.28", "GFM.1.03.30"),
_("Concept %(concept)s is a domainItemType and must be periodType duration"
),
modelObject=modelConcept,
concept=modelConcept.qname)
#6.7.31 (version 27) fractions
if modelConcept.isFraction:
val.modelXbrl.error(
"EFM.6.07.31",
_("Concept %(concept)s is a fraction"),
modelObject=modelConcept,
concept=modelConcept.qname)
#6.7.32 (version 27) instant non numeric
if modelConcept.isItem and (not modelConcept.isNumeric
and not isDuration
and not modelConcept.isAbstract
and not isDomainItemType):
val.modelXbrl.error(
"EFM.6.07.32",
_("Taxonomy schema %(schema)s element %(concept)s is non-numeric but period type is not duration"
),
modelObject=modelConcept,
schema=os.path.basename(modelDocument.uri),
concept=modelConcept.qname)
# 6.8.5 semantic check, check LC3 name
if name:
if not name[0].isupper():
val.modelXbrl.log(
"ERROR-SEMANTIC", ("EFM.6.08.05.firstLetter",
"GFM.2.03.05.firstLetter"),
_("Concept %(concept)s name must start with a capital letter"
),
modelObject=modelConcept,
concept=modelConcept.qname)
if namePattern.search(name):
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.05.disallowedCharacter",
"GFM.2.03.05.disallowedCharacter"),
_("Concept %(concept)s has disallowed name character"
),
modelObject=modelConcept,
concept=modelConcept.qname)
if len(name) > 200:
val.modelXbrl.log(
"ERROR-SEMANTIC",
"EFM.6.08.05.nameLength",
_("Concept %(concept)s name length %(namelength)s exceeds 200 characters"
),
modelObject=modelConcept,
concept=modelConcept.qname,
namelength=len(name))
if isEFM:
label = modelConcept.label(lang="en-US",
fallbackToQname=False)
if label:
# allow Joe's Bar, N.A. to be JoesBarNA -- remove ', allow A. as not article "a"
lc3name = ''.join(
re.sub(r"['.-]", "", (
w[0] or w[2] or w[3] or w[4])).title()
for w in re.findall(
r"((\w+')+\w+)|(A[.-])|([.-]A(?=\W|$))|(\w+)",
label
) # EFM implies this should allow - and . re.findall(r"[\w\-\.]+", label)
if w[4].lower() not in ("the", "a", "an"))
if not (name == lc3name or
(name and lc3name and lc3name[0].isdigit()
and name[1:] == lc3name and
(name[0].isalpha() or name[0] == '_'))):
val.modelXbrl.log(
"WARNING-SEMANTIC",
"EFM.6.08.05.LC3",
_("Concept %(concept)s should match expected LC3 composition %(lc3name)s"
),
modelObject=modelConcept,
concept=modelConcept.qname,
lc3name=lc3name)
if conceptType is not None:
# 6.8.6 semantic check
if not isDomainItemType and conceptType.qname != XbrlConst.qnXbrliDurationItemType:
nameProblems = nonDomainItemNameProblemPattern.findall(
name)
if any(
any(t) for t in nameProblems
): # list of tuples with possibly nonempty strings
val.modelXbrl.log(
"WARNING-SEMANTIC",
("EFM.6.08.06", "GFM.2.03.06"),
_("Concept %(concept)s should not contain company or period information, found: %(matches)s"
),
modelObject=modelConcept,
concept=modelConcept.qname,
matches=", ".join(''.join(t)
for t in nameProblems))
if conceptType.qname == XbrlConst.qnXbrliMonetaryItemType:
if not modelConcept.balance:
# 6.8.11 may not appear on a income or balance statement
if any(
linkroleDefinitionBalanceIncomeSheet.
match(roleType.definition)
for rel in val.modelXbrl.
relationshipSet(XbrlConst.parentChild).
toModelObject(modelConcept)
for roleType in val.modelXbrl.
roleTypes.get(rel.linkrole, ())):
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.11", "GFM.2.03.11"),
_("Concept %(concept)s must have a balance because it appears in a statement of income or balance sheet"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.11.5 semantic check, must have a documentation label
stdLabel = modelConcept.label(
lang="en-US", fallbackToQname=False)
defLabel = modelConcept.label(
preferredLabel=XbrlConst.
documentationLabel,
lang="en-US",
fallbackToQname=False)
if not defLabel or ( # want different words than std label
stdLabel
and re.findall(r"\w+", stdLabel)
== re.findall(r"\w+", defLabel)):
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.11.05", "GFM.2.04.04"),
_("Concept %(concept)s is monetary without a balance and must have a documentation label that disambiguates its sign"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.8.16 semantic check
if conceptType.qname == XbrlConst.qnXbrliDateItemType and modelConcept.periodType != "duration":
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.16", "GFM.2.03.16"),
_("Concept %(concept)s of type xbrli:dateItemType must have periodType duration"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.8.17 semantic check
if conceptType.qname == XbrlConst.qnXbrliStringItemType and modelConcept.periodType != "duration":
val.modelXbrl.log(
"ERROR-SEMANTIC",
("EFM.6.08.17", "GFM.2.03.17"),
_("Concept %(concept)s of type xbrli:stringItemType must have periodType duration"
),
modelObject=modelConcept,
concept=modelConcept.qname)
# 6.7.8 check for embedded linkbase
for e in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
if isinstance(e, ModelObject):
val.modelXbrl.error(
("EFM.6.07.08", "GFM.1.03.08"),
_("Taxonomy schema %(schema)s contains an embedded linkbase"
),
modelObject=e,
schema=modelDocument.basename)
break
requiredUsedOns = {
XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink,
XbrlConst.qnLinkDefinitionLink
}
standardUsedOns = {
XbrlConst.qnLinkLabel,
XbrlConst.qnLinkReference,
XbrlConst.qnLinkDefinitionArc,
XbrlConst.qnLinkCalculationArc,
XbrlConst.qnLinkPresentationArc,
XbrlConst.qnLinkLabelArc,
XbrlConst.qnLinkReferenceArc,
# per WH, private footnote arc and footnore resource roles are not allowed
XbrlConst.qnLinkFootnoteArc,
XbrlConst.qnLinkFootnote,
}
# 6.7.9 role types authority
for e in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.xbrl.org/2003/linkbase}roleType"):
if isinstance(e, ModelObject):
roleURI = e.get("roleURI")
if targetNamespaceAuthority != UrlUtil.authority(roleURI):
val.modelXbrl.error(
("EFM.6.07.09", "GFM.1.03.09"),
_("RoleType %(roleType)s does not match authority %(targetNamespaceAuthority)s"
),
modelObject=e,
roleType=roleURI,
targetNamespaceAuthority=targetNamespaceAuthority,
targetNamespace=modelDocument.targetNamespace)
# 6.7.9 end with .../role/lc3 name
if not roleTypePattern.match(roleURI):
val.modelXbrl.warning(
("EFM.6.07.09.roleEnding", "GFM.1.03.09"),
"RoleType %(roleType)s should end with /role/{LC3name}",
modelObject=e,
roleType=roleURI)
# 6.7.10 only one role type declaration in DTS
modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI)
if modelRoleTypes is not None:
modelRoleType = modelRoleTypes[0]
definition = modelRoleType.definitionNotStripped
usedOns = modelRoleType.usedOns
if len(modelRoleTypes) == 1:
# 6.7.11 used on's for pre, cal, def if any has a used on
if not usedOns.isdisjoint(requiredUsedOns) and len(
requiredUsedOns - usedOns) > 0:
val.modelXbrl.error(
("EFM.6.07.11", "GFM.1.03.11"),
_("RoleType %(roleType)s missing used on %(usedOn)s"
),
modelObject=e,
roleType=roleURI,
usedOn=requiredUsedOns - usedOns)
# 6.7.12 definition match pattern
if (val.disclosureSystem.roleDefinitionPattern
is not None and
(definition is None or not val.disclosureSystem.
roleDefinitionPattern.match(definition))):
val.modelXbrl.error(
("EFM.6.07.12", "GFM.1.03.12-14"),
_("RoleType %(roleType)s definition \"%(definition)s\" must match {Sortcode} - {Type} - {Title}"
),
modelObject=e,
roleType=roleURI,
definition=(definition or ""))
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log(
"ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("RoleType %(roleuri)s is defined using role types already defined by standard roles for: %(qnames)s"
),
modelObject=e,
roleuri=roleURI,
qnames=', '.join(
str(qn) for qn in usedOns & standardUsedOns))
# 6.7.13 arcrole types authority
for e in modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.xbrl.org/2003/linkbase}arcroleType"):
if isinstance(e, ModelObject):
arcroleURI = e.get("arcroleURI")
if targetNamespaceAuthority != UrlUtil.authority(arcroleURI):
val.modelXbrl.error(
("EFM.6.07.13", "GFM.1.03.15"),
_("ArcroleType %(arcroleType)s does not match authority %(targetNamespaceAuthority)s"
),
modelObject=e,
arcroleType=arcroleURI,
targetNamespaceAuthority=targetNamespaceAuthority,
targetNamespace=modelDocument.targetNamespace)
# 6.7.13 end with .../arcrole/lc3 name
if not arcroleTypePattern.match(arcroleURI):
val.modelXbrl.warning(
("EFM.6.07.13.arcroleEnding", "GFM.1.03.15"),
_("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"
),
modelObject=e,
arcroleType=arcroleURI)
# 6.7.15 definition match pattern
modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI]
definition = modelRoleTypes[0].definition
if definition is None or not arcroleDefinitionPattern.match(
definition):
val.modelXbrl.error(
("EFM.6.07.15", "GFM.1.03.17"),
_("ArcroleType %(arcroleType)s definition must be non-empty"
),
modelObject=e,
arcroleType=arcroleURI)
# semantic checks
usedOns = modelRoleTypes[0].usedOns
if usedOns & standardUsedOns: # semantics check
val.modelXbrl.log(
"ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
_("ArcroleType %(arcroleuri)s is defined using role types already defined by standard arcroles for: %(qnames)s"
),
modelObject=e,
arcroleuri=arcroleURI,
qnames=', '.join(
str(qn) for qn in usedOns & standardUsedOns))
#6.3.3 filename check
m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9]).xsd$",
modelDocument.basename)
if m:
try: # check date value
datetime.datetime.strptime(m.group(1), "%Y%m%d").date()
# date and format are ok, check "should" part of 6.3.3
if val.fileNameBasePart:
expectedFilename = "{0}-{1}.xsd".format(
val.fileNameBasePart, val.fileNameDatePart)
if modelDocument.basename != expectedFilename:
val.modelXbrl.log(
"WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance",
"GFM.1.01.01.matchInstance"),
_('Schema file name warning: %(filename)s, should match %(expectedFilename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
expectedFilename=expectedFilename)
except ValueError:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid schema file base name part (date) in "{base}-{yyyymmdd}.xsd": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
else:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid schema file name, must match "{base}-{yyyymmdd}.xsd": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01"))
elif modelDocument.type == ModelDocument.Type.LINKBASE:
# if it is part of the submission (in same directory) check name
labelRels = None
if modelDocument.filepath.startswith(
val.modelXbrl.modelDocument.filepathdir):
#6.3.3 filename check
extLinkElt = XmlUtil.descendant(
modelDocument.xmlRootElement, XbrlConst.link, "*",
"{http://www.w3.org/1999/xlink}type", "extended")
if extLinkElt is None: # no ext link element
val.modelXbrl.error(
(val.EFM60303 + ".noLinkElement",
"GFM.1.01.01.noLinkElement"),
_('Invalid linkbase file name: %(filename)s, has no extended link element, cannot determine link type.'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03.noLinkElement",
"EFM.6.23.01.noLinkElement",
"GFM.1.01.01.noLinkElement"))
elif extLinkElt.localName not in extLinkEltFileNameEnding:
val.modelXbrl.error(
"EFM.6.03.02",
_('Invalid linkbase link element %(linkElement)s in %(filename)s'
),
modelObject=modelDocument,
linkElement=extLinkElt.localName,
filename=modelDocument.basename)
else:
m = re.match(
r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9])(_[a-z]{3}).xml$",
modelDocument.basename)
expectedSuffix = extLinkEltFileNameEnding[extLinkElt.localName]
if m and m.group(2) == expectedSuffix:
try: # check date value
datetime.datetime.strptime(m.group(1), "%Y%m%d").date()
# date and format are ok, check "should" part of 6.3.3
if val.fileNameBasePart:
expectedFilename = "{0}-{1}{2}.xml".format(
val.fileNameBasePart, val.fileNameDatePart,
expectedSuffix)
if modelDocument.basename != expectedFilename:
val.modelXbrl.log(
"WARNING-SEMANTIC",
("EFM.6.03.03.matchInstance",
"GFM.1.01.01.matchInstance"),
_('Linkbase name warning: %(filename)s should match %(expectedFilename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
expectedFilename=expectedFilename)
except ValueError:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid linkbase base file name part (date) in "{base}-{yyyymmdd}_{suffix}.xml": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
messageCodes=("EFM.6.03.03", "EFM.6.23.01",
"GFM.1.01.01"))
else:
val.modelXbrl.error(
(val.EFM60303, "GFM.1.01.01"),
_('Invalid linkbase name, must match "{base}-{yyyymmdd}%(expectedSuffix)s.xml": %(filename)s'
),
modelObject=modelDocument,
filename=modelDocument.basename,
expectedSuffix=expectedSuffix,
messageCodes=("EFM.6.03.03", "EFM.6.23.01",
"GFM.1.01.01"))
if extLinkElt.localName == "labelLink":
if labelRels is None:
labelRels = val.modelXbrl.relationshipSet(
XbrlConst.conceptLabel)
for labelElt in XmlUtil.children(extLinkElt,
XbrlConst.link, "label"):
# 6.10.9
if XbrlConst.isNumericRole(labelElt.role):
for rel in labelRels.toModelObject(labelElt):
if rel.fromModelObject is not None and not rel.fromModelObject.isNumeric:
val.modelXbrl.error(
"EFM.6.10.09",
_("Label of non-numeric concept %(concept)s has a numeric role: %(role)s"
),
modelObject=(labelElt,
rel.fromModelObject),
concept=rel.fromModelObject.qname,
role=labelElt.role)
def expectedVersioningReport(self):
XmlUtil.text(XmlUtil.text(XmlUtil.descendant(XmlUtil.descendant(self, None, "result"), None, "versioningReport")))
def sphinxTestcaseVariationReadMeFirstUris(modelTestcaseVariation, *args, **kwargs):
xbrlElement = XmlUtil.descendant(modelTestcaseVariation, 'http://www.corefiling.com/sphinx-conformance-harness/2.0', "xbrl")
if xbrlElement is not None:
modelTestcaseVariation._readMeFirstUris.append(xbrlElement.textValue)
return True # found it
return False # not a sphinx test case variation
def checkDTSdocument(val, modelDocument, *args, **kwargs):
modelXbrl = val.modelXbrl
if modelDocument.type in (ModelDocument.Type.SCHEMA, ModelDocument.Type.LINKBASE):
isSchema = modelDocument.type == ModelDocument.Type.SCHEMA
docinfo = modelDocument.xmlDocument.docinfo
if docinfo and docinfo.xml_version != "1.0":
modelXbrl.error("SBR.NL.2.2.0.02" if isSchema else "SBR.NL.2.3.0.02",
_('%(docType)s xml version must be "1.0" but is "%(xmlVersion)s"'),
modelObject=modelDocument, docType=modelDocument.gettype().title(),
xmlVersion=docinfo.xml_version)
if modelDocument.documentEncoding.lower() != "utf-8":
modelXbrl.error("SBR.NL.2.2.0.03" if isSchema else "SBR.NL.2.3.0.03",
_('%(docType)s encoding must be "utf-8" but is "%(xmlEncoding)s"'),
modelObject=modelDocument, docType=modelDocument.gettype().title(),
xmlEncoding=modelDocument.documentEncoding)
lookingForPrecedingComment = True
for commentNode in modelDocument.xmlRootElement.itersiblings(preceding=True):
if isinstance(commentNode, etree._Comment):
if lookingForPrecedingComment:
lookingForPrecedingComment = False
else:
modelXbrl.error("SBR.NL.2.2.0.05" if isSchema else "SBR.NL.2.3.0.05",
_('%(docType)s must have only one comment node before schema element'),
modelObject=modelDocument, docType=modelDocument.gettype().title())
if lookingForPrecedingComment:
modelXbrl.error("SBR.NL.2.2.0.04" if isSchema else "SBR.NL.2.3.0.04",
_('%(docType)s must have comment node only on line 2'),
modelObject=modelDocument, docType=modelDocument.gettype().title())
# check namespaces are used
for prefix, ns in modelDocument.xmlRootElement.nsmap.items():
if ((prefix not in val.valUsedPrefixes) and
(modelDocument.type != ModelDocument.Type.SCHEMA or ns != modelDocument.targetNamespace)):
modelXbrl.error("SBR.NL.2.2.0.11" if modelDocument.type == ModelDocument.Type.SCHEMA else "SBR.NL.2.3.0.08",
_('%(docType)s namespace declaration "%(declaration)s" is not used'),
modelObject=modelDocument, docType=modelDocument.gettype().title(),
declaration=("xmlns" + (":" + prefix if prefix else "") + "=" + ns))
if isSchema and val.annotationsCount > 1:
modelXbrl.error("SBR.NL.2.2.0.22",
_('Schema has %(annotationsCount)s xs:annotation elements, only 1 allowed'),
modelObject=modelDocument, annotationsCount=val.annotationsCount)
if modelDocument.type == ModelDocument.Type.LINKBASE:
if not val.containsRelationship:
modelXbrl.error("SBR.NL.2.3.0.12",
"Linkbase has no relationships",
modelObject=modelDocument)
# check file name suffixes
extLinkElt = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended")
if extLinkElt is not None:
expectedSuffix = None
if extLinkElt.localName == "labelLink":
anyLabel = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/XML/1998/namespace}lang", "*")
if anyLabel is not None:
xmlLang = anyLabel.get("{http://www.w3.org/XML/1998/namespace}lang")
expectedSuffix = "-lab-{0}.xml".format(xmlLang)
else:
expectedSuffix = {"referenceLink": "-ref.xml",
"presentationLink": "-pre.xml",
"definitionLink": "-def.xml"}.get(extLinkElt.localName, None)
if expectedSuffix:
if not modelDocument.uri.endswith(expectedSuffix):
modelXbrl.error("SBR.NL.3.2.1.09",
"Linkbase filename expected to end with %(expectedSuffix)s: %(filename)s",
modelObject=modelDocument, expectedSuffix=expectedSuffix, filename=modelDocument.uri)
elif extLinkElt.qname == XbrlConst.qnGenLink:
anyLabel = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/XML/1998/namespace}lang", "*")
if anyLabel is not None:
xmlLang = anyLabel.get("{http://www.w3.org/XML/1998/namespace}lang")
expectedSuffix = "-generic-lab-{0}.xml".format(xmlLang)
elif XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "reference") is not None:
expectedSuffix = "-generic-ref.xml"
if expectedSuffix and not modelDocument.uri.endswith(expectedSuffix):
modelXbrl.error("SBR.NL.3.2.1.10",
"Generic linkbase filename expected to end with %(expectedSuffix)s: %(filename)s",
modelObject=modelDocument, expectedSuffix=expectedSuffix, filename=modelDocument.uri)
# label checks
for qnLabel in (XbrlConst.qnLinkLabel, XbrlConst.qnGenLabel):
for modelLabel in modelDocument.xmlRootElement.iterdescendants(tag=qnLabel.clarkNotation):
if isinstance(modelLabel, ModelResource):
if not modelLabel.text or not modelLabel.text[:1].isupper():
modelXbrl.error("SBR.NL.3.2.7.05",
_("Labels MUST have a capital first letter, label %(label)s: %(text)s"),
modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64])
if modelLabel.role in (XbrlConst.standardLabel, XbrlConst.genStandardLabel):
if len(modelLabel.text) > 255:
modelXbrl.error("SBR.NL.3.2.7.06",
_("Labels with the 'standard' role MUST NOT exceed 255 characters, label %(label)s: %(text)s"),
modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64])
if modelLabel.role in (XbrlConst.standardLabel, XbrlConst.genStandardLabel):
if len(modelLabel.text) > 255:
modelXbrl.error("SBR.NL.3.2.7.06",
_("Labels with the 'standard' role MUST NOT exceed 255 characters, label %(label)s: %(text)s"),
modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64])
for modelResource in modelDocument.xmlRootElement.iter():
# locator checks
if isinstance(modelResource, ModelLocator):
hrefModelObject = modelResource.dereference()
if isinstance(hrefModelObject, ModelObject):
expectedLocLabel = hrefModelObject.id + "_loc"
if modelResource.xlinkLabel != expectedLocLabel:
modelXbrl.error("SBR.NL.3.2.11.01",
_("Locator @xlink:label names MUST be concatenated from: @id from the XML node, underscore, 'loc', expected %(expectedLocLabel)s, found %(foundLocLabel)s"),
modelObject=modelResource, expectedLocLabel=expectedLocLabel, foundLocLabel=modelResource.xlinkLabel)
# xlinkLabel checks
if isinstance(modelResource, ModelResource):
if re.match(r"[^a-zA-Z0-9_-]", modelResource.xlinkLabel):
modelXbrl.error("SBR.NL.3.2.11.03",
_("@xlink:label names MUST use a-zA-Z0-9_- characters only: %(xlinkLabel)s"),
modelObject=modelResource, xlinkLabel=modelResource.xlinkLabel)
elif modelDocument.targetNamespace: # SCHEMA with targetNamespace
# check for unused imports
for referencedDocument in modelDocument.referencesDocument.keys():
if (referencedDocument.type == ModelDocument.Type.SCHEMA and
referencedDocument.targetNamespace not in {XbrlConst.xbrli, XbrlConst.link} and
referencedDocument.targetNamespace not in val.referencedNamespaces):
modelXbrl.error("SBR.NL.2.2.0.15",
_("A schema import schemas of which no content is being addressed: %(importedFile)s"),
modelObject=modelDocument, importedFile=referencedDocument.basename)
if modelDocument.targetNamespace != modelDocument.targetNamespace.lower():
modelXbrl.error("SBR.NL.3.2.3.02",
_("Namespace URI's MUST be lower case: %(namespaceURI)s"),
modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace)
if len(modelDocument.targetNamespace) > 255:
modelXbrl.error("SBR.NL.3.2.3.03",
_("Namespace URI's MUST NOT be longer than 255 characters: %(namespaceURI)s"),
modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace)
if re.match(r"[^a-z0-9_/-]", modelDocument.targetNamespace):
modelXbrl.error("SBR.NL.3.2.3.04",
_("Namespace URI's MUST use only signs from a-z0-9_-/: %(namespaceURI)s"),
modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace)
if not modelDocument.targetNamespace.startswith('http://www.nltaxonomie.nl'):
modelXbrl.error("SBR.NL.3.2.3.05",
_("Namespace URI's MUST start with 'http://www.nltaxonomie.nl': %(namespaceURI)s"),
modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace)
namespacePrefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement, modelDocument.targetNamespace)
if not namespacePrefix:
modelXbrl.error("SBR.NL.3.2.4.01",
_("TargetNamespaces MUST have a prefix: %(namespaceURI)s"),
modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace)
elif namespacePrefix in val.prefixNamespace:
modelXbrl.error("SBR.NL.3.2.4.02",
_("Namespace prefix MUST be unique within the NT but prefix '%(prefix)s' is used by both %(namespaceURI)s and %(namespaceURI2)s."),
modelObject=modelDocument, prefix=namespacePrefix,
namespaceURI=modelDocument.targetNamespace, namespaceURI2=val.prefixNamespace[namespacePrefix])
else:
val.prefixNamespace[namespacePrefix] = modelDocument.targetNamespace
val.namespacePrefix[modelDocument.targetNamespace] = namespacePrefix
if namespacePrefix in {"xsi", "xsd", "xs", "xbrli", "link", "xlink", "xbrldt", "xbrldi", "gen", "xl"}:
modelXbrl.error("SBR.NL.3.2.4.03",
_("Namespace prefix '%(prefix)s' reserved by organizations for international specifications is used %(namespaceURI)s."),
modelObject=modelDocument, prefix=namespacePrefix, namespaceURI=modelDocument.targetNamespace)
if len(namespacePrefix) > 20:
modelXbrl.warning("SBR.NL.3.2.4.06",
_("Namespace prefix '%(prefix)s' SHOULD not exceed 20 characters %(namespaceURI)s."),
modelObject=modelDocument, prefix=namespacePrefix, namespaceURI=modelDocument.targetNamespace)
# check every non-targetnamespace prefix against its schema
requiredLinkrole = None # only set for extension taxonomies
# check folder names
if modelDocument.filepathdir.startswith(modelXbrl.uriDir):
partnerPrefix = None
pathDir = modelDocument.filepathdir[len(modelXbrl.uriDir) + 1:].replace("\\", "/")
lastPathSegment = None
for pathSegment in pathDir.split("/"):
if pathSegment.lower() != pathSegment:
modelXbrl.error("SBR.NL.3.2.1.02",
_("Folder names must be in lower case: %(folder)s"),
modelObject=modelDocument, folder=pathSegment)
if len(pathSegment) >= 15 :
modelXbrl.error("SBR.NL.3.2.1.03",
_("Folder names must be less than 15 characters: %(folder)s"),
modelObject=modelDocument, folder=pathSegment)
if pathSegment in ("bd", "kvk", "cbs"):
partnerPrefix = pathSegment + '-'
lastPathSegment = pathSegment
if modelDocument.basename.lower() != modelDocument.basename:
modelXbrl.error("SBR.NL.3.2.1.05",
_("File names must be in lower case: %(file)s"),
modelObject=modelDocument, file=modelDocument.basename)
if partnerPrefix and not modelDocument.basename.startswith(partnerPrefix):
modelXbrl.error("SBR.NL.3.2.1.14",
"NT Partner DTS files MUST start with %(partnerPrefix)s consistently: %(filename)s",
modelObject=modelDocument, partnerPrefix=partnerPrefix, filename=modelDocument.uri)
if modelDocument.type == ModelDocument.Type.SCHEMA:
if modelDocument.targetNamespace:
nsParts = modelDocument.targetNamespace.split("/")
# [0] = https, [1] = // [2] = nl.taxonomie [3] = year or version
nsYrOrVer = nsParts[3]
requiredNamespace = "http://www.nltaxonomie.nl/" + nsYrOrVer + "/" + pathDir + "/" + modelDocument.basename[:-4]
requiredLinkrole = "http://www.nltaxonomie.nl/" + nsYrOrVer + "/" + pathDir + "/"
if modelDocument == modelXbrl.modelDocument: # entry point
nsYr = "{year}"
if '2009' <= nsParts[3] < '2020': # must be a year, use as year
nsYr = nsParts[3]
else: # look for year in parts of basename of required namespace
for nsPart in nsParts:
for baseNamePart in nsPart.split('-'):
if '2009' <= baseNamePart < '2020':
nsYr = baseNamePart
break
if not requiredNamespace.endswith('-' + nsYr):
requiredNamespace += '-' + nsYr
if not modelDocument.targetNamespace.startswith(requiredNamespace):
modelXbrl.error("SBR.NL.3.2.3.06",
_("Namespace URI's MUST be constructed like %(requiredNamespace)s: %(namespaceURI)s"),
modelObject=modelDocument, requiredNamespace=requiredNamespace, namespaceURI=modelDocument.targetNamespace)
else:
requiredLinkrole = ''
# concept checks
for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"):
if isinstance(modelConcept, ModelConcept):
# 6.7.16 name not duplicated in standard taxonomies
name = modelConcept.get("name")
if name:
''' removed per RH 2013-03-25
substititutionGroupQname = modelConcept.substitutionGroupQname
if substititutionGroupQname:
if name.endswith("Member") ^ (substititutionGroupQname.localName == "domainMemberItem" and
substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")):
modelXbrl.error("SBR.NL.3.2.5.11",
_("Concept %(concept)s must end in Member to be in sbr:domainMemberItem substitution group"),
modelObject=modelConcept, concept=modelConcept.qname)
if name.endswith("Domain") ^ (substititutionGroupQname.localName == "domainItem" and
substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")):
modelXbrl.error("SBR.NL.3.2.5.12",
_("Concept %(concept)s must end in Domain to be in sbr:domainItem substitution group"),
modelObject=modelConcept, concept=modelConcept.qname)
if name.endswith("TypedAxis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem and
modelConcept.isTypedDimension):
modelXbrl.error("SBR.NL.3.2.5.14",
_("Concept %(concept)s must end in TypedAxis to be in xbrldt:dimensionItem substitution group if they represent a typed dimension"),
modelObject=modelConcept, concept=modelConcept.qname)
if (name.endswith("Axis") and
not name.endswith("TypedAxis")) ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem and
modelConcept.isExplicitDimension):
modelXbrl.error("SBR.NL.3.2.5.13",
_("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group if they represent an explicit dimension"),
modelObject=modelConcept, concept=modelConcept.qname)
if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem):
modelXbrl.error("SBR.NL.3.2.5.15",
_("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"),
modelObject=modelConcept, concept=modelConcept.qname)
if name.endswith("Title") ^ (substititutionGroupQname.localName == "presentationItem" and
substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")):
modelXbrl.error("SBR.NL.3.2.5.16",
_("Concept %(concept)s must end in Title to be in sbr:presentationItem substitution group"),
modelObject=modelConcept, concept=modelConcept.qname)
'''
if len(name) > 200:
modelXbrl.error("SBR.NL.3.2.12.02" if modelConcept.isLinkPart
else "SBR.NL.3.2.5.21" if (modelConcept.isItem or modelConcept.isTuple)
else "SBR.NL.3.2.14.01",
_("Concept %(concept)s name length %(namelength)s exceeds 200 characters"),
modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name))
# type checks
for typeType in ("simpleType", "complexType"):
for modelType in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}" + typeType):
if isinstance(modelType, ModelType):
name = modelType.get("name")
if name is None:
name = ""
if modelType.get("ref") is not None:
continue # don't validate ref's here
if len(name) > 200:
modelXbrl.error("SBR.NL.3.2.5.21",
_("Type %(type)s name length %(namelength)s exceeds 200 characters"),
modelObject=modelType, type=modelType.qname, namelength=len(name))
if modelType.qnameDerivedFrom and modelType.qnameDerivedFrom.namespaceURI != XbrlConst.xbrli:
modelXbrl.error("SBR.NL.3.2.8.01",
_("Custom datatypes MUST be a restriction from XII defined datatypes: %(type)s"),
modelObject=modelType, type=modelType.qname)
if re.match(r"[^a-zA-Z0-9_-]", name):
modelXbrl.error("SBR.NL.3.2.8.02",
_("Datatype names MUST use characters a-zA-Z0-9_- only: %(type)s"),
modelObject=modelDocument, type=modelType.qname)
if modelType.facets and "enumeration" in modelType.facets:
if not modelType.qnameDerivedFrom == XbrlConst.qnXbrliStringItemType:
modelXbrl.error("SBR.NL.3.2.13.01",
_("Enumerations MUST use a restriction on xbrli:stringItemType: %(type)s"),
modelObject=modelDocument, type=modelType.qname)
if lastPathSegment == "entrypoints":
if not modelDocument.xmlRootElement.id:
modelXbrl.error("SBR.NL.2.2.0.23",
_("xs:schema/@id MUST be present in schema files in the reports/{NT partner}/entrypoints/ folder"),
modelObject=modelDocument)
# check for idObject conflicts
for id, modelObject in modelDocument.idObjects.items():
if id in val.idObjects:
modelXbrl.error("SBR.NL.3.2.6.01",
_("ID %(id)s must be unique in the DTS but is present on two elements."),
modelObject=(modelObject, val.idObjects[id]), id=id)
else:
val.idObjects[id] = modelObject
for roleURI, modelRoleTypes in modelXbrl.roleTypes.items():
if not roleURI.startswith("http://www.xbrl.org"):
usedOns = set.union(*[modelRoleType.usedOns for modelRoleType in modelRoleTypes])
# check roletypes for linkroles (only)
if usedOns & {XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink, XbrlConst.qnLinkDefinitionLink,
XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference, XbrlConst.qnLinkFootnote}:
if len(modelRoleTypes) > 1:
modelXbrl.error("SBR.NL.3.2.9.01",
_("Linkrole URI's MUST be unique in the NT: %(linkrole)s"),
modelObject=modelRoleTypes, linkrole=roleURI)
if roleURI.lower() != roleURI:
modelXbrl.error("SBR.NL.3.2.9.02",
_("Linkrole URI's MUST be in lowercase: %(linkrole)s"),
modelObject=modelRoleTypes, linkrole=roleURI)
if re.match(r"[^a-z0-9_/-]", roleURI):
modelXbrl.error("SBR.NL.3.2.9.03",
_("Linkrole URI's MUST use characters a-z0-9_-/ only: %(linkrole)s"),
modelObject=modelRoleTypes, linkrole=roleURI)
if len(roleURI) > 255:
modelXbrl.error("SBR.NL.3.2.9.04",
_("Linkrole URI's MUST NOT be longer than 255 characters, length is %(len)s: %(linkrole)s"),
modelObject=modelRoleTypes, len=len(roleURI), linkrole=roleURI)
''' removed per RH 2013-03-13 e-mail
if not roleURI.startswith('http://www.nltaxonomie.nl'):
modelXbrl.error("SBR.NL.3.2.9.05",
_("Linkrole URI's MUST start with 'http://www.nltaxonomie.nl': %(linkrole)s"),
modelObject=modelRoleTypes, linkrole=roleURI)
if (requiredLinkrole and
not roleURI.startswith(requiredLinkrole) and
re.match(r".*(domain$|axis$|table$|lineitem$)", roleURI)):
modelXbrl.error("SBR.NL.3.2.9.06",
_("Linkrole URI's MUST have the following construct: http://www.nltaxonomie.nl / {folder path} / {functional name} - {domain or axis or table or lineitem}: %(linkrole)s"),
modelObject=modelRoleTypes, linkrole=roleURI)
'''
for modelRoleType in modelRoleTypes:
if len(modelRoleType.id) > 255:
modelXbrl.error("SBR.NL.3.2.10.02",
_("Linkrole @id MUST NOT exceed 255 characters, length is %(length)s: %(linkroleID)s"),
modelObject=modelRoleType, length=len(modelRoleType.id), linkroleID=modelRoleType.id)
partnerPrefix = modelRoleTypes[0].modelDocument.basename.split('-')
if partnerPrefix: # first element before dash is prefix
urnPartnerLinkroleStart = "urn:{0}:linkrole:".format(partnerPrefix[0])
if not roleURI.startswith(urnPartnerLinkroleStart):
modelXbrl.error("SBR.NL.3.2.9.10",
_("Linkrole MUST start with urn:{NT partner code}:linkrole:, \nexpecting: %(expectedStart)s..., \nfound: %(linkrole)s"),
modelObject=modelRoleType, expectedStart=urnPartnerLinkroleStart, linkrole=roleURI)
def dir(self):
self.open()
if not self.isOpen:
return None
elif self.filesDir is not None:
return self.filesDir
elif self.isZip:
files = []
for zipinfo in self.fs.infolist():
files.append(zipinfo.filename)
self.filesDir = files
elif self.isTarGz:
self.filesDir = self.fs.getnames()
elif self.isEis:
files = []
for docElt in self.eisDocument.iter(
tag="{http://www.sec.gov/edgar/common}document"):
outfn = docElt.findtext(
"{http://www.sec.gov/edgar/common}conformedName")
if outfn:
files.append(outfn)
self.filesDir = files
elif self.isXfd:
files = []
for data in self.xfdDocument.iter(tag="data"):
outfn = data.findtext("filename")
if outfn:
if len(outfn) > 2 and outfn[0].isalpha() and \
outfn[1] == ':' and outfn[2] == '\\':
continue
files.append(outfn)
self.filesDir = files
elif self.isRss:
files = [] # return title, descr, pubdate, linst doc
edgr = "http://www.sec.gov/Archives/edgar"
try:
for dsElt in XmlUtil.descendants(self.rssDocument, None,
"item"):
instDoc = None
for instDocElt in XmlUtil.descendants(
dsElt, edgr, "xbrlFile"):
if instDocElt.get(
"(http://www.sec.gov/Archives/edgar}description"
).endswith("INSTANCE DOCUMENT"):
instDoc = instDocElt.get(
"(http://www.sec.gov/Archives/edgar}url")
break
if not instDoc:
continue
files.append((
XmlUtil.text(XmlUtil.descendant(dsElt, None, "title")),
# tooltip
"{0}\n {1}\n {2}\n {3}\n {4}".format(
XmlUtil.text(
XmlUtil.descendant(dsElt, edgr,
"companyName")),
XmlUtil.text(
XmlUtil.descendant(dsElt, edgr, "formType")),
XmlUtil.text(
XmlUtil.descendant(dsElt, edgr, "filingDate")),
XmlUtil.text(
XmlUtil.descendant(dsElt, edgr, "cikNumber")),
XmlUtil.text(
XmlUtil.descendant(dsElt, edgr, "period"))),
XmlUtil.text(
XmlUtil.descendant(dsElt, None, "description")),
XmlUtil.text(XmlUtil.descendant(
dsElt, None, "pubDate")),
instDoc))
self.filesDir = files
except (EnvironmentError, etree.LxmlError) as err:
pass
elif self.isInstalledTaxonomyPackage:
files = []
baseurlPathLen = len(os.path.dirname(self.baseurl)) + 1
def packageDirsFiles(dir):
for file in os.listdir(dir):
path = dir + "/" + file # must have / and not \\ even on windows
files.append(path[baseurlPathLen:])
if os.path.isdir(path):
packageDirsFiles(path)
packageDirsFiles(self.baseurl[0:baseurlPathLen - 1])
self.filesDir = files
return self.filesDir
def period(self):
per = XmlUtil.text(XmlUtil.descendant(self, edgr, "period"))
if per and len(per) == 8:
return "{0}-{1}-{2}".format(per[0:4],per[4:6],per[6:8])
return None
def accessionNumber(self):
return XmlUtil.text(XmlUtil.descendant(self, edgr, "accessionNumber"))
def resultXbrlInstanceUri(self):
resultInstance = XmlUtil.descendant(XmlUtil.descendant(self, None, "result"), None, "instance")
if resultInstance is not None:
return XmlUtil.text(resultInstance)
return None
def formType(self):
return XmlUtil.text(XmlUtil.descendant(self, edgr, "formType"))
def sphinxTestcaseVariationExpectedSeverity(modelTestcaseVariation, *args, **kwargs):
issueElement = XmlUtil.descendant(modelTestcaseVariation, 'http://www.corefiling.com/sphinx-conformance-harness/2.0', "issue")
if issueElement is not None:
return issueElement.get("severity")
return None # no issue or not a sphinx test case variation
def resultIsTable(self):
result = XmlUtil.descendant(self, None, "result")
if result is not None :
child = XmlUtil.child(result, None, "table")
return child is not None and XmlUtil.text(child).endswith(".xml")
return False
def dir(self):
self.open()
if not self.isOpen:
return None
elif self.filesDir is not None:
return self.filesDir
elif self.isZip:
files = []
for zipinfo in self.fs.infolist():
files.append(zipinfo.filename)
self.filesDir = files
elif self.isTarGz:
self.filesDir = self.fs.getnames()
elif self.isEis:
files = []
for docElt in self.eisDocument.iter(tag="{http://www.sec.gov/edgar/common}document"):
outfn = docElt.findtext("{http://www.sec.gov/edgar/common}conformedName")
if outfn:
files.append(outfn);
self.filesDir = files
elif self.isXfd:
files = []
for data in self.xfdDocument.iter(tag="data"):
outfn = data.findtext("filename")
if outfn:
if len(outfn) > 2 and outfn[0].isalpha() and \
outfn[1] == ':' and outfn[2] == '\\':
continue
files.append(outfn);
self.filesDir = files
elif self.isRss:
files = [] # return title, descr, pubdate, linst doc
edgr = "http://www.sec.gov/Archives/edgar"
try:
for dsElt in XmlUtil.descendants(self.rssDocument, None, "item"):
instDoc = None
for instDocElt in XmlUtil.descendants(dsElt, edgr, "xbrlFile"):
if instDocElt.get("(http://www.sec.gov/Archives/edgar}description").endswith("INSTANCE DOCUMENT"):
instDoc = instDocElt.get("(http://www.sec.gov/Archives/edgar}url")
break
if not instDoc:
continue
files.append((
XmlUtil.text(XmlUtil.descendant(dsElt, None, "title")),
# tooltip
"{0}\n {1}\n {2}\n {3}\n {4}".format(
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "companyName")),
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "formType")),
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "filingDate")),
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "cikNumber")),
XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "period"))),
XmlUtil.text(XmlUtil.descendant(dsElt, None, "description")),
XmlUtil.text(XmlUtil.descendant(dsElt, None, "pubDate")),
instDoc))
self.filesDir = files
except (EnvironmentError,
etree.LxmlError) as err:
pass
elif self.isInstalledTaxonomyPackage:
files = []
baseurlPathLen = len(os.path.dirname(self.baseurl)) + 1
def packageDirsFiles(dir):
for file in os.listdir(dir):
path = dir + "/" + file # must have / and not \\ even on windows
files.append(path[baseurlPathLen:])
if os.path.isdir(path):
packageDirsFiles(path)
packageDirsFiles(self.baseurl[0:baseurlPathLen - 1])
self.filesDir = files
return self.filesDir
def cikNumber(self):
return XmlUtil.text(XmlUtil.descendant(self, edgr, "cikNumber"))
