def create(self,
newDocumentType=None,
url=None,
schemaRefs=None,
createModelDocument=True):
self.modelXbrl = ModelXbrl.create(self, newDocumentType, url,
schemaRefs, createModelDocument)
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def saveTargetDocument(self):
targetUrl = self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
self.targetDocumentPreferredFilename, self.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
self.modelXbrl.modelManager.showStatus(
_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(
self.modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=self.targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(
targetInstance) # need dimension defaults
for context in self.modelXbrl.contexts.values():
newCntx = targetInstance.createContext(
context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in self.modelXbrl.units.values():
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0],
measures[1],
id=unit.id)
self.modelXbrl.modelManager.showStatus(
_("Creating and validating facts"))
for fact in self.modelXbrl.facts:
if fact.isItem:
attrs = [("contextRef", fact.contextID)]
if fact.isNumeric:
attrs.append(("unitRef", fact.unitID))
if fact.get("decimals"):
attrs.append(("decimals", fact.get("decimals")))
if fact.get("precision"):
attrs.append(("precision", fact.get("precision")))
if fact.isNil:
attrs.append(
("{http://www.w3.org/2001/XMLSchema-instance}nil",
"true"))
text = None
else:
text = fact.xValue if fact.xValid else fact.elementText
newFact = targetInstance.createFact(fact.qname,
attributes=attrs,
text=text)
targetInstance.saveInstance(overrideFilepath=targetUrl)
self.modelXbrl.modelManager.showStatus(_("Saved extracted instance"),
5000)
def create(self,
newDocumentType=None,
url=None,
schemaRefs=None,
createModelDocument=True,
isEntry=False,
errorCaptureLevel=None,
initialXml=None,
base=None):
self.modelXbrl = ModelXbrl.create(self, newDocumentType, url,
schemaRefs, createModelDocument,
isEntry, errorCaptureLevel,
initialXml, base)
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def __init__(self, cntlr, isCmdLine=False):
self.cntlr = cntlr
# setup tester
xml = "<rootElement/>"
self.modelXbrl = ModelXbrl.create(cntlr.modelManager, ModelDocument.Type.UnknownNonXML, initialXml=xml, isEntry=True)
self.validator = ValidateXbrl.ValidateXbrl(self.modelXbrl)
self.validator.validate(self.modelXbrl) # required to set up
cntlr.showStatus(_("Initializing Formula Grammar"))
XPathParser.initializeParser(cntlr.modelManager)
cntlr.showStatus(None)
self.trRegs = sorted(ixtNamespaces.keys())
self.trPrefixNSs = dict((qn.prefix, qn.namespaceURI)
for qn in self.modelXbrl.modelManager.customTransforms.keys())
self.trRegs.extend(sorted(self.trPrefixNSs.keys()))
self.trPrefixNSs.update(ixtNamespaces)
def saveTargetDocument(self):
targetUrl = self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(self.targetDocumentPreferredFilename, self.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
self.modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(self.modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=self.targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
for context in self.modelXbrl.contexts.values():
newCntx = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in self.modelXbrl.units.values():
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
self.modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
for fact in self.modelXbrl.facts:
if fact.isItem:
attrs = [("contextRef", fact.contextID)]
if fact.isNumeric:
attrs.append(("unitRef", fact.unitID))
if fact.get("decimals"):
attrs.append(("decimals", fact.get("decimals")))
if fact.get("precision"):
attrs.append(("precision", fact.get("precision")))
if fact.isNil:
attrs.append(("{http://www.w3.org/2001/XMLSchema-instance}nil","true"))
text = None
else:
text = fact.xValue if fact.xValid else fact.elementText
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text)
targetInstance.saveInstance(overrideFilepath=targetUrl)
self.modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def saveTargetDocument(modelXbrl,
targetDocumentFilename,
targetDocumentSchemaRefs,
outputZip=None,
filingFiles=None,
*args,
**kwargs):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
targetDocumentFilename, modelXbrl.modelDocument.filepath)
def addLocallyReferencedFile(elt, filingFiles):
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src") and not isHttpUrl(
attrValue) and not os.path.isabs(attrValue):
attrValue = attrValue.partition('#')[0] # remove anchor
if attrValue: # ignore anchor references to base document
attrValue = os.path.normpath(
attrValue
) # change url path separators to host separators
file = os.path.join(sourceDir, attrValue)
if modelXbrl.fileSource.isInArchive(
file,
checkExistence=True) or os.path.exists(file):
filingFiles.add(file)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(
_("Extracting instance ") + os.path.basename(targetUrl))
rootElt = modelXbrl.modelDocument.xmlRootElement
# take baseXmlLang from <html> or <base>
baseXmlLang = rootElt.get(
"{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang")
for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.w3.org/1999/xhtml}body"):
baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang"
) or rootElt.get("lang") or baseXmlLang
targetInstance = ModelXbrl.create(
modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True,
discover=False) # don't attempt to load DTS
if baseXmlLang:
targetInstance.modelDocument.xmlRootElement.set(
"{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
ValidateXbrlDimensions.loadDimensionDefaults(
targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
addChild(targetInstance.modelDocument.xmlRootElement,
roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in sorted(modelXbrl.contexts.values(),
key=lambda c: elementChildSequence(c)):
ignore = targetInstance.createContext(
context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
ignore = targetInstance.createUnit(measures[0],
measures[1],
id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
if fact.concept is not None and fact.concept.baseXsdType in (
"string", "normalizedString"): # default
xmlLang = fact.xmlLang
if xmlLang is not None and xmlLang != baseXmlLang:
attrs[
"{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang
newFact = targetInstance.createFact(fact.qname,
attributes=attrs,
text=text,
parent=parent)
# if fact.isFraction, create numerator and denominator
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references so that referenced files are included in the zip.
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML("<body>\n{0}\n</body>\n".format(
xmltext)).iter():
addLocallyReferencedFile(elt, filingFiles)
except (XMLSyntaxError, UnicodeDecodeError):
pass # TODO: Why ignore UnicodeDecodeError?
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
modelXbrl.modelManager.showStatus(
_("Creating and validating footnotes and relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
footnoteIdCount = {}
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and any(
lP.modelDocument.type == Type.INLINEXBRL
for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype):
if HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ArcPrototype):
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID,
0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(
attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
newChild = addChild(newLink,
linkChild.qname,
attributes=attributes)
xmlLang = linkChild.xmlLang
if xmlLang is not None and xmlLang != baseXmlLang: # default
newChild.set(
"{http://www.w3.org/XML/1998/namespace}lang",
xmlLang)
copyIxFootnoteHtml(
linkChild,
newChild,
targetModelDocument=targetInstance.modelDocument,
withText=True)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyIxFootnoteHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
addLocallyReferencedFile(elt, filingFiles)
targetInstance.saveInstance(overrideFilepath=targetUrl,
outputZip=outputZip)
if getattr(modelXbrl, "isTestcaseVariation", False):
modelXbrl.extractedInlineInstance = True # for validation comparison
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def runFromExcel(self, options):
#testGenFileName = options.excelfilename
testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls"
testGenDir = os.path.dirname(testGenFileName)
schemaDir = os.path.dirname(testGenDir) + os.sep + "schema"
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
LogHandler(self) # start logger
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt'
modelTestcases = ModelXbrl.create(self.modelManager, url=testGenFileName, isEntry=True)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow))
# start index file
indexFiles = [testGenDir + os.sep + 'creation-testcases-index.xml',
testGenDir + os.sep + 'consumption-testcases-index.xml']
indexDocs = []
testcasesElements = []
for purpose in ("Creation","Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 {0} Tests" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today)
)
doc = etree.parse(file)
file.close()
indexDocs.append(doc)
testcasesElements.append(doc.getroot())
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
try:
row = testcaseIndexSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY or # must have directory
row[1].ctype == xlrd.XL_CELL_EMPTY or # from
row[2].ctype == xlrd.XL_CELL_EMPTY): # to
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
overrideReport = row[3].value
description = row[4].value
if description is None or len(description) == 0:
continue # test not ready to run
assignment = row[5].value
expectedEvents = row[6].value # comma space separated if multiple
note = row[7].value
useCase = row[8].value
base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and assignment.lower() not in ("n.a.", "error") and expectedEvents != "N.A.":
modelDTSfrom = modelDTSto = None
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base)
else:
modelDTS = ModelXbrl.create(self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts",
base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS, uri.strip(), base=base)
if doc is not None:
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom is not None and modelDTSto is not None:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace("from.xsd","report.xml")
reportFile = reportDir + "out" + os.sep + reportName
#reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile,
base)
testcasesDir = os.path.dirname(os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml",
testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"]
for i,testcaseFile in enumerate(testcaseFiles):
etree.SubElement(testcasesElements[i], "testcase",
attrib={"uri":
testcaseFile[len(testGenDir)+1:].replace("\\","/")} )
# start testcase file
testcaseDocs = []
testcaseElements = []
testcaseNumber = testcaseName[0:4]
if testcaseNumber.isnumeric():
testcaseNumberElement = "<number>{0}</number>".format(testcaseNumber)
testcaseName = testcaseName[5:]
else:
testcaseNumberElement = ""
testDirSegments = testDir.split('/')
if len(testDirSegments) >= 2 and '-' in testDirSegments[1]:
testedModule = testDirSegments[1][testDirSegments[1].index('-') + 1:]
else:
testedModule = ''
for purpose in ("Creation","Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'{0}'
'<name>{1}</name>'
# '<description>{0}</description>'
'<reference>'
'{2}'
'{3}'
'</reference>'
'</testcase>'.format(testcaseNumberElement,
testcaseName,
'<name>{0}</name>'.format(testedModule) if testedModule else '',
'<id>{0}</id>'.format(useCase) if useCase else '')
)
doc = etree.parse(file)
file.close()
testcaseDocs.append(doc)
testcaseElements.append(doc.getroot())
priorTestcasesDir = testcasesDir
variationSeq = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(modelTestcases)
modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto,
assignment=assignment,
schemaDir=schemaDir)
# check for expected elements
if expectedEvents:
for expectedEvent in expectedEvents.split(","):
if expectedEvent not in ("No change", "N.A."):
prefix, sep, localName = expectedEvent.partition(':')
if sep and len(modelVersReport.xmlDocument.findall(
'//{{{0}}}{1}'.format(
XbrlConst.verPrefixNS.get(prefix),
localName))) == 0:
modelTestcases.warning("warning",
"Generated test case %(reportName)s missing expected event %(event)s",
reportName=reportName,
event=expectedEvent)
modelVersReport.close()
uriFromParts = uriFrom.split('_')
if len(uriFromParts) >= 2:
variationId = uriFromParts[1]
else:
variationId = "_{0:02n}".format(variationSeq)
for i,testcaseElt in enumerate(testcaseElements):
variationElement = etree.SubElement(testcaseElt, "{http://xbrl.org/2008/conformance}variation",
attrib={"id": variationId})
nameElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
nameElement.text = description
''' (removed per RH 2011/10/04
if note:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
paramElement.text = "Note: " + note
if useCase:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference")
paramElement.set("specification", "versioning-requirements")
paramElement.set("useCase", useCase)
'''
dataElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}data")
if i == 0: # result is report
if expectedEvents:
paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter",
attrib={"name":"expectedEvent",
"value":expectedEvents.replace(',',' ')},
nsmap={"conf":"http://xbrl.org/2008/conformance",
None:""})
if assignment:
paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter",
attrib={"name":"assignment",
"value":assignment},
nsmap={"conf":"http://xbrl.org/2008/conformance",
None:""})
for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")):
for schemaURI in schemaURIs.split(","):
schemaElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}schema")
schemaElement.set("dts",dtsAttr)
if i == 0:
schemaElement.set("readMeFirst","true")
schemaElement.text=os.path.basename(schemaURI.strip())
resultElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}result")
reportElement = etree.SubElement(resultElement if i == 0 else dataElement,
"{http://xbrl.org/2008/conformance}versioningReport")
if i == 1:
reportElement.set("readMeFirst","true")
reportElement.text = "report/" + reportName
variationSeq += 1
except Exception as err:
modelTestcases.error("exception",
_("Exception: %(error)s, Excel row: %(excelRow)s"),
error=err,
excelRow=iRow,
exc_info=True)
# add tests-error-code index files to consumption
for testcaseFile in self.testcaseFiles(testGenDir + os.sep + "tests-error-code"):
etree.SubElement(testcasesElements[1], "testcase",
attrib={"uri":
testcaseFile[len(testGenDir)+1:].replace("\\","/")} )
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i,indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def createTargetInstance(modelXbrl, targetUrl, targetDocumentSchemaRefs, filingFiles, baseXmlLang=None, defaultXmlLang=None):
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True,
discover=False) # don't attempt to load DTS
if baseXmlLang:
targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
if defaultXmlLang is None:
defaultXmlLang = baseXmlLang # allows facts/footnotes to override baseXmlLang
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in sorted(modelXbrl.contexts.values(), key=lambda c: c.objectIndex): # contexts may come from multiple IXDS files
ignore = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in sorted(modelXbrl.units.values(), key=lambda u: u.objectIndex): # units may come from multiple IXDS files
measures = unit.measures
ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default
xmlLang = fact.xmlLang
if xmlLang is not None and xmlLang != defaultXmlLang:
attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
# if fact.isFraction, create numerator and denominator
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references so that referenced files are included in the zip.
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter():
addLocallyReferencedFile(elt, filingFiles)
except (XMLSyntaxError, UnicodeDecodeError):
pass # TODO: Why ignore UnicodeDecodeError?
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
footnoteIdCount = {}
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and
any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype):
if HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
addChild(newLink, linkChild.qname,
attributes=attributes)
elif isinstance(linkChild, ArcPrototype):
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
newChild = addChild(newLink, linkChild.qname,
attributes=attributes)
xmlLang = linkChild.xmlLang
if xmlLang is not None and xmlLang != defaultXmlLang: # default
newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang)
copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyIxFootnoteHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
addLocallyReferencedFile(elt,filingFiles)
return targetInstance
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
elif resultIsTaxonomyPackage:
from arelle import PackageManager, PrototypeInstanceObject
dtsName = readMeFirstUri
modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
else: # need own file source, may need instance discovery
filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
if filesource and not filesource.selection and filesource.isArchive:
for _archiveFile in filesource.dir: # find instance document in archive
filesource.select(_archiveFile)
if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
break # use this selection
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel)
modelXbrl.isTestcaseVariation = True
if modelXbrl.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif resultIsVersioningReport or resultIsTaxonomyPackage:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif resultIsTaxonomyPackage:
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
RenderingEvaluator.init(modelXbrl)
if modelXbrl.hasFormulae:
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
compareIxResultInstance = getattr(modelXbrl, "extractedInlineInstance", False) and modelTestcaseVariation.resultXbrlInstanceUri
if compareIxResultInstance:
formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
errMsgPrefix = "ix"
else: # delete input instances before formula output comparision
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
errMsgPrefix = "formula"
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts),
messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
else:
formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
def factFootnotes(fact, footnotesRelSet):
footnotes = []
footnoteRels = footnotesRelSet.fromModelObject(fact)
if footnoteRels:
# most process rels in same order between two instances, use labels to sort
for i, footnoteRel in enumerate(sorted(footnoteRels,
key=lambda r: (r.fromLabel,r.toLabel))):
modelObject = footnoteRel.toModelObject
if isinstance(modelObject, ModelResource):
xml = modelObject.viewText().strip()
footnotes.append("Footnote {}: {}".format(
i+1, # compare footnote with HTML serialized
xml,
#re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))
))
elif isinstance(modelObject, ModelFact):
footnotes.append("Footnoted fact {}: {} context: {} value: {}".format(
i+1,
modelObject.qname,
modelObject.contextID,
collapseWhitespace(modelObject.value)))
return footnotes
for expectedInstanceFact in expectedInstance.facts:
unmatchedFactsStack = []
formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True)
if formulaOutputFact is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = expectedInstanceFact
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else: # compare footnotes
expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
if expectedInstanceFactFootnotes != formulaOutputFactFootnotes:
formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
_("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=expectedInstanceFactFootnotes, footnotes2=formulaOutputFactFootnotes,
messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
if compareIxResultInstance:
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def __init__(self, mainWin):
parent = mainWin.parent
super(DialogTransformTester, self).__init__(parent)
self.mainWin = mainWin
self.parent = parent
parentGeometry = re.match("(\d+)x(\d+)[+]?([-]?\d+)[+]?([-]?\d+)",
parent.geometry())
dialogX = int(parentGeometry.group(3))
dialogY = int(parentGeometry.group(4))
self.selectedGroup = None
self.transient(self.parent)
self.title(_("Transformation Tester"))
frame = Frame(self)
# setup tester
xml = "<rootElement/>"
self.modelXbrl = ModelXbrl.create(mainWin.modelManager,
ModelDocument.Type.UnknownNonXML,
initialXml=xml,
isEntry=True)
self.validator = ValidateXbrl.ValidateXbrl(self.modelXbrl)
self.validator.validate(self.modelXbrl) # required to set up
mainWin.showStatus(_("Initializing Formula Grammar"))
XPathParser.initializeParser(mainWin.modelManager)
mainWin.showStatus(None)
self.trRegs = sorted(ixtNamespaces.keys())
self.trReg = self.trRegs[-1] # default is latest
self.trPrefixNSs = dict(
(qn.prefix, qn.namespaceURI)
for qn in self.modelXbrl.modelManager.customTransforms.keys())
self.trRegs.extend(sorted(self.trPrefixNSs.keys()))
self.trPrefixNSs.update(ixtNamespaces)
self.trNames = self.getTrNames()
# load grid
trRegLabel = label(frame, 0, 0, _("Registry:"))
self.trRegName = gridCombobox(
frame,
1,
0,
value=self.trReg,
values=self.trRegs,
comboboxselected=self.trRegComboBoxSelected)
trRegToolTipMessage = _("Select Transformation Registry")
ToolTip(self.trRegName, text=trRegToolTipMessage, wraplength=360)
ToolTip(trRegLabel, text=trRegToolTipMessage, wraplength=360)
trNameLabel = label(frame, 0, 1, _("Transform:"))
self.trNameName = gridCombobox(
frame,
1,
1,
value="",
values=self.trNames,
comboboxselected=self.trNameComboBoxSelected)
trRegToolTipMessage = _("Select or enter transform")
ToolTip(self.trRegName, text=trRegToolTipMessage, wraplength=360)
ToolTip(trRegLabel, text=trRegToolTipMessage, wraplength=360)
sourceLabel = label(frame, 0, 2, _("Source text:"))
ToolTip(sourceLabel,
text=_("Enter the source text which is to be transformed. "),
wraplength=240)
self.sourceVar = StringVar()
self.sourceVar.set("")
sourceEntry = Entry(frame, textvariable=self.sourceVar, width=50)
sourceLabel.grid(row=2, column=0, sticky=W)
sourceEntry.grid(row=2, column=1, sticky=EW, pady=3, padx=3)
resultLabel = label(frame, 1, 3, _("Result:"))
ToolTip(sourceLabel, text=_("Transformation result. "), wraplength=240)
self.resultVar = StringVar()
self.resultVar.set("")
resultEntry = Entry(frame, textvariable=self.resultVar, width=50)
resultLabel.grid(row=3, column=0, sticky=W)
resultEntry.grid(row=3, column=1, sticky=EW, pady=3, padx=3)
mainWin.showStatus(None)
btnPad = 2 if mainWin.isMSW else 0 # buttons too narrow on windows
okButton = Button(frame,
text=_("Transform"),
width=8 + btnPad,
command=self.ok)
cancelButton = Button(frame,
text=_("Done"),
width=4 + btnPad,
command=self.close)
cancelButton.grid(row=4,
column=0,
sticky=E,
columnspan=2,
pady=3,
padx=3)
okButton.grid(row=4, column=0, sticky=E, columnspan=2, pady=3, padx=64)
ToolTip(okButton,
text=_("Transform the source entered. "),
wraplength=240)
ToolTip(cancelButton, text=_("Close this dialog. "), wraplength=240)
frame.grid(row=0, column=0, sticky=(N, S, E, W))
frame.columnconfigure(1, weight=3)
frame.columnconfigure(2, weight=1)
window = self.winfo_toplevel()
window.columnconfigure(0, weight=1)
self.geometry("+{0}+{1}".format(dialogX + 150, dialogY + 100))
#self.bind("<Return>", self.ok)
#self.bind("<Escape>", self.close)
self.protocol("WM_DELETE_WINDOW", self.close)
self.grab_set()
self.wait_window(self)
def create(self, newDocumentType=None, url=None, schemaRefs=None, createModelDocument=True):
self.modelXbrl = ModelXbrl.create(self, newDocumentType, url, schemaRefs, createModelDocument)
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def create(self, newDocumentType=None, url=None, schemaRefs=None, createModelDocument=True, isEntry=False, errorCaptureLevel=None, initialXml=None, base=None):
self.modelXbrl = ModelXbrl.create(self, newDocumentType, url, schemaRefs, createModelDocument, isEntry, errorCaptureLevel, initialXml, base)
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def runFromExcel(self, options):
#testGenFileName = options.excelfilename
testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls"
testGenDir = os.path.dirname(testGenFileName)
schemaDir = os.path.dirname(testGenDir) + os.sep + "schema"
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
LogHandler(self) # start logger
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt'
modelTestcases = ModelXbrl.create(self.modelManager,
url=testGenFileName,
isEntry=True)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(
_("[info] xls loaded in {0:.2} secs at {1}").format(
time.time() - startedAt, timeNow))
# start index file
indexFiles = [
testGenDir + os.sep + 'creation-testcases-index.xml',
testGenDir + os.sep + 'consumption-testcases-index.xml'
]
indexDocs = []
testcasesElements = []
for purpose in ("Creation", "Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 {0} Tests" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today))
doc = etree.parse(file)
file.close()
indexDocs.append(doc)
testcasesElements.append(doc.getroot())
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
try:
row = testcaseIndexSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY
or # must have directory
row[1].ctype == xlrd.XL_CELL_EMPTY or # from
row[2].ctype == xlrd.XL_CELL_EMPTY): # to
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
overrideReport = row[3].value
description = row[4].value
if description is None or len(description) == 0:
continue # test not ready to run
assignment = row[5].value
expectedEvents = row[
6].value # comma space separated if multiple
note = row[7].value
useCase = row[8].value
base = os.path.join(os.path.dirname(testGenFileName),
testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and assignment.lower() not in (
"n.a.", "error") and expectedEvents != "N.A.":
modelDTSfrom = modelDTSto = None
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"),
True), (uriTo,
_("loading to DTS"),
False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager,
URIs,
msg,
base=base)
else:
modelDTS = ModelXbrl.create(
self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(
URIs.replace(", ", "_") + ".dts", base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS,
uri.strip(),
base=base)
if doc is not None:
DTSdoc.referencesDocument[
doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom is not None and modelDTSto is not None:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace(
"from.xsd", "report.xml")
reportFile = reportDir + "out" + os.sep + reportName
#reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile, base)
testcasesDir = os.path.dirname(
os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i, testcaseFile in enumerate(
testcaseFiles):
with open(testcaseFile,
"w",
encoding="utf-8") as fh:
XmlUtil.writexml(fh,
testcaseDocs[i],
encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [
testcasesDir + os.sep + testcaseName +
"-creation-testcase.xml",
testcasesDir + os.sep + testcaseName +
"-consumption-testcase.xml"
]
for i, testcaseFile in enumerate(testcaseFiles):
etree.SubElement(
testcasesElements[i],
"testcase",
attrib={
"uri":
testcaseFile[len(testGenDir) +
1:].replace("\\", "/")
})
# start testcase file
testcaseDocs = []
testcaseElements = []
testcaseNumber = testcaseName[0:4]
if testcaseNumber.isnumeric():
testcaseNumberElement = "<number>{0}</number>".format(
testcaseNumber)
testcaseName = testcaseName[5:]
else:
testcaseNumberElement = ""
testDirSegments = testDir.split('/')
if len(testDirSegments
) >= 2 and '-' in testDirSegments[1]:
testedModule = testDirSegments[1][
testDirSegments[1].index('-') + 1:]
else:
testedModule = ''
for purpose in ("Creation", "Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'{0}'
'<name>{1}</name>'
# '<description>{0}</description>'
'<reference>'
'{2}'
'{3}'
'</reference>'
'</testcase>'.format(
testcaseNumberElement, testcaseName,
'<name>{0}</name>'.format(testedModule)
if testedModule else '',
'<id>{0}</id>'.format(useCase)
if useCase else ''))
doc = etree.parse(file)
file.close()
testcaseDocs.append(doc)
testcaseElements.append(doc.getroot())
priorTestcasesDir = testcasesDir
variationSeq = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(
modelTestcases)
modelVersReport.diffDTSes(reportFullPath,
modelDTSfrom,
modelDTSto,
assignment=assignment,
schemaDir=schemaDir)
# check for expected elements
if expectedEvents:
for expectedEvent in expectedEvents.split(","):
if expectedEvent not in ("No change", "N.A."):
prefix, sep, localName = expectedEvent.partition(
':')
if sep and len(
modelVersReport.xmlDocument.
findall('//{{{0}}}{1}'.format(
XbrlConst.verPrefixNS.get(
prefix), localName))) == 0:
modelTestcases.warning(
"warning",
"Generated test case %(reportName)s missing expected event %(event)s",
reportName=reportName,
event=expectedEvent)
modelVersReport.close()
uriFromParts = uriFrom.split('_')
if len(uriFromParts) >= 2:
variationId = uriFromParts[1]
else:
variationId = "_{0:02n}".format(variationSeq)
for i, testcaseElt in enumerate(testcaseElements):
variationElement = etree.SubElement(
testcaseElt,
"{http://xbrl.org/2008/conformance}variation",
attrib={"id": variationId})
nameElement = etree.SubElement(
variationElement,
"{http://xbrl.org/2008/conformance}description"
)
nameElement.text = description
''' (removed per RH 2011/10/04
if note:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
paramElement.text = "Note: " + note
if useCase:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference")
paramElement.set("specification", "versioning-requirements")
paramElement.set("useCase", useCase)
'''
dataElement = etree.SubElement(
variationElement,
"{http://xbrl.org/2008/conformance}data")
if i == 0: # result is report
if expectedEvents:
paramElement = etree.SubElement(
dataElement,
"{http://xbrl.org/2008/conformance}parameter",
attrib={
"name":
"expectedEvent",
"value":
expectedEvents.replace(',', ' ')
},
nsmap={
"conf":
"http://xbrl.org/2008/conformance",
None: ""
})
if assignment:
paramElement = etree.SubElement(
dataElement,
"{http://xbrl.org/2008/conformance}parameter",
attrib={
"name": "assignment",
"value": assignment
},
nsmap={
"conf":
"http://xbrl.org/2008/conformance",
None: ""
})
for schemaURIs, dtsAttr in ((uriFrom, "from"),
(uriTo, "to")):
for schemaURI in schemaURIs.split(","):
schemaElement = etree.SubElement(
dataElement,
"{http://xbrl.org/2008/conformance}schema"
)
schemaElement.set("dts", dtsAttr)
if i == 0:
schemaElement.set(
"readMeFirst", "true")
schemaElement.text = os.path.basename(
schemaURI.strip())
resultElement = etree.SubElement(
variationElement,
"{http://xbrl.org/2008/conformance}result")
reportElement = etree.SubElement(
resultElement if i == 0 else dataElement,
"{http://xbrl.org/2008/conformance}versioningReport"
)
if i == 1:
reportElement.set("readMeFirst", "true")
reportElement.text = "report/" + reportName
variationSeq += 1
except Exception as err:
modelTestcases.error(
"exception",
_("Exception: %(error)s, Excel row: %(excelRow)s"),
error=err,
excelRow=iRow,
exc_info=True)
# add tests-error-code index files to consumption
for testcaseFile in self.testcaseFiles(testGenDir + os.sep +
"tests-error-code"):
etree.SubElement(testcasesElements[1],
"testcase",
attrib={
"uri":
testcaseFile[len(testGenDir) + 1:].replace(
"\\", "/")
})
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i, testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i, indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY):
for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri):
self.validateTestcase(doc) # testcases doc's are sorted by their uri (file names), e.g., for formula
elif hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings:
errorCaptureLevel = logging._checkLevel("WARNING")
else:
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
elif resultIsTaxonomyPackage:
from arelle import PackageManager, PrototypeInstanceObject
dtsName = readMeFirstUri
modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
else: # need own file source, may need instance discovery
filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
if filesource and not filesource.selection and filesource.isArchive:
for _archiveFile in filesource.dir or (): # find instance document in archive
filesource.select(_archiveFile)
if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
break # use this selection
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel)
modelXbrl.isTestcaseVariation = True
if modelXbrl.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif resultIsVersioningReport or resultIsTaxonomyPackage:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif resultIsTaxonomyPackage:
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement)
for d in modelTestcaseVariation.dataUris.values() for uri in d
if not UrlUtil.isAbsolute(uri))
foundDataFiles = set()
variationBase = os.path.dirname(baseForElement)
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
for _inputDTS in inputDTS:
for docUrl, doc in _inputDTS.urlDocs.items():
if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET:
if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file
if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles:
docUrl = docUrl.replace("-formula.xml", ".xf")
foundDataFiles.add(docUrl)
if expectedDataFiles - foundDataFiles:
modelXbrl.info("arelle:testcaseDataNotUsed",
_("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"),
modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles)))
if foundDataFiles - expectedDataFiles:
modelXbrl.info("arelle:testcaseDataUnexpected",
_("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"),
modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles)))
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
try:
RenderingEvaluator.init(modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrlHasFormulae = modelXbrl.hasFormulae
if modelXbrlHasFormulae:
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
try:
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and
modelTestcaseVariation.resultXbrlInstanceUri is not None)
if compareIxResultInstance:
formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
errMsgPrefix = "ix"
else: # delete input instances before formula output comparision
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
errMsgPrefix = "formula"
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
_matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
_("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts),
messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
else:
formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
def factFootnotes(fact, footnotesRelSet):
footnotes = {}
footnoteRels = footnotesRelSet.fromModelObject(fact)
if footnoteRels:
# most process rels in same order between two instances, use labels to sort
for i, footnoteRel in enumerate(sorted(footnoteRels,
key=lambda r: (r.fromLabel,r.toLabel))):
modelObject = footnoteRel.toModelObject
if isinstance(modelObject, ModelResource):
xml = modelObject.viewText().strip()
footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))
elif isinstance(modelObject, ModelFact):
footnotes["Footnoted fact {}".format(i+1)] = \
"{} context: {} value: {}".format(
modelObject.qname,
modelObject.contextID,
collapseWhitespace(modelObject.value))
return footnotes
for expectedInstanceFact in expectedInstance.facts:
unmatchedFactsStack = []
formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False)
#formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True)
if formulaOutputFact is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = expectedInstanceFact
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else: # compare footnotes
expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or
set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())):
formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
_("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()),
messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
if compareIxResultInstance:
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
_statusCounts = OrderedDict((("pass",0),("fail",0)))
for tv in getattr(testcase, "testcaseVariations", ()):
_statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1
self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k))
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def validateTestcase(self, testcase):
self.modelXbrl.error(_("testcase {0}").format(os.path.basename(testcase.uri)))
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation.element)
# try to load instance document
self.modelXbrl.error(_("variation {0} {1}: {2}").format(modelTestcaseVariation.id, modelTestcaseVariation.name, modelTestcaseVariation.expected))
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
ModelDocument.Type.DTSENTRIES,
self.modelXbrl.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = ModelDocument.load(modelXbrl, readMeFirstUri, base=baseForElement)
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
else: # not a multi-schemaRef versioning report
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement)
if modelXbrl.modelDocument is None:
self.modelXbrl.error(_("Testcase {0} {1} document not loaded: {2}").format(
modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(readMeFirstUri)))
modelTestcaseVariation.status = "not loadable"
modelXbrl.close()
elif resultIsVersioningReport:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl) \
.validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
modelXbrl.close()
elif testcase.type == ModelDocument.Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
self.modelXbrl.error(_("Testcase {0} {1} DTSes not loaded, unable to generate versioning report").format(
modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(readMeFirstUri)))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes:
inputDTS.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
parameters = modelTestcaseVariation.parameters.copy()
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName:
parameters[dtsName] = (None, inputDTS)
self.instValidator.validate(modelXbrl, parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
if modelXbrl.formulaOutputInstance and len(modelTestcaseVariation.actual) == 0:
# if an output instance is created, validate it
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance)
if len(modelTestcaseVariation.actual) == 0: # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement)
if expectedInstance.modelDocument is None:
self.modelXbrl.error(_("Testcase {0} {1} expected result instance not loaded: {2}").format(
modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(modelTestcaseVariation.resultXbrlInstance)))
modelTestcaseVariation.status = "result not loadable"
expectedInstance.close()
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error(
_("Formula output {0} facts, expected {1} facts").format(
len(formulaOutputInstance.facts),
len(expectedInstance.facts)),
"err", "formula:resultFactCounts")
else:
for fact in expectedInstance.facts:
if not formulaOutputInstance.matchFact(fact):
formulaOutputInstance.error(
_("Formula output missing expected fact {0}").format(
fact),
"err", "formula:expectedFactMissing")
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance)
formulaOutputInstance.close()
formulaOutputInstance = None
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def runFromExcel(self, options):
testGenFileName = options.excelfilename
#testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls"
testGenDir = os.path.dirname(testGenFileName)
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt'
modelTestcases = ModelXbrl.create(self.modelManager)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(
_("[info] xls loaded in {0:.2} secs at {1}").format(
time.time() - startedAt, timeNow))
# start index file
indexFiles = [
testGenDir + os.sep + 'creationTestcasesIndex.xml',
testGenDir + os.sep + 'consumptionTestcasesIndex.xml'
]
indexDocs = [
xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today))
for purpose in ("Creation", "Consumption")
]
testcasesElements = [
XmlUtil.child(indexDoc, None, "testcases")
for indexDoc in indexDocs
]
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
row = testcaseIndexSheet.row(iRow)
if row[0].ctype == xlrd.XL_CELL_EMPTY or row[
1].ctype == xlrd.XL_CELL_EMPTY or row[
2].ctype == xlrd.XL_CELL_EMPTY:
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
intention = row[3].value
if intention is None or len(intention) == 0:
continue # test not ready to run
reason = row[4].value
expectedEvent = row[5].value
base = os.path.join(os.path.dirname(testGenFileName),
testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and reason.lower() not in (
"n.a.", "error") and expectedEvent != "N.A.":
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"),
True), (uriTo, _("loading to DTS"),
False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager,
URIs,
msg,
base=base)
else:
modelDTS = ModelXbrl.create(
self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(
URIs.replace(", ", "_") + ".dts", base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS,
uri.strip(),
base=base)
DTSdoc.referencesDocument[
doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom and modelDTSto:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace(
"from.xsd", "report.xml")
reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile, base)
testcasesDir = os.path.dirname(
os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i, testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w",
encoding="utf-8") as fh:
XmlUtil.writexml(fh,
testcaseDocs[i],
encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [
testcasesDir + os.sep + testcaseName +
"-creation-testcase.xml", testcasesDir + os.sep +
testcaseName + "-consumption-testcase.xml"
]
for i, testcaseFile in enumerate(testcaseFiles):
XmlUtil.addChild(
testcasesElements[i], None, "testcase",
("uri", testcaseFile[len(testGenDir) +
1:].replace("\\", "/")))
# start testcase file
testcaseDocs = [
xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'<name>{0}</name>'
'<description>{0}</description>'
'</testcase>'.format(testcaseName, purpose,
today))
for purpose in ("Creation", "Consumption")
]
testcaseElements = [
XmlUtil.child(testcaseDoc, conformanceNS,
"testcase")
for testcaseDoc in testcaseDocs
]
priorTestcasesDir = testcasesDir
variationID = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(
modelTestcases)
modelVersReport.diffDTSes(reportFullPath, modelDTSfrom,
modelDTSto)
# check for expected elements
if expectedEvent and expectedEvent not in ("No change",
"N.A."):
if len(
modelVersReport.xmlDocument.
getElementsByTagNameNS('*',
expectedEvent)) == 0:
modelTestcases.error(
"Generated test case {0} missing expected event {1}"
.format(reportName,
expectedEvent), "wrn", "missingEvent")
modelVersReport.close([])
for i, testcaseElt in enumerate(testcaseElements):
variationElement = XmlUtil.addChild(
testcaseElt,
conformanceNS,
"variation",
attributes=("id", "_{0:02n}".format(variationID)))
XmlUtil.addChild(variationElement,
conformanceNS,
"name",
text=intention)
dataElement = XmlUtil.addChild(variationElement,
conformanceNS, "data")
for schemaURIs, dtsAttr in ((uriFrom, "from"), (uriTo,
"to")):
for schemaURI in schemaURIs.split(","):
XmlUtil.addChild(
dataElement,
conformanceNS,
"schema",
attributes=((("dts", dtsAttr), ) +
((("readMeFirst",
"true"), ) if i == 0 else
())),
text=os.path.basename(schemaURI.strip()))
resultElement = XmlUtil.addChild(
variationElement, conformanceNS, "result")
XmlUtil.addChild(
resultElement if i == 0 else dataElement,
conformanceNS,
"versioningReport",
attributes=(("readMeFirst", "true") if i == 1 else
()),
text="report/" + reportName)
variationID += 1
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i, testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i, indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def runFromExcel(self, options):
testGenFileName = options.excelfilename
#testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls"
testGenDir = os.path.dirname(testGenFileName)
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt'
modelTestcases = ModelXbrl.create(self.modelManager)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow))
# start index file
indexFiles = [testGenDir + os.sep + 'creationTestcasesIndex.xml',
testGenDir + os.sep + 'consumptionTestcasesIndex.xml']
indexDocs = [xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today)
) for purpose in ("Creation","Consumption")]
testcasesElements = [XmlUtil.child(indexDoc, None, "testcases") for indexDoc in indexDocs]
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
row = testcaseIndexSheet.row(iRow)
if row[0].ctype == xlrd.XL_CELL_EMPTY or row[1].ctype == xlrd.XL_CELL_EMPTY or row[2].ctype == xlrd.XL_CELL_EMPTY:
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
intention = row[3].value
if intention is None or len(intention) == 0:
continue # test not ready to run
reason = row[4].value
expectedEvent = row[5].value
base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and reason.lower() not in ("n.a.", "error") and expectedEvent != "N.A.":
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base)
else:
modelDTS = ModelXbrl.create(self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts",
base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS, uri.strip(), base=base)
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom and modelDTSto:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace("from.xsd","report.xml")
reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile,
base)
testcasesDir = os.path.dirname(os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml",
testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"]
for i,testcaseFile in enumerate(testcaseFiles):
XmlUtil.addChild(testcasesElements[i], None, "testcase",
("uri",
testcaseFile[len(testGenDir)+1:].replace("\\","/")) )
# start testcase file
testcaseDocs = [xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'<name>{0}</name>'
'<description>{0}</description>'
'</testcase>'.format(testcaseName,purpose,today)
) for purpose in ("Creation","Consumption")]
testcaseElements = [XmlUtil.child(testcaseDoc, conformanceNS, "testcase") for testcaseDoc in testcaseDocs]
priorTestcasesDir = testcasesDir
variationID = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(modelTestcases)
modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto)
# check for expected elements
if expectedEvent and expectedEvent not in (
"No change", "N.A."):
if len(modelVersReport.xmlDocument.getElementsByTagNameNS('*',expectedEvent)) == 0:
modelTestcases.error(
"Generated test case {0} missing expected event {1}".format(
reportName,
expectedEvent),
"wrn", "missingEvent")
modelVersReport.close([])
for i,testcaseElt in enumerate(testcaseElements):
variationElement = XmlUtil.addChild(testcaseElt, conformanceNS, "variation",
attributes=("id", "_{0:02n}".format(variationID)))
XmlUtil.addChild(variationElement, conformanceNS, "name", text=intention)
dataElement = XmlUtil.addChild(variationElement, conformanceNS, "data")
for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")):
for schemaURI in schemaURIs.split(","):
XmlUtil.addChild(dataElement, conformanceNS, "schema",
attributes=((("dts",dtsAttr),) +
((("readMeFirst","true"),) if i == 0 else ())),
text=os.path.basename(schemaURI.strip()))
resultElement = XmlUtil.addChild(variationElement, conformanceNS, "result")
XmlUtil.addChild(resultElement if i == 0 else dataElement,
conformanceNS, "versioningReport",
attributes=(("readMeFirst","true") if i == 1 else ()),
text="report/" + reportName)
variationID += 1
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i,indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def validate(val):
formulaOptions = val.modelXbrl.modelManager.formulaOptions
XPathParser.initializeParser(val)
val.modelXbrl.modelManager.showStatus(_("Compiling formulae"))
initialErrorCount = val.modelXbrl.logCountErr
# global parameter names
parameterQnames = set()
instanceQnames = set()
parameterDependencies = {}
instanceDependencies = defaultdict(
set) # None-key entries are non-formula dependencies
dependencyResolvedParameters = set()
orderedParameters = []
orderedInstances = []
for paramQname, modelParameter in val.modelXbrl.qnameParameters.items():
if isinstance(modelParameter, ModelParameter):
modelParameter.compile()
parameterDependencies[paramQname] = modelParameter.variableRefs()
parameterQnames.add(paramQname)
if isinstance(modelParameter, ModelInstance):
instanceQnames.add(paramQname)
# duplicates checked on loading modelDocument
#resolve dependencies
resolvedAParameter = True
while (resolvedAParameter):
resolvedAParameter = False
for paramQname in parameterQnames:
if paramQname not in dependencyResolvedParameters and \
len(parameterDependencies[paramQname] - dependencyResolvedParameters) == 0:
dependencyResolvedParameters.add(paramQname)
orderedParameters.append(paramQname)
resolvedAParameter = True
# anything unresolved?
for paramQname in parameterQnames:
if paramQname not in dependencyResolvedParameters:
circularOrUndefDependencies = parameterDependencies[
paramQname] - dependencyResolvedParameters
undefinedVars = circularOrUndefDependencies - parameterQnames
paramsCircularDep = circularOrUndefDependencies - undefinedVars
if len(undefinedVars) > 0:
val.modelXbrl.error(
_("Undefined dependencies in parameter {0}, to names {1}"
).format(paramQname, ", ".join(
(str(v) for v in undefinedVars))), "err",
"xbrlve:unresolvedDependency")
if len(paramsCircularDep) > 0:
val.modelXbrl.error(
_("Cyclic dependencies in parameter {0}, to names {1}"
).format(paramQname, ", ".join(
(str(d) for d in paramsCircularDep))), "err",
"xbrlve:parameterCyclicDependencies")
for custFnSig in val.modelXbrl.modelCustomFunctionSignatures.values():
custFnQname = custFnSig.qname
if custFnQname.namespaceURI == "XbrlConst.xfi":
val.modelXbrl.error(
_("Custom function {0} has namespace reserved for functions in the function registry {1}"
).format(str(custFnQname), custFnQname.namespaceURI), "err",
"xbrlve:noProhibitedNamespaceForCustomFunction")
# any custom function implementations?
for modelRel in val.modelXbrl.relationshipSet(
XbrlConst.functionImplementation).fromModelObject(custFnSig):
custFnImpl = modelRel.toModelObject
custFnSig.customFunctionImplementation = custFnImpl
if len(custFnImpl.inputNames) != len(custFnSig.inputTypes):
val.modelXbrl.error(
_("Custom function {0} signature has {1} parameters but implementation has {2}, must be matching"
).format(str(custFnQname), len(custFnSig.inputTypes),
len(custFnImpl.inputNames)), "err",
"xbrlcfie:inputMismatch")
for custFnImpl in val.modelXbrl.modelCustomFunctionImplementations:
if not val.modelXbrl.relationshipSet(
XbrlConst.functionImplementation).toModelObject(custFnImpl):
val.modelXbrl.error(
_("Custom function implementation {0} has no relationship from any custom function signature"
).format(custFnImpl.xlinkLabel), "err",
"xbrlcfie:missingCFIRelationship")
custFnImpl.compile()
# xpathContext is needed for filter setup for expressions such as aspect cover filter
# determine parameter values
xpathContext = XPathContext.create(val.modelXbrl)
for paramQname in orderedParameters:
if not isinstance(modelParameter, ModelInstance):
modelParameter = val.modelXbrl.qnameParameters[paramQname]
asType = modelParameter.asType
asLocalName = asType.localName if asType else "string"
try:
if val.parameters and paramQname in val.parameters:
paramDataType, paramValue = val.parameters[paramQname]
typeLocalName = paramDataType.localName if paramDataType else "string"
value = FunctionXs.call(xpathContext, None, typeLocalName,
[paramValue])
result = FunctionXs.call(xpathContext, None, asLocalName,
[value])
if formulaOptions.traceParameterInputValue:
val.modelXbrl.error(
_("Parameter {0} input {1}").format(
paramQname, result), "info", "formula:trace")
else:
result = modelParameter.evaluate(xpathContext, asType)
if formulaOptions.traceParameterExpressionResult:
val.modelXbrl.error(
_("Parameter {0} result {1}").format(
paramQname, result), "info", "formula:trace")
xpathContext.inScopeVars[
paramQname] = result # make visible to subsequent parameter expression
except XPathContext.XPathException as err:
val.modelXbrl.error(
_("Parameter \n{0} \nException: \n{1}").format(
paramQname, err.message), "err",
"xbrlve:parameterTypeMismatch"
if err.code == "err:FORG0001" else err.code)
produceOutputXbrlInstance = False
instanceProducingVariableSets = defaultdict(list)
for modelVariableSet in val.modelXbrl.modelVariableSets:
varSetInstanceDependencies = set()
if isinstance(modelVariableSet, ModelFormula):
instanceQname = None
for modelRel in val.modelXbrl.relationshipSet(
XbrlConst.formulaInstance).fromModelObject(
modelVariableSet):
instance = modelRel.toModelObject
if isinstance(instance, ModelInstance):
if instanceQname is None:
instanceQname = instance.qname
else:
val.modelXbrl.error(
_("Multiple output instances for formula {0}, to names {1}, {2}"
).format(modelVariableSet.xlinkLabel,
instanceQname, instance.qname), "info",
"arelle:multipleOutputInstances")
if instanceQname is None:
instanceQname = XbrlConst.qnStandardOutputInstance
instanceQnames.add(instanceQname)
modelVariableSet.outputInstanceQname = instanceQname
if val.validateSBRNL:
val.modelXbrl.error(
_("Formula linkbase {0} formula:formula {1} is not allowed"
).format(
os.path.basename(modelVariableSet.modelDocument.uri),
modelVariableSet.xlinkLabel), "err",
"SBR.NL.2.3.9.03")
else:
instanceQname = None
modelVariableSet.countSatisfied = 0
modelVariableSet.countNotSatisfied = 0
checkValidationMessages(val, modelVariableSet)
instanceProducingVariableSets[instanceQname].append(modelVariableSet)
modelVariableSet.outputInstanceQname = instanceQname
if modelVariableSet.aspectModel not in ("non-dimensional",
"dimensional"):
val.modelXbrl.error(
_("Variable set {0}, aspect model {1} not recognized").format(
modelVariableSet.xlinkLabel, modelVariableSet.aspectModel),
"err", "xbrlve:unknownAspectModel")
modelVariableSet.compile()
modelVariableSet.hasConsistencyAssertion = False
#determine dependencies within variable sets
nameVariables = {}
qnameRels = {}
definedNamesSet = set()
for modelRel in val.modelXbrl.relationshipSet(
XbrlConst.variableSet).fromModelObject(modelVariableSet):
varqname = modelRel.variableQname
if varqname:
qnameRels[varqname] = modelRel
toVariable = modelRel.toModelObject
if varqname not in definedNamesSet:
definedNamesSet.add(varqname)
if varqname not in nameVariables:
nameVariables[varqname] = toVariable
elif nameVariables[varqname] != toVariable:
val.modelXbrl.error(
_("Multiple variables named {1} in variable set {0}"
).format(modelVariableSet.xlinkLabel, varqname),
"err", "xbrlve:duplicateVariableNames")
fromInstanceQnames = None
for instRel in val.modelXbrl.relationshipSet(
XbrlConst.instanceVariable).toModelObject(toVariable):
fromInstance = instRel.fromModelObject
if isinstance(fromInstance, ModelInstance):
fromInstanceQname = fromInstance.qname
varSetInstanceDependencies.add(fromInstanceQname)
instanceDependencies[instanceQname].add(
fromInstanceQname)
if fromInstanceQnames is None:
fromInstanceQnames = set()
fromInstanceQnames.add(fromInstanceQname)
if fromInstanceQnames is None:
varSetInstanceDependencies.add(
XbrlConst.qnStandardInputInstance)
if instanceQname:
instanceDependencies[instanceQname].add(
XbrlConst.qnStandardInputInstance)
toVariable.fromInstanceQnames = fromInstanceQnames
else:
val.modelXbrl.error(
_("Variables name {1} cannot be determined on arc from {0}"
).format(modelVariableSet.xlinkLabel,
modelRel.variablename), "err",
"xbrlve:variableNameResolutionFailure")
definedNamesSet |= parameterQnames
variableDependencies = {}
for modelRel in val.modelXbrl.relationshipSet(
XbrlConst.variableSet).fromModelObject(modelVariableSet):
variable = modelRel.toModelObject
if isinstance(
variable,
(ModelParameter,
ModelVariable)): # ignore anything not parameter or variable
varqname = modelRel.variableQname
depVars = variable.variableRefs()
variableDependencies[varqname] = depVars
if len(depVars
) > 0 and formulaOptions.traceVariablesDependencies:
val.modelXbrl.error(
_("Variable set {0}, variable {1}, dependences {2}").
format(modelVariableSet.xlinkLabel, varqname,
depVars), "info", "formula:trace")
definedNamesSet.add(varqname)
# check for fallback value variable references
if isinstance(variable, ModelFactVariable):
for depVar in XPathParser.variableReferencesSet(
variable.fallbackValueProg, variable.element):
if depVar in qnameRels and isinstance(
qnameRels[depVar].toModelObject,
ModelVariable):
val.modelXbrl.error(
_("Variable set {0} fallbackValue '{1}' cannot refer to variable {2}"
).format(modelVariableSet.xlinkLabel,
variable.fallbackValue, depVar),
"err",
"xbrlve:factVariableReferenceNotAllowed")
# check for covering aspect not in variable set aspect model
checkFilterAspectModel(val, modelVariableSet,
variable.filterRelationships,
xpathContext)
orderedNameSet = set()
orderedNameList = []
orderedAVariable = True
while (orderedAVariable):
orderedAVariable = False
for varqname, depVars in variableDependencies.items():
if varqname not in orderedNameSet and len(depVars -
parameterQnames -
orderedNameSet) == 0:
orderedNameList.append(varqname)
orderedNameSet.add(varqname)
orderedAVariable = True
if varqname in instanceQnames:
varSetInstanceDependencies.add(varqname)
instanceDependencies[instanceQname].add(varqname)
elif isinstance(nameVariables.get(varqname), ModelInstance):
instqname = nameVariables[varqname].qname
varSetInstanceDependencies.add(instqname)
instanceDependencies[instanceQname].add(instqname)
# anything unresolved?
for varqname, depVars in variableDependencies.items():
if varqname not in orderedNameSet:
circularOrUndefVars = depVars - parameterQnames - orderedNameSet
undefinedVars = circularOrUndefVars - definedNamesSet
varsCircularDep = circularOrUndefVars - undefinedVars
if len(undefinedVars) > 0:
val.modelXbrl.error(
_("Undefined variable dependencies in variable st {0}, from variable {1} to {2}"
).format(modelVariableSet.xlinkLabel, varqname,
undefinedVars), "err",
"xbrlve:unresolvedDependency")
if len(varsCircularDep) > 0:
val.modelXbrl.error(
_("Cyclic dependencies in variable set {0}, from variable {1} to {2}"
).format(modelVariableSet.xlinkLabel, varqname,
varsCircularDep), "err",
"xbrlve:cyclicDependencies")
# check unresolved variable set dependencies
for varSetDepVarQname in modelVariableSet.variableRefs():
if varSetDepVarQname not in orderedNameSet and varSetDepVarQname not in parameterQnames:
val.modelXbrl.error(
_("Undefined variable dependency in variable set {0}, {1}"
).format(modelVariableSet.xlinkLabel, varSetDepVarQname),
"err", "xbrlve:unresolvedDependency")
if varSetDepVarQname in instanceQnames:
varSetInstanceDependencies.add(varSetDepVarQname)
instanceDependencies[instanceQname].add(varSetDepVarQname)
elif isinstance(nameVariables.get(varSetDepVarQname),
ModelInstance):
instqname = nameVariables[varSetDepVarQname].qname
varSetInstanceDependencies.add(instqname)
instanceDependencies[instanceQname].add(instqname)
if formulaOptions.traceVariablesOrder:
val.modelXbrl.error(
_("Variable set {0}, variables order: {1}").format(
modelVariableSet.xlinkLabel, orderedNameList), "info",
"formula:trace")
if (formulaOptions.traceVariablesDependencies
and len(varSetInstanceDependencies) > 0
and varSetInstanceDependencies !=
{XbrlConst.qnStandardInputInstance}):
val.modelXbrl.error(
_("Variable set {0}, instance dependences {1}").format(
modelVariableSet.xlinkLabel, varSetInstanceDependencies),
"info", "formula:trace")
modelVariableSet.orderedVariableRelationships = []
for varqname in orderedNameList:
if varqname in qnameRels:
modelVariableSet.orderedVariableRelationships.append(
qnameRels[varqname])
# check existence assertion variable dependencies
if isinstance(modelVariableSet, ModelExistenceAssertion):
for depVar in modelVariableSet.variableRefs():
if depVar in qnameRels and isinstance(
qnameRels[depVar].toModelObject, ModelVariable):
val.modelXbrl.error(
_("Existence Assertion {0}, cannot refer to variable {1}"
).format(modelVariableSet.xlinkLabel, depVar), "err",
"xbrleae:variableReferenceNotAllowed")
# check messages variable dependencies
checkValidationMessageVariables(val, modelVariableSet, qnameRels)
# check preconditions
modelVariableSet.preconditions = []
for modelRel in val.modelXbrl.relationshipSet(
XbrlConst.variableSetPrecondition).fromModelObject(
modelVariableSet):
precondition = modelRel.toModelObject
if isinstance(precondition, ModelPrecondition):
modelVariableSet.preconditions.append(precondition)
# check for variable sets referencing fact or general variables
for modelRel in val.modelXbrl.relationshipSet(
XbrlConst.variableSetFilter).fromModelObject(modelVariableSet):
varSetFilter = modelRel.toModelObject
if modelRel.isCovered:
val.modelXbrl.error(
_("Variable set {0}, filter {1}, cannot be covered"
).format(modelVariableSet.xlinkLabel,
varSetFilter.xlinkLabel), "wrn",
"arelle:variableSetFilterCovered")
modelRel._isCovered = False # block group filter from being able to covere
for depVar in varSetFilter.variableRefs():
if depVar in qnameRels and isinstance(
qnameRels[depVar].toModelObject, ModelVariable):
val.modelXbrl.error(
_("Variable set {0}, filter {1}, cannot refer to variable {2}"
).format(modelVariableSet.xlinkLabel,
varSetFilter.xlinkLabel, depVar), "err",
"xbrlve:factVariableReferenceNotAllowed")
# check aspects of formula
if isinstance(modelVariableSet, ModelFormula):
checkFormulaRules(val, modelVariableSet, nameVariables)
# determine instance dependency order
orderedInstancesSet = set()
stdInpInst = {XbrlConst.qnStandardInputInstance}
orderedInstancesList = []
orderedAnInstance = True
while (orderedAnInstance):
orderedAnInstance = False
for instqname, depInsts in instanceDependencies.items():
if instqname and instqname not in orderedInstancesSet and len(
depInsts - stdInpInst - orderedInstancesSet) == 0:
orderedInstancesList.append(instqname)
orderedInstancesSet.add(instqname)
orderedAnInstance = True
orderedInstancesList.append(
None) # assertions come after all formulas that produce outputs
# anything unresolved?
for instqname, depInsts in instanceDependencies.items():
if instqname not in orderedInstancesSet:
# can also be satisfied from an input DTS
missingDependentInstances = depInsts - stdInpInst
if val.parameters:
missingDependentInstances -= val.parameters.keys()
if instqname:
if missingDependentInstances:
val.modelXbrl.error(
_("Cyclic dependencies of instance {0} produced by a formula, with variables consuming instances {1}"
).format(instqname, missingDependentInstances),
"err", "xbrlvarinste:instanceVariableRecursionCycle")
elif instqname == XbrlConst.qnStandardOutputInstance:
orderedInstancesSet.add(instqname)
orderedInstancesList.append(
instqname
) # standard output formula, all input dependencies in parameters
''' future check? if instance has no external input or producing formula
else:
val.modelXbrl.error(
_("Unresolved dependencies of an assertion's variables on instances {0}").format(
depInsts - stdInpInst ),
"err", "xbrlvarinste:instanceVariableRecursionCycle")
'''
if formulaOptions.traceVariablesOrder and len(orderedInstancesList) > 1:
val.modelXbrl.error(
_("Variable instances processing order: {0}").format(
orderedInstancesList), "info", "formula:trace")
# linked consistency assertions
for modelRel in val.modelXbrl.relationshipSet(
XbrlConst.consistencyAssertionFormula).modelRelationships:
if modelRel.fromModelObject and modelRel.toModelObject and isinstance(
modelRel.toModelObject, ModelFormula):
consisAsser = modelRel.fromModelObject
consisAsser.countSatisfied = 0
consisAsser.countNotSatisfied = 0
if consisAsser.hasProportionalAcceptanceRadius and consisAsser.hasAbsoluteAcceptanceRadius:
val.modelXbrl.error(
_("Consistency assertion {0} has both absolute and proportional acceptance radii"
).format(consisAsser.xlinkLabel), "err",
"xbrlcae:acceptanceRadiusConflict")
consisAsser.orderedVariableRelationships = []
for consisParamRel in val.modelXbrl.relationshipSet(
XbrlConst.consistencyAssertionParameter).fromModelObject(
consisAsser):
if isinstance(consisParamRel.toModelObject, ModelVariable):
val.modelXbrl.error(
_("Consistency assertion {0} has relationship to a {1} {2}"
).format(
consisAsser.xlinkLabel,
consisParamRel.toModelObject.element.localName,
consisParamRel.toModelObject.xlinkLabel), "err",
"xbrlcae:variablesNotAllowed")
else:
consisAsser.orderedVariableRelationships.append(
consisParamRel)
consisAsser.compile()
modelRel.toModelObject.hasConsistencyAssertion = True
if initialErrorCount < val.modelXbrl.logCountErr:
return # don't try to execute
# formula output instances
if instanceQnames:
schemaRefs = [
val.modelXbrl.modelDocument.relativeUri(referencedDoc.uri)
for referencedDoc in
val.modelXbrl.modelDocument.referencesDocument.keys()
if referencedDoc.type == ModelDocument.Type.SCHEMA
]
outputXbrlInstance = None
for instanceQname in instanceQnames:
if instanceQname == XbrlConst.qnStandardInputInstance:
continue # always present the standard way
if val.parameters and instanceQname in val.parameters:
namedInstance = val.parameters[instanceQname][1]
else: # empty intermediate instance
uri = val.modelXbrl.modelDocument.filepath[:-4] + "-output-XBRL-instance"
if instanceQname != XbrlConst.qnStandardOutputInstance:
uri = uri + "-" + instanceQname.localName
uri = uri + ".xml"
namedInstance = ModelXbrl.create(
val.modelXbrl.modelManager,
newDocumentType=ModelDocument.Type.INSTANCE,
url=uri,
schemaRefs=schemaRefs,
isEntry=True)
xpathContext.inScopeVars[instanceQname] = namedInstance
if instanceQname == XbrlConst.qnStandardOutputInstance:
outputXbrlInstance = namedInstance
# evaluate consistency assertions
# evaluate variable sets not in consistency assertions
for instanceQname in orderedInstancesList:
for modelVariableSet in instanceProducingVariableSets[instanceQname]:
# produce variable evaluations
from arelle.FormulaEvaluator import evaluate
try:
evaluate(xpathContext, modelVariableSet)
except XPathContext.XPathException as err:
val.modelXbrl.error(
_("Variable set \n{0} \nException: \n{1}").format(
modelVariableSet, err.message), "err", err.code)
# log assertion result counts
asserTests = {}
for exisValAsser in val.modelXbrl.modelVariableSets:
if isinstance(exisValAsser, ModelVariableSetAssertion):
asserTests[exisValAsser.id] = (exisValAsser.countSatisfied,
exisValAsser.countNotSatisfied)
if formulaOptions.traceAssertionResultCounts:
val.modelXbrl.error(
_("{0} Assertion {1} evaluations : {2} satisfied, {3} not satisfied"
).format(
"Existence"
if isinstance(exisValAsser,
ModelExistenceAssertion) else "Value",
exisValAsser.id, exisValAsser.countSatisfied,
exisValAsser.countNotSatisfied), "info",
"formula:trace")
for modelRel in val.modelXbrl.relationshipSet(
XbrlConst.consistencyAssertionFormula).modelRelationships:
if modelRel.fromModelObject and modelRel.toModelObject and isinstance(
modelRel.toModelObject, ModelFormula):
consisAsser = modelRel.fromModelObject
asserTests[consisAsser.id] = (consisAsser.countSatisfied,
consisAsser.countNotSatisfied)
if formulaOptions.traceAssertionResultCounts:
val.modelXbrl.error(
_("Consistency Assertion {0} evaluations : {1} satisfied, {2} not satisfied"
).format(consisAsser.id, consisAsser.countSatisfied,
consisAsser.countNotSatisfied), "info",
"formula:trace")
if asserTests:
val.modelXbrl.error(
_("Assertion results {0}").format(asserTests), "asrtNoLog",
asserTests)
# display output instance
if outputXbrlInstance:
if val.modelXbrl.formulaOutputInstance:
# close prior instance, usually closed by caller to validate as it may affect UI on different thread
val.modelXbrl.formulaOutputInstance.close()
val.modelXbrl.formulaOutputInstance = outputXbrlInstance
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(
self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(
modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info(
"info",
_("Variation %(id)s %(name)s: %(expected)s - %(description)s"
),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri, tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(
self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.
normalizeUrl(readMeFirstUri[:-4] + ".dts",
baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl,
readMeFirstUri,
base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[
doc] = ModelDocumentReference(
"import",
DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
else: # not a multi-schemaRef versioning report
modelXbrl = ModelXbrl.load(
self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if modelXbrl.modelDocument is None:
self.modelXbrl.error(
"arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"
),
modelXbrl=testcase,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
file=os.path.basename(readMeFirstUri))
modelXbrl.close()
self.determineNotLoadedTestStatus(
modelTestcaseVariation)
elif resultIsVersioningReport:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(
self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation,
modelXbrl)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(
modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(
modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall,
modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation,
modelXbrl)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri,
baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(
self.modelXbrl.modelManager, versReportFile,
_("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(
self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation,
modelVersReport)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(
self.modelXbrl).diffDTSes(versReportFile,
inputDTSes["from"],
inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
self.modelXbrl.error(
"arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"
),
modelXbrl=testcase,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
for pluginXbrlMethod in pluginClassMethods(
"TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
parameters = modelTestcaseVariation.parameters.copy()
for dtsName, inputDTS in inputDTSes.items(
): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (
None, inputDTS
) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(
inputDTS
) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (
None, inputDTS
) # allow error detection in validateFormula
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
RenderingEvaluator.init(modelXbrl)
try:
self.instValidator.validate(modelXbrl, parameters)
except Exception as err:
self.modelXbrl.error(
"exception",
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"
),
modelXbrl=modelXbrl,
instance=modelXbrl.modelDocument.basename,
error=err,
exc_info=True)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods(
"Validate.Infoset"):
pluginXbrlMethod(
modelXbrl,
modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(
self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
self.modelXbrl.error(
"arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"
),
modelXbrl=testcase,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
file=os.path.basename(
modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator,
modelXbrl, infoset)
infoset.close()
if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.resultTableUri,
baseForElement)
if not any(
alternativeValidation(modelXbrl,
resultTableUri)
for alternativeValidation in
pluginClassMethods("Validate.TableInfoset")):
ViewFileRenderedGrid.viewRenderedGrid(
modelXbrl, resultTableUri,
diffToFile=True) # false to save infoset files
self.determineTestStatus(
modelTestcaseVariation,
modelXbrl) # include infoset errors in status
self.instValidator.close()
if modelXbrl.formulaOutputInstance and self.noErrorCodes(
modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(
modelXbrl.formulaOutputInstance,
modelTestcaseVariation.parameters)
self.determineTestStatus(
modelTestcaseVariation,
modelXbrl.formulaOutputInstance)
if self.noErrorCodes(
modelTestcaseVariation.actual
): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(
self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error(
"arelle:notLoaded",
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"
),
modelXbrl=testcase,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
file=os.path.basename(
modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(
formulaOutputInstance.facts):
formulaOutputInstance.error(
"formula:resultFactCounts",
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"
),
modelXbrl=modelXbrl,
countFacts=len(
formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts))
else:
for fact in expectedInstance.facts:
unmatchedFactsStack = []
if formulaOutputInstance.matchFact(
fact,
unmatchedFactsStack,
deemP0inf=True) is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[
-1]
else:
missingFact = fact
formulaOutputInstance.error(
"formula:expectedFactMissing",
_("Formula output missing expected fact %(fact)s"
),
modelXbrl=missingFact,
fact=missingFact.qname)
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation,
formulaOutputInstance)
formulaOutputInstance.close()
del formulaOutputInstance
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(
self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def saveTargetDocument(modelXbrl,
targetDocumentFilename,
targetDocumentSchemaRefs,
outputZip=None,
filingFiles=None,
*args,
**kwargs):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
targetDocumentFilename, modelXbrl.modelDocument.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(
_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(
targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
addChild(targetInstance.modelDocument.xmlRootElement,
roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in modelXbrl.contexts.values():
newCntx = targetInstance.createContext(
context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0],
measures[1],
id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem:
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
newFact = targetInstance.createFact(fact.qname,
attributes=attrs,
text=text,
parent=parent)
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML(
"<body>\n{0}\n</body>\n".format(xmltext)):
if elt.tag in ("a", "img") and not isHttpUrl(
attrValue) and not os.path.isabs(
attrvalue):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src"):
filingFiles.add(attrValue)
except (XMLSyntaxError, UnicodeDecodeError):
pass
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
# footnote links
footnoteIdCount = {}
modelXbrl.modelManager.showStatus(
_("Creating and validating footnotes & relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and any(
lP.modelDocument.type == Type.INLINEXBRL
for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype):
if HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ArcPrototype):
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID,
0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(
attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
newChild = addChild(newLink,
linkChild.qname,
attributes=attributes)
copyIxFootnoteHtml(
linkChild,
newChild,
targetModelDocument=targetInstance.modelDocument,
withText=True)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyIxFootnoteHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in (
"href", "src") and not isHttpUrl(
attrValue
) and not os.path.isabs(attrvalue):
filingFiles.add(attrValue)
targetInstance.saveInstance(overrideFilepath=targetUrl,
outputZip=outputZip)
if getattr(modelXbrl, "isTestcaseVariation", False):
modelXbrl.extractedInlineInstance = True # for validation comparison
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None, *args, **kwargs):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
def addLocallyReferencedFile(elt,filingFiles):
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrValue):
attrValue = attrValue.partition('#')[0] # remove anchor
if attrValue: # ignore anchor references to base document
attrValue = os.path.normpath(attrValue) # change url path separators to host separators
file = os.path.join(sourceDir,attrValue)
if modelXbrl.fileSource.isInArchive(file, checkExistence=True) or os.path.exists(file):
filingFiles.add(file)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
rootElt = modelXbrl.modelDocument.xmlRootElement
# take baseXmlLang from <html> or <base>
baseXmlLang = rootElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang")
for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/1999/xhtml}body"):
baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang") or baseXmlLang
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True,
discover=False) # don't attempt to load DTS
if baseXmlLang:
targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in sorted(modelXbrl.contexts.values(), key=lambda c: elementChildSequence(c)):
ignore = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default
xmlLang = fact.xmlLang
if xmlLang is not None and xmlLang != baseXmlLang:
attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
# if fact.isFraction, create numerator and denominator
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references so that referenced files are included in the zip.
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter():
addLocallyReferencedFile(elt, filingFiles)
except (XMLSyntaxError, UnicodeDecodeError):
pass # TODO: Why ignore UnicodeDecodeError?
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
footnoteIdCount = {}
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and
any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype):
if HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
addChild(newLink, linkChild.qname,
attributes=attributes)
elif isinstance(linkChild, ArcPrototype):
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
newChild = addChild(newLink, linkChild.qname,
attributes=attributes)
xmlLang = linkChild.xmlLang
if xmlLang is not None and xmlLang != baseXmlLang: # default
newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang)
copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyIxFootnoteHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
addLocallyReferencedFile(elt,filingFiles)
targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip)
if getattr(modelXbrl, "isTestcaseVariation", False):
modelXbrl.extractedInlineInstance = True # for validation comparison
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def createTargetInstance(modelXbrl,
targetUrl,
targetDocumentSchemaRefs,
filingFiles,
baseXmlLang=None,
defaultXmlLang=None):
def addLocallyReferencedFile(elt, filingFiles):
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src") and not isHttpUrl(
attrValue) and not os.path.isabs(attrValue):
attrValue = attrValue.partition('#')[0] # remove anchor
if attrValue: # ignore anchor references to base document
attrValue = os.path.normpath(
attrValue
) # change url path separators to host separators
file = os.path.join(sourceDir, attrValue)
if modelXbrl.fileSource.isInArchive(
file,
checkExistence=True) or os.path.exists(file):
filingFiles.add(file)
targetInstance = ModelXbrl.create(
modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True,
discover=False) # don't attempt to load DTS
ixTargetRootElt = modelXbrl.ixTargetRootElements[getattr(
modelXbrl, "ixdsTarget", None)]
langIsSet = False
# copy ix resources target root attributes
for attrName, attrValue in ixTargetRootElt.items():
if attrName != "target": # ix:references target is not mapped to xbrli:xbrl
targetInstance.modelDocument.xmlRootElement.set(
attrName, attrValue)
if attrName == "{http://www.w3.org/XML/1998/namespace}lang":
langIsSet = True
defaultXmlLang = attrValue
if attrName.startswith("{"):
ns, _sep, ln = attrName[1:].rpartition("}")
if ns:
prefix = xmlnsprefix(ixTargetRootElt, ns)
if prefix not in (None, "xml"):
setXmlns(targetInstance.modelDocument, prefix, ns)
if not langIsSet and baseXmlLang:
targetInstance.modelDocument.xmlRootElement.set(
"{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
if defaultXmlLang is None:
defaultXmlLang = baseXmlLang # allows facts/footnotes to override baseXmlLang
ValidateXbrlDimensions.loadDimensionDefaults(
targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
addChild(targetInstance.modelDocument.xmlRootElement,
roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in sorted(modelXbrl.contexts.values(),
key=lambda c: c.objectIndex
): # contexts may come from multiple IXDS files
ignore = targetInstance.createContext(
context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in sorted(modelXbrl.units.values(), key=lambda u: u.objectIndex
): # units may come from multiple IXDS files
measures = unit.measures
ignore = targetInstance.createUnit(measures[0],
measures[1],
id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
for attrName, attrValue in fact.items():
if attrName.startswith("{"):
attrs[qname(
attrName, fact.nsmap
)] = attrValue # using qname allows setting prefix in extracted instance
newFact = targetInstance.createFact(fact.qname,
attributes=attrs,
text=text,
parent=parent)
# if fact.isFraction, create numerator and denominator
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references so that referenced files are included in the zip.
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML("<body>\n{0}\n</body>\n".format(
xmltext)).iter():
addLocallyReferencedFile(elt, filingFiles)
except (XMLSyntaxError, UnicodeDecodeError):
pass # TODO: Why ignore UnicodeDecodeError?
elif fact.isTuple:
attrs = {}
if fact.id:
attrs["id"] = fact.id
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
for attrName, attrValue in fact.items():
if attrName.startswith("{"):
attrs[qname(attrName, fact.nsmap)] = attrValue
newTuple = targetInstance.createFact(fact.qname,
attributes=attrs,
parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
modelXbrl.modelManager.showStatus(
_("Creating and validating footnotes and relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
footnoteIdCount = {}
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and any(
lP.modelDocument.type == Type.INLINEXBRL
for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype):
if HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ArcPrototype):
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID,
0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(
attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
newChild = addChild(newLink,
linkChild.qname,
attributes=attributes)
xmlLang = linkChild.xmlLang
if xmlLang is not None and xmlLang != defaultXmlLang: # default
newChild.set(
"{http://www.w3.org/XML/1998/namespace}lang",
xmlLang)
copyIxFootnoteHtml(
linkChild,
newChild,
targetModelDocument=targetInstance.modelDocument,
withText=True)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyIxFootnoteHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
addLocallyReferencedFile(elt, filingFiles)
return targetInstance
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s"),
modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected)
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
ModelDocument.Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = ModelDocument.load(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
else: # not a multi-schemaRef versioning report
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource)
if modelXbrl.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelXbrl.close()
self.determineNotLoadedTestStatus(modelTestcaseVariation)
elif resultIsVersioningReport:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
modelXbrl.close()
elif testcase.type == ModelDocument.Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
parameters = modelTestcaseVariation.parameters.copy()
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName:
parameters[dtsName] = (None, inputDTS)
self.instValidator.validate(modelXbrl, parameters)
if modelTestcaseVariation.resultIsInfoset:
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource)
if infoset.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
self.determineTestStatus(modelTestcaseVariation, modelXbrl) # include infoset errors in status
self.instValidator.close()
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result not loadable"
expectedInstance.close()
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("formula:resultFactCounts",
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts))
else:
for fact in expectedInstance.facts:
if formulaOutputInstance.matchFact(fact) is None:
formulaOutputInstance.error("formula:expectedFactMissing",
_("Formula output missing expected fact %(fact)s"),
modelXbrl=fact, fact=fact.qname)
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance)
formulaOutputInstance.close()
formulaOutputInstance = None
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY):
for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri):
self.validateTestcase(doc) # testcases doc's are sorted by their uri (file names), e.g., for formula
elif hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcaseVariationsByTarget(testcase.testcaseVariations):
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s%(name)s%(target)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=(" {}".format(modelTestcaseVariation.name) if modelTestcaseVariation.name else ""),
target=(" target {}".format(modelTestcaseVariation.ixdsTarget) if modelTestcaseVariation.ixdsTarget else ""),
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings:
errorCaptureLevel = logging._checkLevel("WARNING")
else:
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
elif resultIsTaxonomyPackage:
from arelle import PackageManager, PrototypeInstanceObject
dtsName = readMeFirstUri
modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel,
ixdsTarget=modelTestcaseVariation.ixdsTarget)
else: # need own file source, may need instance discovery
filesource = FileSource.openFileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr, base=baseForElement)
if filesource and not filesource.selection and filesource.isArchive:
try:
if filesource.isTaxonomyPackage:
_rptPkgIxdsOptions = {}
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxdsOptions"):
pluginXbrlMethod(self, _rptPkgIxdsOptions)
filesource.loadTaxonomyPackageMappings()
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxds"):
filesource.select(pluginXbrlMethod(filesource, **_rptPkgIxdsOptions))
else:
from arelle.CntlrCmdLine import filesourceEntrypointFiles
entrypoints = filesourceEntrypointFiles(filesource)
if entrypoints:
# resolve an IXDS in entrypoints
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ArchiveIxds"):
pluginXbrlMethod(self, filesource,entrypoints)
filesource.select(entrypoints[0].get("file", None) )
except Exception as err:
self.modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, entry URL: %(instance)s"),
modelXbrl=self.modelXbrl, instance=readMeFirstUri, error=err)
continue # don't try to load this entry URL
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel,
ixdsTarget=modelTestcaseVariation.ixdsTarget)
modelXbrl.isTestcaseVariation = True
if modelXbrl.modelDocument is None:
modelXbrl.info("arelle:notLoaded",
_("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif resultIsVersioningReport or resultIsTaxonomyPackage:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError))
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif resultIsTaxonomyPackage:
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement)
for d in modelTestcaseVariation.dataUris.values() for uri in d
if not UrlUtil.isAbsolute(uri))
foundDataFiles = set()
variationBase = os.path.dirname(baseForElement)
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
for _inputDTS in inputDTS:
for docUrl, doc in _inputDTS.urlDocs.items():
if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET:
if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file
if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles:
docUrl = docUrl.replace("-formula.xml", ".xf")
foundDataFiles.add(docUrl)
if expectedDataFiles - foundDataFiles:
modelXbrl.info("arelle:testcaseDataNotUsed",
_("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"),
modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles)))
if foundDataFiles - expectedDataFiles:
modelXbrl.info("arelle:testcaseDataUnexpected",
_("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"),
modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles)))
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
try:
RenderingEvaluator.init(modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrlHasFormulae = modelXbrl.hasFormulae
if modelXbrlHasFormulae and self.modelXbrl.modelManager.formulaOptions.formulaAction != "none":
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError))
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
try:
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and
modelTestcaseVariation.resultXbrlInstanceUri is not None)
if compareIxResultInstance:
formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
errMsgPrefix = "ix"
else: # delete input instances before formula output comparision
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
errMsgPrefix = "formula"
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
_matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
_("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.ExpectedInstance.Loaded"):
pluginXbrlMethod(expectedInstance, formulaOutputInstance)
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts),
messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
else:
formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
def factFootnotes(fact, footnotesRelSet):
footnotes = {}
footnoteRels = footnotesRelSet.fromModelObject(fact)
if footnoteRels:
# most process rels in same order between two instances, use labels to sort
for i, footnoteRel in enumerate(sorted(footnoteRels,
key=lambda r: (r.fromLabel,r.toLabel))):
modelObject = footnoteRel.toModelObject
if isinstance(modelObject, ModelResource):
xml = collapseWhitespace(modelObject.viewText().strip())
footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))
elif isinstance(modelObject, ModelFact):
footnotes["Footnoted fact {}".format(i+1)] = \
"{} context: {} value: {}".format(
modelObject.qname,
modelObject.contextID,
collapseWhitespace(modelObject.value))
return footnotes
for expectedInstanceFact in expectedInstance.facts:
unmatchedFactsStack = []
formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False)
#formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True)
if formulaOutputFact is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = expectedInstanceFact
# is it possible to show value mismatches?
expectedFacts = formulaOutputInstance.factsByQname.get(missingFact.qname)
if len(expectedFacts) == 1:
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s, extracted value \"%(value1)s\", expected value \"%(value2)s\""),
modelXbrl=missingFact, fact=missingFact.qname, value1=missingFact.xValue, value2=next(iter(expectedFacts)).xValue,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else:
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else: # compare footnotes
expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or
set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())):
formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
_("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()),
messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
if compareIxResultInstance:
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
_statusCounts = OrderedDict((("pass",0),("fail",0)))
for tv in getattr(testcase, "testcaseVariations", ()):
_statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1
self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k))
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in modelXbrl.contexts.values():
newCntx = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem:
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)):
if elt.tag in ("a", "img") and not isHttpUrl(attrValue) and not os.path.isabs(attrvalue):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src"):
filingFiles.add(attrValue)
except (XMLSyntaxError, UnicodeDecodeError):
pass
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
# footnote links
footnoteIdCount = {}
modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and
any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype):
if HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
addChild(newLink, linkChild.qname,
attributes=attributes)
elif isinstance(linkChild, ArcPrototype):
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
newChild = addChild(newLink, linkChild.qname,
attributes=attributes)
copyIxFootnoteHtml(linkChild, newChild, withText=True)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyIxFootnoteHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrvalue):
filingFiles.add(attrValue)
targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
else: # need own file source, may need instance discovery
filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
if filesource and not filesource.selection and filesource.isArchive:
for _archiveFile in filesource.dir: # find instance document in archive
filesource.select(_archiveFile)
if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
break # use this selection
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel)
if modelXbrl.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelXbrl.close()
self.determineNotLoadedTestStatus(modelTestcaseVariation)
elif resultIsVersioningReport:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
self.modelXbrl.error("exception",
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
RenderingEvaluator.init(modelXbrl)
if modelXbrl.hasFormulae:
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
self.modelXbrl.error("exception",
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("formula:resultFactCounts",
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts))
else:
for fact in expectedInstance.facts:
unmatchedFactsStack = []
if formulaOutputInstance.matchFact(fact, unmatchedFactsStack, deemP0inf=True) is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = fact
formulaOutputInstance.error("formula:expectedFactMissing",
_("Formula output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname)
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def saveTargetDocument(modelXbrl, targetDocumentFilename,
targetDocumentSchemaRefs):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
targetDocumentFilename, modelXbrl.modelDocument.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(
_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(
targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement,
roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in modelXbrl.contexts.values():
newCntx = targetInstance.createContext(
context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0],
measures[1],
id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in modelXbrl.facts:
if fact.isItem:
attrs = {"contextRef": fact.contextID}
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
newFact = targetInstance.createFact(fact.qname,
attributes=attrs,
text=text,
parent=parent)
newFactForOldObjId[fact.objectIndex] = newFact
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
# footnote links
modelXbrl.modelManager.showStatus(
_("Creating and validating footnotes & relationships"))
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
any(lP.modelDocument.type == Type.INLINEXBRL
for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
newLink = XmlUtil.addChild(
targetInstance.modelDocument.xmlRootElement,
linkqname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
if isinstance(
linkChild, LocPrototype
) and "{http://www.w3.org/1999/xlink}href" not in linkChild.attributes:
linkChild.attributes["{http://www.w3.org/1999/xlink}href"] = \
"#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
XmlUtil.addChild(newLink,
linkChild.qname,
attributes=linkChild.attributes,
text=linkChild.textValue)
targetInstance.saveInstance(overrideFilepath=targetUrl)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in modelXbrl.contexts.values():
newCntx = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in modelXbrl.facts:
if fact.isItem:
attrs = {"contextRef": fact.contextID}
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
newFactForOldObjId[fact.objectIndex] = newFact
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
# footnote links
modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships"))
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
newLink = XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, linkqname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
if isinstance(linkChild, LocPrototype) and "{http://www.w3.org/1999/xlink}href" not in linkChild.attributes:
linkChild.attributes["{http://www.w3.org/1999/xlink}href"] = \
"#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
XmlUtil.addChild(newLink, linkChild.qname,
attributes=linkChild.attributes,
text=linkChild.textValue)
targetInstance.saveInstance(overrideFilepath=targetUrl)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def validate(val):
formulaOptions = val.modelXbrl.modelManager.formulaOptions
XPathParser.initializeParser(val)
val.modelXbrl.modelManager.showStatus(_("Compiling formulae"))
initialErrorCount = val.modelXbrl.logCountErr
# global parameter names
parameterQnames = set()
instanceQnames = set()
parameterDependencies = {}
instanceDependencies = defaultdict(set) # None-key entries are non-formula dependencies
dependencyResolvedParameters = set()
orderedParameters = []
orderedInstances = []
for paramQname, modelParameter in val.modelXbrl.qnameParameters.items():
if isinstance(modelParameter, ModelParameter):
modelParameter.compile()
parameterDependencies[paramQname] = modelParameter.variableRefs()
parameterQnames.add(paramQname)
if isinstance(modelParameter, ModelInstance):
instanceQnames.add(paramQname)
# duplicates checked on loading modelDocument
#resolve dependencies
resolvedAParameter = True
while (resolvedAParameter):
resolvedAParameter = False
for paramQname in parameterQnames:
if paramQname not in dependencyResolvedParameters and \
len(parameterDependencies[paramQname] - dependencyResolvedParameters) == 0:
dependencyResolvedParameters.add(paramQname)
orderedParameters.append(paramQname)
resolvedAParameter = True
# anything unresolved?
for paramQname in parameterQnames:
if paramQname not in dependencyResolvedParameters:
circularOrUndefDependencies = parameterDependencies[paramQname] - dependencyResolvedParameters
undefinedVars = circularOrUndefDependencies - parameterQnames
paramsCircularDep = circularOrUndefDependencies - undefinedVars
if len(undefinedVars) > 0:
val.modelXbrl.error(
_("Undefined dependencies in parameter {0}, to names {1}").format(
paramQname, ", ".join((str(v) for v in undefinedVars))),
"err", "xbrlve:unresolvedDependency")
if len(paramsCircularDep) > 0:
val.modelXbrl.error(
_("Cyclic dependencies in parameter {0}, to names {1}").format(
paramQname, ", ".join((str(d) for d in paramsCircularDep)) ),
"err", "xbrlve:parameterCyclicDependencies")
for custFnSig in val.modelXbrl.modelCustomFunctionSignatures.values():
custFnQname = custFnSig.qname
if custFnQname.namespaceURI == "XbrlConst.xfi":
val.modelXbrl.error(
_("Custom function {0} has namespace reserved for functions in the function registry {1}").format(
str(custFnQname), custFnQname.namespaceURI ),
"err", "xbrlve:noProhibitedNamespaceForCustomFunction")
# any custom function implementations?
for modelRel in val.modelXbrl.relationshipSet(XbrlConst.functionImplementation).fromModelObject(custFnSig):
custFnImpl = modelRel.toModelObject
custFnSig.customFunctionImplementation = custFnImpl
if len(custFnImpl.inputNames) != len(custFnSig.inputTypes):
val.modelXbrl.error(
_("Custom function {0} signature has {1} parameters but implementation has {2}, must be matching").format(
str(custFnQname), len(custFnSig.inputTypes), len(custFnImpl.inputNames) ),
"err", "xbrlcfie:inputMismatch")
for custFnImpl in val.modelXbrl.modelCustomFunctionImplementations:
if not val.modelXbrl.relationshipSet(XbrlConst.functionImplementation).toModelObject(custFnImpl):
val.modelXbrl.error(
_("Custom function implementation {0} has no relationship from any custom function signature").format(
custFnImpl.xlinkLabel),
"err", "xbrlcfie:missingCFIRelationship")
custFnImpl.compile()
# xpathContext is needed for filter setup for expressions such as aspect cover filter
# determine parameter values
xpathContext = XPathContext.create(val.modelXbrl)
for paramQname in orderedParameters:
if not isinstance(modelParameter, ModelInstance):
modelParameter = val.modelXbrl.qnameParameters[paramQname]
asType = modelParameter.asType
asLocalName = asType.localName if asType else "string"
try:
if val.parameters and paramQname in val.parameters:
paramDataType, paramValue = val.parameters[paramQname]
typeLocalName = paramDataType.localName if paramDataType else "string"
value = FunctionXs.call(xpathContext, None, typeLocalName, [paramValue])
result = FunctionXs.call(xpathContext, None, asLocalName, [value])
if formulaOptions.traceParameterInputValue:
val.modelXbrl.error( _("Parameter {0} input {1}").format( paramQname, result),
"info", "formula:trace")
else:
result = modelParameter.evaluate(xpathContext, asType)
if formulaOptions.traceParameterExpressionResult:
val.modelXbrl.error( _("Parameter {0} result {1}").format( paramQname, result),
"info", "formula:trace")
xpathContext.inScopeVars[paramQname] = result # make visible to subsequent parameter expression
except XPathContext.XPathException as err:
val.modelXbrl.error( _("Parameter \n{0} \nException: \n{1}").format( paramQname, err.message),
"err", "xbrlve:parameterTypeMismatch" if err.code == "err:FORG0001" else err.code)
produceOutputXbrlInstance = False
instanceProducingVariableSets = defaultdict(list)
for modelVariableSet in val.modelXbrl.modelVariableSets:
varSetInstanceDependencies = set()
if isinstance(modelVariableSet, ModelFormula):
instanceQname = None
for modelRel in val.modelXbrl.relationshipSet(XbrlConst.formulaInstance).fromModelObject(modelVariableSet):
instance = modelRel.toModelObject
if isinstance(instance, ModelInstance):
if instanceQname is None:
instanceQname = instance.qname
else:
val.modelXbrl.error(
_("Multiple output instances for formula {0}, to names {1}, {2}").format(
modelVariableSet.xlinkLabel, instanceQname, instance.qname ),
"info", "arelle:multipleOutputInstances")
if instanceQname is None:
instanceQname = XbrlConst.qnStandardOutputInstance
instanceQnames.add(instanceQname)
modelVariableSet.outputInstanceQname = instanceQname
if val.validateSBRNL:
val.modelXbrl.error(
_("Formula linkbase {0} formula:formula {1} is not allowed").format(
os.path.basename(modelVariableSet.modelDocument.uri), modelVariableSet.xlinkLabel),
"err", "SBR.NL.2.3.9.03")
else:
instanceQname = None
modelVariableSet.countSatisfied = 0
modelVariableSet.countNotSatisfied = 0
checkValidationMessages(val, modelVariableSet)
instanceProducingVariableSets[instanceQname].append(modelVariableSet)
modelVariableSet.outputInstanceQname = instanceQname
if modelVariableSet.aspectModel not in ("non-dimensional", "dimensional"):
val.modelXbrl.error(
_("Variable set {0}, aspect model {1} not recognized").format(
modelVariableSet.xlinkLabel, modelVariableSet.aspectModel),
"err", "xbrlve:unknownAspectModel")
modelVariableSet.compile()
modelVariableSet.hasConsistencyAssertion = False
#determine dependencies within variable sets
nameVariables = {}
qnameRels = {}
definedNamesSet = set()
for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSet).fromModelObject(modelVariableSet):
varqname = modelRel.variableQname
if varqname:
qnameRels[varqname] = modelRel
toVariable = modelRel.toModelObject
if varqname not in definedNamesSet:
definedNamesSet.add(varqname)
if varqname not in nameVariables:
nameVariables[varqname] = toVariable
elif nameVariables[varqname] != toVariable:
val.modelXbrl.error(
_("Multiple variables named {1} in variable set {0}").format(
modelVariableSet.xlinkLabel, varqname ),
"err", "xbrlve:duplicateVariableNames")
fromInstanceQnames = None
for instRel in val.modelXbrl.relationshipSet(XbrlConst.instanceVariable).toModelObject(toVariable):
fromInstance = instRel.fromModelObject
if isinstance(fromInstance, ModelInstance):
fromInstanceQname = fromInstance.qname
varSetInstanceDependencies.add(fromInstanceQname)
instanceDependencies[instanceQname].add(fromInstanceQname)
if fromInstanceQnames is None: fromInstanceQnames = set()
fromInstanceQnames.add(fromInstanceQname)
if fromInstanceQnames is None:
varSetInstanceDependencies.add(XbrlConst.qnStandardInputInstance)
if instanceQname: instanceDependencies[instanceQname].add(XbrlConst.qnStandardInputInstance)
toVariable.fromInstanceQnames = fromInstanceQnames
else:
val.modelXbrl.error(
_("Variables name {1} cannot be determined on arc from {0}").format(
modelVariableSet.xlinkLabel, modelRel.variablename ),
"err", "xbrlve:variableNameResolutionFailure")
definedNamesSet |= parameterQnames
variableDependencies = {}
for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSet).fromModelObject(modelVariableSet):
variable = modelRel.toModelObject
if isinstance(variable, (ModelParameter,ModelVariable)): # ignore anything not parameter or variable
varqname = modelRel.variableQname
depVars = variable.variableRefs()
variableDependencies[varqname] = depVars
if len(depVars) > 0 and formulaOptions.traceVariablesDependencies:
val.modelXbrl.error(_("Variable set {0}, variable {1}, dependences {2}").format(
modelVariableSet.xlinkLabel, varqname, depVars),
"info", "formula:trace")
definedNamesSet.add(varqname)
# check for fallback value variable references
if isinstance(variable, ModelFactVariable):
for depVar in XPathParser.variableReferencesSet(variable.fallbackValueProg, variable.element):
if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable):
val.modelXbrl.error(_("Variable set {0} fallbackValue '{1}' cannot refer to variable {2}").format(
modelVariableSet.xlinkLabel, variable.fallbackValue, depVar),
"err", "xbrlve:factVariableReferenceNotAllowed")
# check for covering aspect not in variable set aspect model
checkFilterAspectModel(val, modelVariableSet, variable.filterRelationships, xpathContext)
orderedNameSet = set()
orderedNameList = []
orderedAVariable = True
while (orderedAVariable):
orderedAVariable = False
for varqname, depVars in variableDependencies.items():
if varqname not in orderedNameSet and len(depVars - parameterQnames - orderedNameSet) == 0:
orderedNameList.append(varqname)
orderedNameSet.add(varqname)
orderedAVariable = True
if varqname in instanceQnames:
varSetInstanceDependencies.add(varqname)
instanceDependencies[instanceQname].add(varqname)
elif isinstance(nameVariables.get(varqname), ModelInstance):
instqname = nameVariables[varqname].qname
varSetInstanceDependencies.add(instqname)
instanceDependencies[instanceQname].add(instqname)
# anything unresolved?
for varqname, depVars in variableDependencies.items():
if varqname not in orderedNameSet:
circularOrUndefVars = depVars - parameterQnames - orderedNameSet
undefinedVars = circularOrUndefVars - definedNamesSet
varsCircularDep = circularOrUndefVars - undefinedVars
if len(undefinedVars) > 0:
val.modelXbrl.error(
_("Undefined variable dependencies in variable st {0}, from variable {1} to {2}").format(
modelVariableSet.xlinkLabel, varqname, undefinedVars),
"err", "xbrlve:unresolvedDependency")
if len(varsCircularDep) > 0:
val.modelXbrl.error(
_("Cyclic dependencies in variable set {0}, from variable {1} to {2}").format(
modelVariableSet.xlinkLabel, varqname, varsCircularDep ),
"err", "xbrlve:cyclicDependencies")
# check unresolved variable set dependencies
for varSetDepVarQname in modelVariableSet.variableRefs():
if varSetDepVarQname not in orderedNameSet and varSetDepVarQname not in parameterQnames:
val.modelXbrl.error(
_("Undefined variable dependency in variable set {0}, {1}").format(
modelVariableSet.xlinkLabel, varSetDepVarQname),
"err", "xbrlve:unresolvedDependency")
if varSetDepVarQname in instanceQnames:
varSetInstanceDependencies.add(varSetDepVarQname)
instanceDependencies[instanceQname].add(varSetDepVarQname)
elif isinstance(nameVariables.get(varSetDepVarQname), ModelInstance):
instqname = nameVariables[varSetDepVarQname].qname
varSetInstanceDependencies.add(instqname)
instanceDependencies[instanceQname].add(instqname)
if formulaOptions.traceVariablesOrder:
val.modelXbrl.error(_("Variable set {0}, variables order: {1}").format(
modelVariableSet.xlinkLabel, orderedNameList), "info", "formula:trace")
if (formulaOptions.traceVariablesDependencies and len(varSetInstanceDependencies) > 0 and
varSetInstanceDependencies != {XbrlConst.qnStandardInputInstance}):
val.modelXbrl.error(_("Variable set {0}, instance dependences {1}").format(
modelVariableSet.xlinkLabel, varSetInstanceDependencies),
"info", "formula:trace")
modelVariableSet.orderedVariableRelationships = []
for varqname in orderedNameList:
if varqname in qnameRels:
modelVariableSet.orderedVariableRelationships.append(qnameRels[varqname])
# check existence assertion variable dependencies
if isinstance(modelVariableSet, ModelExistenceAssertion):
for depVar in modelVariableSet.variableRefs():
if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable):
val.modelXbrl.error(_("Existence Assertion {0}, cannot refer to variable {1}").format(
modelVariableSet.xlinkLabel, depVar),
"err", "xbrleae:variableReferenceNotAllowed")
# check messages variable dependencies
checkValidationMessageVariables(val, modelVariableSet, qnameRels)
# check preconditions
modelVariableSet.preconditions = []
for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSetPrecondition).fromModelObject(modelVariableSet):
precondition = modelRel.toModelObject
if isinstance(precondition, ModelPrecondition):
modelVariableSet.preconditions.append(precondition)
# check for variable sets referencing fact or general variables
for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSetFilter).fromModelObject(modelVariableSet):
varSetFilter = modelRel.toModelObject
if modelRel.isCovered:
val.modelXbrl.error(_("Variable set {0}, filter {1}, cannot be covered").format(
modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel),
"wrn", "arelle:variableSetFilterCovered")
modelRel._isCovered = False # block group filter from being able to covere
for depVar in varSetFilter.variableRefs():
if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable):
val.modelXbrl.error(_("Variable set {0}, filter {1}, cannot refer to variable {2}").format(
modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel, depVar),
"err", "xbrlve:factVariableReferenceNotAllowed")
# check aspects of formula
if isinstance(modelVariableSet, ModelFormula):
checkFormulaRules(val, modelVariableSet, nameVariables)
# determine instance dependency order
orderedInstancesSet = set()
stdInpInst = {XbrlConst.qnStandardInputInstance}
orderedInstancesList = []
orderedAnInstance = True
while (orderedAnInstance):
orderedAnInstance = False
for instqname, depInsts in instanceDependencies.items():
if instqname and instqname not in orderedInstancesSet and len(depInsts - stdInpInst - orderedInstancesSet) == 0:
orderedInstancesList.append(instqname)
orderedInstancesSet.add(instqname)
orderedAnInstance = True
orderedInstancesList.append(None) # assertions come after all formulas that produce outputs
# anything unresolved?
for instqname, depInsts in instanceDependencies.items():
if instqname not in orderedInstancesSet:
# can also be satisfied from an input DTS
missingDependentInstances = depInsts - stdInpInst
if val.parameters: missingDependentInstances -= val.parameters.keys()
if instqname:
if missingDependentInstances:
val.modelXbrl.error(
_("Cyclic dependencies of instance {0} produced by a formula, with variables consuming instances {1}").format(
instqname, missingDependentInstances ),
"err", "xbrlvarinste:instanceVariableRecursionCycle")
elif instqname == XbrlConst.qnStandardOutputInstance:
orderedInstancesSet.add(instqname)
orderedInstancesList.append(instqname) # standard output formula, all input dependencies in parameters
''' future check? if instance has no external input or producing formula
else:
val.modelXbrl.error(
_("Unresolved dependencies of an assertion's variables on instances {0}").format(
depInsts - stdInpInst ),
"err", "xbrlvarinste:instanceVariableRecursionCycle")
'''
if formulaOptions.traceVariablesOrder and len(orderedInstancesList) > 1:
val.modelXbrl.error(_("Variable instances processing order: {0}").format(
orderedInstancesList), "info", "formula:trace")
# linked consistency assertions
for modelRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).modelRelationships:
if modelRel.fromModelObject and modelRel.toModelObject and isinstance(modelRel.toModelObject,ModelFormula):
consisAsser = modelRel.fromModelObject
consisAsser.countSatisfied = 0
consisAsser.countNotSatisfied = 0
if consisAsser.hasProportionalAcceptanceRadius and consisAsser.hasAbsoluteAcceptanceRadius:
val.modelXbrl.error( _("Consistency assertion {0} has both absolute and proportional acceptance radii").format(
consisAsser.xlinkLabel),
"err", "xbrlcae:acceptanceRadiusConflict")
consisAsser.orderedVariableRelationships = []
for consisParamRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionParameter).fromModelObject(consisAsser):
if isinstance(consisParamRel.toModelObject, ModelVariable):
val.modelXbrl.error( _("Consistency assertion {0} has relationship to a {1} {2}").format(
consisAsser.xlinkLabel, consisParamRel.toModelObject.element.localName, consisParamRel.toModelObject.xlinkLabel),
"err", "xbrlcae:variablesNotAllowed")
else:
consisAsser.orderedVariableRelationships.append(consisParamRel)
consisAsser.compile()
modelRel.toModelObject.hasConsistencyAssertion = True
if initialErrorCount < val.modelXbrl.logCountErr:
return # don't try to execute
# formula output instances
if instanceQnames:
schemaRefs = [val.modelXbrl.modelDocument.relativeUri(referencedDoc.uri)
for referencedDoc in val.modelXbrl.modelDocument.referencesDocument.keys()
if referencedDoc.type == ModelDocument.Type.SCHEMA]
outputXbrlInstance = None
for instanceQname in instanceQnames:
if instanceQname == XbrlConst.qnStandardInputInstance:
continue # always present the standard way
if val.parameters and instanceQname in val.parameters:
namedInstance = val.parameters[instanceQname][1]
else: # empty intermediate instance
uri = val.modelXbrl.modelDocument.filepath[:-4] + "-output-XBRL-instance"
if instanceQname != XbrlConst.qnStandardOutputInstance:
uri = uri + "-" + instanceQname.localName
uri = uri + ".xml"
namedInstance = ModelXbrl.create(val.modelXbrl.modelManager,
newDocumentType=ModelDocument.Type.INSTANCE,
url=uri,
schemaRefs=schemaRefs,
isEntry=True)
xpathContext.inScopeVars[instanceQname] = namedInstance
if instanceQname == XbrlConst.qnStandardOutputInstance:
outputXbrlInstance = namedInstance
# evaluate consistency assertions
# evaluate variable sets not in consistency assertions
for instanceQname in orderedInstancesList:
for modelVariableSet in instanceProducingVariableSets[instanceQname]:
# produce variable evaluations
from arelle.FormulaEvaluator import evaluate
try:
evaluate(xpathContext, modelVariableSet)
except XPathContext.XPathException as err:
val.modelXbrl.error( _("Variable set \n{0} \nException: \n{1}").format( modelVariableSet, err.message),
"err", err.code)
# log assertion result counts
asserTests = {}
for exisValAsser in val.modelXbrl.modelVariableSets:
if isinstance(exisValAsser, ModelVariableSetAssertion):
asserTests[exisValAsser.id] = (exisValAsser.countSatisfied, exisValAsser.countNotSatisfied)
if formulaOptions.traceAssertionResultCounts:
val.modelXbrl.error( _("{0} Assertion {1} evaluations : {2} satisfied, {3} not satisfied").format(
"Existence" if isinstance(exisValAsser, ModelExistenceAssertion) else "Value",
exisValAsser.id, exisValAsser.countSatisfied, exisValAsser.countNotSatisfied),
"info", "formula:trace")
for modelRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).modelRelationships:
if modelRel.fromModelObject and modelRel.toModelObject and isinstance(modelRel.toModelObject,ModelFormula):
consisAsser = modelRel.fromModelObject
asserTests[consisAsser.id] = (consisAsser.countSatisfied, consisAsser.countNotSatisfied)
if formulaOptions.traceAssertionResultCounts:
val.modelXbrl.error( _("Consistency Assertion {0} evaluations : {1} satisfied, {2} not satisfied").format(
consisAsser.id, consisAsser.countSatisfied, consisAsser.countNotSatisfied),
"info", "formula:trace")
if asserTests:
val.modelXbrl.error( _("Assertion results {0}").format(asserTests),
"asrtNoLog", asserTests)
# display output instance
if outputXbrlInstance:
if val.modelXbrl.formulaOutputInstance:
# close prior instance, usually closed by caller to validate as it may affect UI on different thread
val.modelXbrl.formulaOutputInstance.close()
val.modelXbrl.formulaOutputInstance = outputXbrlInstance
