def validateRssFeed(self):
self.modelXbrl.info("info", "RSS Feed", modelDocument=self.modelXbrl)
from arelle.FileSource import openFileSource
for rssItem in self.modelXbrl.modelDocument.rssItems:
self.modelXbrl.info("info", _("RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"),
modelObject=rssItem, accessionNumber=rssItem.accessionNumber, formType=rssItem.formType, companyName=rssItem.companyName, period=rssItem.period)
modelXbrl = None
try:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
openFileSource(rssItem.zippedUrl, self.modelXbrl.modelManager.cntlr),
_("validating"))
self.instValidator.validate(modelXbrl, self.modelXbrl.modelManager.formulaOptions.typedParameters())
self.instValidator.close()
rssItem.setResults(modelXbrl)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, rssItem.objectId())
for pluginXbrlMethod in pluginClassMethods("Validate.RssItem"):
pluginXbrlMethod(self, modelXbrl, rssItem)
modelXbrl.close()
except Exception as err:
self.modelXbrl.error("exception",
_("RSS item validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=(self.modelXbrl, modelXbrl),
instance=rssItem.zippedUrl, error=err,
exc_info=True)
try:
self.instValidator.close()
if modelXbrl is not None:
modelXbrl.close()
except Exception as err:
pass
del modelXbrl # completely dereference
def load(self, filesource, nextaction=None):
"""Load an entry point modelDocument object(s), which in turn load documents they discover
(for the case of instance, taxonomies, and versioning reports), but defer loading instances
for test case and RSS feeds.
The modelXbrl that is loaded is 'stacked', by this class, so that any modelXbrl operations such as validate,
and close, operate on the most recently loaded modelXbrl, and compareDTSes operates on the two
most recently loaded modelXbrl's.
:param filesource: may be a FileSource object, with the entry point selected, or string file name (or web URL).
:type filesource: FileSource or str
:param nextAction: status line text string, if any, to show upon completion
:type nextAction: str
"""
try:
if filesource.url.startswith("urn:uuid:"): # request for an open modelXbrl
for modelXbrl in self.loadedModelXbrls:
if not modelXbrl.isClosed and modelXbrl.uuid == filesource.url:
return modelXbrl
raise IOError(_("Open file handle is not open: {0}").format(filesource.url))
except AttributeError:
pass # filesource may be a string, which has no url attribute
self.filesource = filesource
modelXbrl = None # loaded modelXbrl
for customLoader in pluginClassMethods("ModelManager.Load"):
modelXbrl = customLoader(self, filesource)
if modelXbrl is not None:
break # custom loader did the loading
if modelXbrl is None: # use default xbrl loader
modelXbrl = ModelXbrl.load(self, filesource, nextaction)
self.modelXbrl = modelXbrl
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def runFromXml(self):
testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\Arelle\roland test cases\1000-Concepts\index.xml"
filesource = FileSource.FileSource(testGenFileName)
startedAt = time.time()
LogHandler(self) # start logger
modelTestcases = self.modelManager.load(filesource, _("views loading"))
self.addToLog(_("[info] loaded in {0:.2} secs").format(time.time() - startedAt))
if modelTestcases.modelDocument.type == ModelDocument.Type.TESTCASESINDEX:
for testcasesElement in modelTestcases.modelDocument.iter(tag="testcases"):
rootAttr = testcasesElement.get("root")
title = testcasesElement.get("title")
self.addToLog(_("[info] testcases {0}").format(title))
if rootAttr is not None:
base = os.path.join(os.path.dirname(modelTestcases.modelDocument.filepath),rootAttr) + os.sep
else:
base = self.filepath
for testcaseElement in testcasesElement.iterchildren(tag="testcase"):
uriFrom = testcaseElement.get("uriFrom")
uriTo = testcaseElement.get("uriTo")
modelDTSfrom = modelDTSto = None
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom is not None and uriTo is not None:
modelDTSfrom = ModelXbrl.load(modelTestcases.modelManager,
uriFrom,
_("loading from DTS"),
base=base)
modelDTSto = ModelXbrl.load(modelTestcases.modelManager,
uriTo,
_("loading to DTS"),
base=base)
if modelDTSfrom is not None and modelDTSto is not None:
# generate differences report
reportName = os.path.basename(uriFrom).replace("from.xsd","report.xml")
reportFile = os.path.dirname(uriFrom) + "\\report\\" + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile,
base)
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
ModelVersReport.ModelVersReport(modelTestcases).diffDTSes(
reportFullPath,
modelDTSfrom, modelDTSto)
def load(self, filesource, nextaction=None):
try:
if filesource.url.startswith("urn:uuid:"): # request for an open modelXbrl
for modelXbrl in self.loadedModelXbrls:
if not modelXbrl.isClosed and modelXbrl.uuid == filesource.url:
return modelXbrl
raise IOError(_("Open file handle is not open: {0}".format(filesource.url)))
except AttributeError:
pass # filesource may be a string, which has no url attribute
self.filesource = filesource
self.modelXbrl = ModelXbrl.load(self, filesource, nextaction)
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def __init__(self, cntlr, isCmdLine=False):
self.cntlr = cntlr
# setup tester
xml = "<rootElement/>"
self.modelXbrl = ModelXbrl.create(cntlr.modelManager, ModelDocument.Type.UnknownNonXML, initialXml=xml, isEntry=True)
self.validator = ValidateXbrl.ValidateXbrl(self.modelXbrl)
self.validator.validate(self.modelXbrl) # required to set up
cntlr.showStatus(_("Initializing Formula Grammar"))
XPathParser.initializeParser(cntlr.modelManager)
cntlr.showStatus(None)
self.trRegs = sorted(ixtNamespaces.keys())
self.trPrefixNSs = dict((qn.prefix, qn.namespaceURI)
for qn in self.modelXbrl.modelManager.customTransforms.keys())
self.trRegs.extend(sorted(self.trPrefixNSs.keys()))
self.trPrefixNSs.update(ixtNamespaces)
def saveTargetDocument(self):
targetUrl = self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(self.targetDocumentPreferredFilename, self.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
self.modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(self.modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=self.targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
for context in self.modelXbrl.contexts.values():
newCntx = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in self.modelXbrl.units.values():
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
self.modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
for fact in self.modelXbrl.facts:
if fact.isItem:
attrs = [("contextRef", fact.contextID)]
if fact.isNumeric:
attrs.append(("unitRef", fact.unitID))
if fact.get("decimals"):
attrs.append(("decimals", fact.get("decimals")))
if fact.get("precision"):
attrs.append(("precision", fact.get("precision")))
if fact.isNil:
attrs.append(("{http://www.w3.org/2001/XMLSchema-instance}nil","true"))
text = None
else:
text = fact.xValue if fact.xValid else fact.elementText
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text)
targetInstance.saveInstance(overrideFilepath=targetUrl)
self.modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def deprecatedConceptDatesFile(modelManager, abbrNs, latestTaxonomyDoc):
if latestTaxonomyDoc is None:
return None
if not abbrNs: # none for an unexpected namespace pattern
return None
cntlr = modelManager.cntlr
_fileName = resourcesFilePath(modelManager, abbrNs.partition("/")[0] + "-deprecated-concepts.json")
_deprecatedLabelRole = latestTaxonomyDoc["deprecatedLabelRole"]
_deprecatedDateMatchPattern = latestTaxonomyDoc["deprecationDatePattern"]
if os.path.exists(_fileName):
return _fileName
# load labels and store file name
modelManager.addToLog(_("loading {} deprecated concepts into {}").format(abbrNs, _fileName), messageCode="info")
deprecatedConceptDates = {}
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = modelManager.validateDisclosureSystem
modelManager.validateDisclosureSystem = False
from arelle import ModelXbrl
deprecationsInstance = ModelXbrl.load(modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
# load from zip (especially after caching) is incredibly faster
openFileSource(latestTaxonomyDoc["deprecatedLabels"], cntlr),
_("built deprecations table in cache"))
modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
if deprecationsInstance is None:
modelManager.addToLog(
_("%(name)s documentation not loaded"),
messageCode="arelle:notLoaded", messageArgs={"modelXbrl": val, "name":_abbrNs})
else:
# load deprecations
for labelRel in deprecationsInstance.relationshipSet(XbrlConst.conceptLabel).modelRelationships:
modelLabel = labelRel.toModelObject
conceptName = labelRel.fromModelObject.name
if modelLabel.role == _deprecatedLabelRole:
match = _deprecatedDateMatchPattern.match(modelLabel.text)
if match is not None:
date = match.group(1)
if date:
deprecatedConceptDates[conceptName] = date
jsonStr = _STR_UNICODE(json.dumps(deprecatedConceptDates, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, _fileName, jsonStr) # 2.7 gets unicode this way
deprecationsInstance.close()
del deprecationsInstance # dereference closed modelXbrl
def validateRssFeed(self):
self.modelXbrl.info("info", "RSS Feed", modelDocument=self.modelXbrl)
from arelle.FileSource import openFileSource
reloadCache = getattr(self.modelXbrl, "reloadCache", False)
for rssItem in self.modelXbrl.modelDocument.rssItems:
if getattr(rssItem, "skipRssItem", False):
self.modelXbrl.info("info", _("skipping RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"),
modelObject=rssItem, accessionNumber=rssItem.accessionNumber, formType=rssItem.formType, companyName=rssItem.companyName, period=rssItem.period)
continue
self.modelXbrl.info("info", _("RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"),
modelObject=rssItem, accessionNumber=rssItem.accessionNumber, formType=rssItem.formType, companyName=rssItem.companyName, period=rssItem.period)
modelXbrl = None
try:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
openFileSource(rssItem.zippedUrl, self.modelXbrl.modelManager.cntlr, reloadCache=reloadCache),
_("validating"), rssItem=rssItem)
for pluginXbrlMethod in pluginClassMethods("RssItem.Xbrl.Loaded"):
pluginXbrlMethod(modelXbrl, {}, rssItem)
if getattr(rssItem, "doNotProcessRSSitem", False) or modelXbrl.modelDocument is None:
modelXbrl.close()
continue # skip entry based on processing criteria
self.instValidator.validate(modelXbrl, self.modelXbrl.modelManager.formulaOptions.typedParameters(self.modelXbrl.prefixedNamespaces))
self.instValidator.close()
rssItem.setResults(modelXbrl)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, rssItem.objectId())
for pluginXbrlMethod in pluginClassMethods("Validate.RssItem"):
pluginXbrlMethod(self, modelXbrl, rssItem)
modelXbrl.close()
except Exception as err:
self.modelXbrl.error("exception:" + type(err).__name__,
_("RSS item validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=(self.modelXbrl, modelXbrl),
instance=rssItem.zippedUrl, error=err,
exc_info=True)
try:
self.instValidator.close()
if modelXbrl is not None:
modelXbrl.close()
except Exception as err:
pass
del modelXbrl # completely dereference
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s"),
modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected)
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
ModelDocument.Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = ModelDocument.load(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
else: # not a multi-schemaRef versioning report
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource)
if modelXbrl.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelXbrl.close()
self.determineNotLoadedTestStatus(modelTestcaseVariation)
elif resultIsVersioningReport:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
modelXbrl.close()
elif testcase.type == ModelDocument.Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
parameters = modelTestcaseVariation.parameters.copy()
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName:
parameters[dtsName] = (None, inputDTS)
self.instValidator.validate(modelXbrl, parameters)
if modelTestcaseVariation.resultIsInfoset:
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource)
if infoset.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
self.determineTestStatus(modelTestcaseVariation, modelXbrl) # include infoset errors in status
self.instValidator.close()
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result not loadable"
expectedInstance.close()
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("formula:resultFactCounts",
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts))
else:
for fact in expectedInstance.facts:
if formulaOutputInstance.matchFact(fact) is None:
formulaOutputInstance.error("formula:expectedFactMissing",
_("Formula output missing expected fact %(fact)s"),
modelXbrl=fact, fact=fact.qname)
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance)
formulaOutputInstance.close()
formulaOutputInstance = None
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def setup(val):
val.linroleDefinitionIsDisclosure = re.compile(r"-\s+Disclosure\s+-\s",
re.IGNORECASE)
val.linkroleDefinitionStatementSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*", # no restriction to type of statement
re.IGNORECASE)
val.ugtNamespace = None
cntlr = val.modelXbrl.modelManager.cntlr
# load deprecated concepts for filed year of us-gaap
for ugt in ugtDocs:
ugtNamespace = ugt["namespace"]
if ugtNamespace in val.modelXbrl.namespaceDocs and len(val.modelXbrl.namespaceDocs[ugtNamespace]) > 0:
val.ugtNamespace = ugtNamespace
usgaapDoc = val.modelXbrl.namespaceDocs[ugtNamespace][0]
deprecationsJsonFile = usgaapDoc.filepathdir + os.sep + "deprecated-concepts.json"
file = None
try:
file = openFileStream(cntlr, deprecationsJsonFile, 'rt', encoding='utf-8')
val.usgaapDeprecations = json.load(file)
file.close()
except Exception:
if file:
file.close()
val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} deprecated concepts into cache").format(ugt["year"]))
startedAt = time.time()
ugtDocLB = ugt["docLB"]
val.usgaapDeprecations = {}
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem
val.modelXbrl.modelManager.validateDisclosureSystem = False
deprecationsInstance = ModelXbrl.load(val.modelXbrl.modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
# load from zip (especially after caching) is incredibly faster
openFileSource(ugtDocLB, cntlr),
_("built deprecations table in cache"))
val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
if deprecationsInstance is None:
val.modelXbrl.error("arelle:notLoaded",
_("US-GAAP documentation not loaded: %(file)s"),
modelXbrl=val, file=os.path.basename(ugtDocLB))
else:
# load deprecations
for labelRel in deprecationsInstance.relationshipSet(XbrlConst.conceptLabel).modelRelationships:
modelDocumentation = labelRel.toModelObject
conceptName = labelRel.fromModelObject.name
if modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedLabel':
val.usgaapDeprecations[conceptName] = (val.usgaapDeprecations.get(conceptName, ('',''))[0], modelDocumentation.text)
elif modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedDateLabel':
val.usgaapDeprecations[conceptName] = (modelDocumentation.text, val.usgaapDeprecations.get(conceptName, ('',''))[1])
jsonStr = _STR_UNICODE(json.dumps(val.usgaapDeprecations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, deprecationsJsonFile, jsonStr) # 2.7 gets unicode this way
deprecationsInstance.close()
del deprecationsInstance # dereference closed modelXbrl
val.modelXbrl.profileStat(_("build us-gaap deprecated concepts cache"), time.time() - startedAt)
ugtCalcsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-calculations.json"
ugtDefaultDimensionsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-default-dimensions.json"
file = None
try:
file = openFileStream(cntlr, ugtCalcsJsonFile, 'rt', encoding='utf-8')
val.usgaapCalculations = json.load(file)
file.close()
file = openFileStream(cntlr, ugtDefaultDimensionsJsonFile, 'rt', encoding='utf-8')
val.usgaapDefaultDimensions = json.load(file)
file.close()
except Exception:
if file:
file.close()
val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} calculations and default dimensions into cache").format(ugt["year"]))
startedAt = time.time()
ugtEntryXsd = ugt["entryXsd"]
val.usgaapCalculations = {}
val.usgaapDefaultDimensions = {}
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem
val.modelXbrl.modelManager.validateDisclosureSystem = False
calculationsInstance = ModelXbrl.load(val.modelXbrl.modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/entire/us-gaap-entryPoint-std-2012-01-31.xsd",
# load from zip (especially after caching) is incredibly faster
openFileSource(ugtEntryXsd, cntlr),
_("built us-gaap calculations cache"))
val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
if calculationsInstance is None:
val.modelXbrl.error("arelle:notLoaded",
_("US-GAAP calculations not loaded: %(file)s"),
modelXbrl=val, file=os.path.basename(ugtEntryXsd))
else:
# load calculations
for ELR in calculationsInstance.relationshipSet(XbrlConst.summationItem).linkRoleUris:
elrRelSet = calculationsInstance.relationshipSet(XbrlConst.summationItem, ELR)
definition = ""
for roleType in calculationsInstance.roleTypes.get(ELR,()):
definition = roleType.definition
break
isStatementSheet = bool(val.linkroleDefinitionStatementSheet.match(definition))
elrUgtCalcs = {"#roots": [c.name for c in elrRelSet.rootConcepts],
"#definition": definition,
"#isStatementSheet": isStatementSheet}
for relFrom, rels in elrRelSet.fromModelObjects().items():
elrUgtCalcs[relFrom.name] = [rel.toModelObject.name for rel in rels]
val.usgaapCalculations[ELR] = elrUgtCalcs
jsonStr = _STR_UNICODE(json.dumps(val.usgaapCalculations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, ugtCalcsJsonFile, jsonStr) # 2.7 gets unicode this way
# load default dimensions
for defaultDimRel in calculationsInstance.relationshipSet(XbrlConst.dimensionDefault).modelRelationships:
if defaultDimRel.fromModelObject is not None and defaultDimRel.toModelObject is not None:
val.usgaapDefaultDimensions[defaultDimRel.fromModelObject.name] = defaultDimRel.toModelObject.name
jsonStr = _STR_UNICODE(json.dumps(val.usgaapDefaultDimensions, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, ugtDefaultDimensionsJsonFile, jsonStr) # 2.7 gets unicode this way
calculationsInstance.close()
del calculationsInstance # dereference closed modelXbrl
val.modelXbrl.profileStat(_("build us-gaap calculations and default dimensions cache"), time.time() - startedAt)
break
val.deprecatedFactConcepts = defaultdict(list)
val.deprecatedDimensions = defaultdict(list)
val.deprecatedMembers = defaultdict(list)
def watchCycle(self):
while not self.stopRequested:
rssWatchOptions = self.rssModelXbrl.modelManager.rssWatchOptions
# check rss expiration
rssHeaders = self.cntlr.webCache.getheaders(self.rssModelXbrl.modelManager.rssWatchOptions.get("feedSourceUri"))
expires = parseRfcDatetime(rssHeaders.get("expires"))
reloadNow = True # texpires and expires > datetime.datetime.now()
# reload rss feed
self.rssModelXbrl.reload('checking RSS items', reloadCache=reloadNow)
if self.stopRequested: break
# setup validator
postLoadActions = []
if rssWatchOptions.get("validateDisclosureSystemRules"):
self.instValidator = ValidateFiling.ValidateFiling(self.rssModelXbrl)
postLoadActions.append(_("validating"))
elif rssWatchOptions.get("validateXbrlRules") or rssWatchOptions.get("validateFormulaAssertions"):
self.instValidator = ValidateXbrl.ValidateXbrl(self.rssModelXbrl)
postLoadActions.append(_("validating"))
if (rssWatchOptions.get("validateFormulaAssertions")):
postLoadActions.append(_("running formulas"))
else:
self.instValidator = None
matchTextExpr = rssWatchOptions.get("matchTextExpr")
if matchTextExpr:
matchPattern = re.compile(matchTextExpr)
postLoadActions.append(_("matching text"))
else:
matchPattern= None
postLoadAction = ', '.join(postLoadActions)
# anything to check new filings for
if (rssWatchOptions.get("validateDisclosureSystemRules") or
rssWatchOptions.get("validateXbrlRules") or
rssWatchOptions.get("validateCalcLinkbase") or
rssWatchOptions.get("validateFormulaAssertions") or
rssWatchOptions.get("alertMatchedFactText") or
any(pluginXbrlMethod(rssWatchOptions)
for pluginXbrlMethod in pluginClassMethods("RssWatch.HasWatchAction"))
):
# form keys in ascending order of pubdate
pubDateRssItems = []
for rssItem in self.rssModelXbrl.modelDocument.rssItems:
pubDateRssItems.append((rssItem.pubDate,rssItem.objectId()))
for pubDate, rssItemObjectId in sorted(pubDateRssItems):
rssItem = self.rssModelXbrl.modelObject(rssItemObjectId)
# update ui thread via modelManager (running in background here)
self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId())
if self.stopRequested:
break
latestPubDate = XmlUtil.datetimeValue(rssWatchOptions.get("latestPubDate"))
if (latestPubDate and
rssItem.pubDate < latestPubDate):
continue
try:
# try zipped URL if possible, else expanded instance document
modelXbrl = ModelXbrl.load(self.rssModelXbrl.modelManager,
openFileSource(rssItem.zippedUrl, self.cntlr),
postLoadAction)
if self.stopRequested:
modelXbrl.close()
break
emailAlert = False
if modelXbrl.modelDocument is None:
modelXbrl.error("arelle.rssWatch",
_("RSS item %(company)s %(form)s document not loaded: %(date)s"),
modelXbrl=modelXbrl, company=rssItem.companyName,
form=rssItem.formType, date=rssItem.filingDate)
rssItem.status = "not loadable"
else:
# validate schema, linkbase, or instance
if self.stopRequested:
modelXbrl.close()
break
if self.instValidator:
self.instValidator.validate(modelXbrl)
if modelXbrl.errors and rssWatchOptions.get("alertValiditionError"):
emailAlert = True
for pluginXbrlMethod in pluginClassMethods("RssWatch.DoWatchAction"):
pluginXbrlMethod(modelXbrl, rssWatchOptions, rssItem)
# check match expression
if matchPattern:
for fact in modelXbrl.factsInInstance:
v = fact.value
if v is not None:
m = matchPattern.search(v)
if m:
fr, to = m.span()
msg = _("Fact Variable {0}\n context {1}\n matched text: {2}").format(
fact.qname, fact.contextID, v[max(0,fr-20):to+20])
modelXbrl.info("arelle.rssInfo",
msg,
modelXbrl=modelXbrl) # msg as code passes it through to the status
if rssWatchOptions.get("alertMatchedFactText"):
emailAlert = True
if (rssWatchOptions.get("formulaFileUri") and rssWatchOptions.get("validateFormulaAssertions") and
self.instValidator):
# attach formulas
ModelDocument.load(modelXbrl, rssWatchOptions["formulaFileUri"])
ValidateFormula.validate(self.instValidator)
rssItem.setResults(modelXbrl)
modelXbrl.close()
del modelXbrl # completely dereference
self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId())
if rssItem.assertionUnsuccessful and rssWatchOptions.get("alertAssertionUnsuccessful"):
emailAlert = True
msg = _("Filing CIK {0}\n "
"company {1}\n "
"published {2}\n "
"form type {3}\n "
"filing date {4}\n "
"period {5}\n "
"year end {6}\n "
"results: {7}").format(
rssItem.cikNumber,
rssItem.companyName,
rssItem.pubDate,
rssItem.formType,
rssItem.filingDate,
rssItem.period,
rssItem.fiscalYearEnd,
rssItem.status)
self.rssModelXbrl.info("arelle:rssWatch", msg, modelXbrl=self.rssModelXbrl)
emailAddress = rssWatchOptions.get("emailAddress")
if emailAlert and emailAddress:
self.rssModelXbrl.modelManager.showStatus(_("sending e-mail alert"))
import smtplib
from email.mime.text import MIMEText
emailMsg = MIMEText(msg)
emailMsg["Subject"] = _("Arelle RSS Watch alert on {0}").format(rssItem.companyName)
emailMsg["From"] = emailAddress
emailMsg["To"] = emailAddress
smtp = smtplib.SMTP()
smtp.sendmail(emailAddress, [emailAddress], emailMsg.as_string())
smtp.quit()
self.rssModelXbrl.modelManager.showStatus(_("RSS item {0}, {1} completed, status {2}").format(rssItem.companyName, rssItem.formType, rssItem.status), 3500)
self.rssModelXbrl.modelManager.cntlr.rssWatchUpdateOption(rssItem.pubDate.strftime('%Y-%m-%dT%H:%M:%S'))
except Exception as err:
self.rssModelXbrl.error("arelle.rssError",
_("RSS item %(company)s, %(form)s, %(date)s, exception: %(error)s"),
modelXbrl=self.rssModelXbrl, company=rssItem.companyName,
form=rssItem.formType, date=rssItem.filingDate, error=err,
exc_info=True)
if self.stopRequested: break
if self.stopRequested:
self.cntlr.showStatus(_("RSS watch, stop requested"), 10000)
else:
import time
time.sleep(600)
self.thread = None # close thread
self.stopRequested = False
def createTargetInstance(modelXbrl, targetUrl, targetDocumentSchemaRefs, filingFiles, baseXmlLang=None, defaultXmlLang=None):
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True,
discover=False) # don't attempt to load DTS
if baseXmlLang:
targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
if defaultXmlLang is None:
defaultXmlLang = baseXmlLang # allows facts/footnotes to override baseXmlLang
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in sorted(modelXbrl.contexts.values(), key=lambda c: c.objectIndex): # contexts may come from multiple IXDS files
ignore = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in sorted(modelXbrl.units.values(), key=lambda u: u.objectIndex): # units may come from multiple IXDS files
measures = unit.measures
ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default
xmlLang = fact.xmlLang
if xmlLang is not None and xmlLang != defaultXmlLang:
attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
# if fact.isFraction, create numerator and denominator
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references so that referenced files are included in the zip.
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter():
addLocallyReferencedFile(elt, filingFiles)
except (XMLSyntaxError, UnicodeDecodeError):
pass # TODO: Why ignore UnicodeDecodeError?
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
footnoteIdCount = {}
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and
any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype):
if HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
addChild(newLink, linkChild.qname,
attributes=attributes)
elif isinstance(linkChild, ArcPrototype):
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
newChild = addChild(newLink, linkChild.qname,
attributes=attributes)
xmlLang = linkChild.xmlLang
if xmlLang is not None and xmlLang != defaultXmlLang: # default
newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang)
copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyIxFootnoteHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
addLocallyReferencedFile(elt,filingFiles)
return targetInstance
def runFromExcel(self, options):
#testGenFileName = options.excelfilename
testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls"
testGenDir = os.path.dirname(testGenFileName)
schemaDir = os.path.dirname(testGenDir) + os.sep + "schema"
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
LogHandler(self) # start logger
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt'
modelTestcases = ModelXbrl.create(self.modelManager, url=testGenFileName, isEntry=True)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow))
# start index file
indexFiles = [testGenDir + os.sep + 'creation-testcases-index.xml',
testGenDir + os.sep + 'consumption-testcases-index.xml']
indexDocs = []
testcasesElements = []
for purpose in ("Creation","Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 {0} Tests" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today)
)
doc = etree.parse(file)
file.close()
indexDocs.append(doc)
testcasesElements.append(doc.getroot())
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
try:
row = testcaseIndexSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY or # must have directory
row[1].ctype == xlrd.XL_CELL_EMPTY or # from
row[2].ctype == xlrd.XL_CELL_EMPTY): # to
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
overrideReport = row[3].value
description = row[4].value
if description is None or len(description) == 0:
continue # test not ready to run
assignment = row[5].value
expectedEvents = row[6].value # comma space separated if multiple
note = row[7].value
useCase = row[8].value
base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and assignment.lower() not in ("n.a.", "error") and expectedEvents != "N.A.":
modelDTSfrom = modelDTSto = None
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base)
else:
modelDTS = ModelXbrl.create(self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts",
base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS, uri.strip(), base=base)
if doc is not None:
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom is not None and modelDTSto is not None:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace("from.xsd","report.xml")
reportFile = reportDir + "out" + os.sep + reportName
#reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile,
base)
testcasesDir = os.path.dirname(os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml",
testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"]
for i,testcaseFile in enumerate(testcaseFiles):
etree.SubElement(testcasesElements[i], "testcase",
attrib={"uri":
testcaseFile[len(testGenDir)+1:].replace("\\","/")} )
# start testcase file
testcaseDocs = []
testcaseElements = []
testcaseNumber = testcaseName[0:4]
if testcaseNumber.isnumeric():
testcaseNumberElement = "<number>{0}</number>".format(testcaseNumber)
testcaseName = testcaseName[5:]
else:
testcaseNumberElement = ""
testDirSegments = testDir.split('/')
if len(testDirSegments) >= 2 and '-' in testDirSegments[1]:
testedModule = testDirSegments[1][testDirSegments[1].index('-') + 1:]
else:
testedModule = ''
for purpose in ("Creation","Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'{0}'
'<name>{1}</name>'
# '<description>{0}</description>'
'<reference>'
'{2}'
'{3}'
'</reference>'
'</testcase>'.format(testcaseNumberElement,
testcaseName,
'<name>{0}</name>'.format(testedModule) if testedModule else '',
'<id>{0}</id>'.format(useCase) if useCase else '')
)
doc = etree.parse(file)
file.close()
testcaseDocs.append(doc)
testcaseElements.append(doc.getroot())
priorTestcasesDir = testcasesDir
variationSeq = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(modelTestcases)
modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto,
assignment=assignment,
schemaDir=schemaDir)
# check for expected elements
if expectedEvents:
for expectedEvent in expectedEvents.split(","):
if expectedEvent not in ("No change", "N.A."):
prefix, sep, localName = expectedEvent.partition(':')
if sep and len(modelVersReport.xmlDocument.findall(
'//{{{0}}}{1}'.format(
XbrlConst.verPrefixNS.get(prefix),
localName))) == 0:
modelTestcases.warning("warning",
"Generated test case %(reportName)s missing expected event %(event)s",
reportName=reportName,
event=expectedEvent)
modelVersReport.close()
uriFromParts = uriFrom.split('_')
if len(uriFromParts) >= 2:
variationId = uriFromParts[1]
else:
variationId = "_{0:02n}".format(variationSeq)
for i,testcaseElt in enumerate(testcaseElements):
variationElement = etree.SubElement(testcaseElt, "{http://xbrl.org/2008/conformance}variation",
attrib={"id": variationId})
nameElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
nameElement.text = description
''' (removed per RH 2011/10/04
if note:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
paramElement.text = "Note: " + note
if useCase:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference")
paramElement.set("specification", "versioning-requirements")
paramElement.set("useCase", useCase)
'''
dataElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}data")
if i == 0: # result is report
if expectedEvents:
paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter",
attrib={"name":"expectedEvent",
"value":expectedEvents.replace(',',' ')},
nsmap={"conf":"http://xbrl.org/2008/conformance",
None:""})
if assignment:
paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter",
attrib={"name":"assignment",
"value":assignment},
nsmap={"conf":"http://xbrl.org/2008/conformance",
None:""})
for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")):
for schemaURI in schemaURIs.split(","):
schemaElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}schema")
schemaElement.set("dts",dtsAttr)
if i == 0:
schemaElement.set("readMeFirst","true")
schemaElement.text=os.path.basename(schemaURI.strip())
resultElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}result")
reportElement = etree.SubElement(resultElement if i == 0 else dataElement,
"{http://xbrl.org/2008/conformance}versioningReport")
if i == 1:
reportElement.set("readMeFirst","true")
reportElement.text = "report/" + reportName
variationSeq += 1
except Exception as err:
modelTestcases.error("exception",
_("Exception: %(error)s, Excel row: %(excelRow)s"),
error=err,
excelRow=iRow,
exc_info=True)
# add tests-error-code index files to consumption
for testcaseFile in self.testcaseFiles(testGenDir + os.sep + "tests-error-code"):
etree.SubElement(testcasesElements[1], "testcase",
attrib={"uri":
testcaseFile[len(testGenDir)+1:].replace("\\","/")} )
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i,indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def validate(val):
formulaOptions = val.modelXbrl.modelManager.formulaOptions
XPathParser.initializeParser(val)
val.modelXbrl.modelManager.showStatus(_("Compiling formulae"))
initialErrorCount = val.modelXbrl.logCountErr
# global parameter names
parameterQnames = set()
instanceQnames = set()
parameterDependencies = {}
instanceDependencies = defaultdict(set) # None-key entries are non-formula dependencies
dependencyResolvedParameters = set()
orderedParameters = []
orderedInstances = []
for paramQname, modelParameter in val.modelXbrl.qnameParameters.items():
if isinstance(modelParameter, ModelParameter):
modelParameter.compile()
parameterDependencies[paramQname] = modelParameter.variableRefs()
parameterQnames.add(paramQname)
if isinstance(modelParameter, ModelInstance):
instanceQnames.add(paramQname)
# duplicates checked on loading modelDocument
#resolve dependencies
resolvedAParameter = True
while (resolvedAParameter):
resolvedAParameter = False
for paramQname in parameterQnames:
if paramQname not in dependencyResolvedParameters and \
len(parameterDependencies[paramQname] - dependencyResolvedParameters) == 0:
dependencyResolvedParameters.add(paramQname)
orderedParameters.append(paramQname)
resolvedAParameter = True
# anything unresolved?
for paramQname in parameterQnames:
if paramQname not in dependencyResolvedParameters:
circularOrUndefDependencies = parameterDependencies[paramQname] - dependencyResolvedParameters
undefinedVars = circularOrUndefDependencies - parameterQnames
paramsCircularDep = circularOrUndefDependencies - undefinedVars
if len(undefinedVars) > 0:
val.modelXbrl.error(
_("Undefined dependencies in parameter {0}, to names {1}").format(
paramQname, ", ".join((str(v) for v in undefinedVars))),
"err", "xbrlve:unresolvedDependency")
if len(paramsCircularDep) > 0:
val.modelXbrl.error(
_("Cyclic dependencies in parameter {0}, to names {1}").format(
paramQname, ", ".join((str(d) for d in paramsCircularDep)) ),
"err", "xbrlve:parameterCyclicDependencies")
for custFnSig in val.modelXbrl.modelCustomFunctionSignatures.values():
custFnQname = custFnSig.qname
if custFnQname.namespaceURI == "XbrlConst.xfi":
val.modelXbrl.error(
_("Custom function {0} has namespace reserved for functions in the function registry {1}").format(
str(custFnQname), custFnQname.namespaceURI ),
"err", "xbrlve:noProhibitedNamespaceForCustomFunction")
# any custom function implementations?
for modelRel in val.modelXbrl.relationshipSet(XbrlConst.functionImplementation).fromModelObject(custFnSig):
custFnImpl = modelRel.toModelObject
custFnSig.customFunctionImplementation = custFnImpl
if len(custFnImpl.inputNames) != len(custFnSig.inputTypes):
val.modelXbrl.error(
_("Custom function {0} signature has {1} parameters but implementation has {2}, must be matching").format(
str(custFnQname), len(custFnSig.inputTypes), len(custFnImpl.inputNames) ),
"err", "xbrlcfie:inputMismatch")
for custFnImpl in val.modelXbrl.modelCustomFunctionImplementations:
if not val.modelXbrl.relationshipSet(XbrlConst.functionImplementation).toModelObject(custFnImpl):
val.modelXbrl.error(
_("Custom function implementation {0} has no relationship from any custom function signature").format(
custFnImpl.xlinkLabel),
"err", "xbrlcfie:missingCFIRelationship")
custFnImpl.compile()
# xpathContext is needed for filter setup for expressions such as aspect cover filter
# determine parameter values
xpathContext = XPathContext.create(val.modelXbrl)
for paramQname in orderedParameters:
if not isinstance(modelParameter, ModelInstance):
modelParameter = val.modelXbrl.qnameParameters[paramQname]
asType = modelParameter.asType
asLocalName = asType.localName if asType else "string"
try:
if val.parameters and paramQname in val.parameters:
paramDataType, paramValue = val.parameters[paramQname]
typeLocalName = paramDataType.localName if paramDataType else "string"
value = FunctionXs.call(xpathContext, None, typeLocalName, [paramValue])
result = FunctionXs.call(xpathContext, None, asLocalName, [value])
if formulaOptions.traceParameterInputValue:
val.modelXbrl.error( _("Parameter {0} input {1}").format( paramQname, result),
"info", "formula:trace")
else:
result = modelParameter.evaluate(xpathContext, asType)
if formulaOptions.traceParameterExpressionResult:
val.modelXbrl.error( _("Parameter {0} result {1}").format( paramQname, result),
"info", "formula:trace")
xpathContext.inScopeVars[paramQname] = result # make visible to subsequent parameter expression
except XPathContext.XPathException as err:
val.modelXbrl.error( _("Parameter \n{0} \nException: \n{1}").format( paramQname, err.message),
"err", "xbrlve:parameterTypeMismatch" if err.code == "err:FORG0001" else err.code)
produceOutputXbrlInstance = False
instanceProducingVariableSets = defaultdict(list)
for modelVariableSet in val.modelXbrl.modelVariableSets:
varSetInstanceDependencies = set()
if isinstance(modelVariableSet, ModelFormula):
instanceQname = None
for modelRel in val.modelXbrl.relationshipSet(XbrlConst.formulaInstance).fromModelObject(modelVariableSet):
instance = modelRel.toModelObject
if isinstance(instance, ModelInstance):
if instanceQname is None:
instanceQname = instance.qname
else:
val.modelXbrl.error(
_("Multiple output instances for formula {0}, to names {1}, {2}").format(
modelVariableSet.xlinkLabel, instanceQname, instance.qname ),
"info", "arelle:multipleOutputInstances")
if instanceQname is None:
instanceQname = XbrlConst.qnStandardOutputInstance
instanceQnames.add(instanceQname)
modelVariableSet.outputInstanceQname = instanceQname
if val.validateSBRNL:
val.modelXbrl.error(
_("Formula linkbase {0} formula:formula {1} is not allowed").format(
os.path.basename(modelVariableSet.modelDocument.uri), modelVariableSet.xlinkLabel),
"err", "SBR.NL.2.3.9.03")
else:
instanceQname = None
modelVariableSet.countSatisfied = 0
modelVariableSet.countNotSatisfied = 0
checkValidationMessages(val, modelVariableSet)
instanceProducingVariableSets[instanceQname].append(modelVariableSet)
modelVariableSet.outputInstanceQname = instanceQname
if modelVariableSet.aspectModel not in ("non-dimensional", "dimensional"):
val.modelXbrl.error(
_("Variable set {0}, aspect model {1} not recognized").format(
modelVariableSet.xlinkLabel, modelVariableSet.aspectModel),
"err", "xbrlve:unknownAspectModel")
modelVariableSet.compile()
modelVariableSet.hasConsistencyAssertion = False
#determine dependencies within variable sets
nameVariables = {}
qnameRels = {}
definedNamesSet = set()
for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSet).fromModelObject(modelVariableSet):
varqname = modelRel.variableQname
if varqname:
qnameRels[varqname] = modelRel
toVariable = modelRel.toModelObject
if varqname not in definedNamesSet:
definedNamesSet.add(varqname)
if varqname not in nameVariables:
nameVariables[varqname] = toVariable
elif nameVariables[varqname] != toVariable:
val.modelXbrl.error(
_("Multiple variables named {1} in variable set {0}").format(
modelVariableSet.xlinkLabel, varqname ),
"err", "xbrlve:duplicateVariableNames")
fromInstanceQnames = None
for instRel in val.modelXbrl.relationshipSet(XbrlConst.instanceVariable).toModelObject(toVariable):
fromInstance = instRel.fromModelObject
if isinstance(fromInstance, ModelInstance):
fromInstanceQname = fromInstance.qname
varSetInstanceDependencies.add(fromInstanceQname)
instanceDependencies[instanceQname].add(fromInstanceQname)
if fromInstanceQnames is None: fromInstanceQnames = set()
fromInstanceQnames.add(fromInstanceQname)
if fromInstanceQnames is None:
varSetInstanceDependencies.add(XbrlConst.qnStandardInputInstance)
if instanceQname: instanceDependencies[instanceQname].add(XbrlConst.qnStandardInputInstance)
toVariable.fromInstanceQnames = fromInstanceQnames
else:
val.modelXbrl.error(
_("Variables name {1} cannot be determined on arc from {0}").format(
modelVariableSet.xlinkLabel, modelRel.variablename ),
"err", "xbrlve:variableNameResolutionFailure")
definedNamesSet |= parameterQnames
variableDependencies = {}
for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSet).fromModelObject(modelVariableSet):
variable = modelRel.toModelObject
if isinstance(variable, (ModelParameter,ModelVariable)): # ignore anything not parameter or variable
varqname = modelRel.variableQname
depVars = variable.variableRefs()
variableDependencies[varqname] = depVars
if len(depVars) > 0 and formulaOptions.traceVariablesDependencies:
val.modelXbrl.error(_("Variable set {0}, variable {1}, dependences {2}").format(
modelVariableSet.xlinkLabel, varqname, depVars),
"info", "formula:trace")
definedNamesSet.add(varqname)
# check for fallback value variable references
if isinstance(variable, ModelFactVariable):
for depVar in XPathParser.variableReferencesSet(variable.fallbackValueProg, variable.element):
if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable):
val.modelXbrl.error(_("Variable set {0} fallbackValue '{1}' cannot refer to variable {2}").format(
modelVariableSet.xlinkLabel, variable.fallbackValue, depVar),
"err", "xbrlve:factVariableReferenceNotAllowed")
# check for covering aspect not in variable set aspect model
checkFilterAspectModel(val, modelVariableSet, variable.filterRelationships, xpathContext)
orderedNameSet = set()
orderedNameList = []
orderedAVariable = True
while (orderedAVariable):
orderedAVariable = False
for varqname, depVars in variableDependencies.items():
if varqname not in orderedNameSet and len(depVars - parameterQnames - orderedNameSet) == 0:
orderedNameList.append(varqname)
orderedNameSet.add(varqname)
orderedAVariable = True
if varqname in instanceQnames:
varSetInstanceDependencies.add(varqname)
instanceDependencies[instanceQname].add(varqname)
elif isinstance(nameVariables.get(varqname), ModelInstance):
instqname = nameVariables[varqname].qname
varSetInstanceDependencies.add(instqname)
instanceDependencies[instanceQname].add(instqname)
# anything unresolved?
for varqname, depVars in variableDependencies.items():
if varqname not in orderedNameSet:
circularOrUndefVars = depVars - parameterQnames - orderedNameSet
undefinedVars = circularOrUndefVars - definedNamesSet
varsCircularDep = circularOrUndefVars - undefinedVars
if len(undefinedVars) > 0:
val.modelXbrl.error(
_("Undefined variable dependencies in variable st {0}, from variable {1} to {2}").format(
modelVariableSet.xlinkLabel, varqname, undefinedVars),
"err", "xbrlve:unresolvedDependency")
if len(varsCircularDep) > 0:
val.modelXbrl.error(
_("Cyclic dependencies in variable set {0}, from variable {1} to {2}").format(
modelVariableSet.xlinkLabel, varqname, varsCircularDep ),
"err", "xbrlve:cyclicDependencies")
# check unresolved variable set dependencies
for varSetDepVarQname in modelVariableSet.variableRefs():
if varSetDepVarQname not in orderedNameSet and varSetDepVarQname not in parameterQnames:
val.modelXbrl.error(
_("Undefined variable dependency in variable set {0}, {1}").format(
modelVariableSet.xlinkLabel, varSetDepVarQname),
"err", "xbrlve:unresolvedDependency")
if varSetDepVarQname in instanceQnames:
varSetInstanceDependencies.add(varSetDepVarQname)
instanceDependencies[instanceQname].add(varSetDepVarQname)
elif isinstance(nameVariables.get(varSetDepVarQname), ModelInstance):
instqname = nameVariables[varSetDepVarQname].qname
varSetInstanceDependencies.add(instqname)
instanceDependencies[instanceQname].add(instqname)
if formulaOptions.traceVariablesOrder:
val.modelXbrl.error(_("Variable set {0}, variables order: {1}").format(
modelVariableSet.xlinkLabel, orderedNameList), "info", "formula:trace")
if (formulaOptions.traceVariablesDependencies and len(varSetInstanceDependencies) > 0 and
varSetInstanceDependencies != {XbrlConst.qnStandardInputInstance}):
val.modelXbrl.error(_("Variable set {0}, instance dependences {1}").format(
modelVariableSet.xlinkLabel, varSetInstanceDependencies),
"info", "formula:trace")
modelVariableSet.orderedVariableRelationships = []
for varqname in orderedNameList:
if varqname in qnameRels:
modelVariableSet.orderedVariableRelationships.append(qnameRels[varqname])
# check existence assertion variable dependencies
if isinstance(modelVariableSet, ModelExistenceAssertion):
for depVar in modelVariableSet.variableRefs():
if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable):
val.modelXbrl.error(_("Existence Assertion {0}, cannot refer to variable {1}").format(
modelVariableSet.xlinkLabel, depVar),
"err", "xbrleae:variableReferenceNotAllowed")
# check messages variable dependencies
checkValidationMessageVariables(val, modelVariableSet, qnameRels)
# check preconditions
modelVariableSet.preconditions = []
for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSetPrecondition).fromModelObject(modelVariableSet):
precondition = modelRel.toModelObject
if isinstance(precondition, ModelPrecondition):
modelVariableSet.preconditions.append(precondition)
# check for variable sets referencing fact or general variables
for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSetFilter).fromModelObject(modelVariableSet):
varSetFilter = modelRel.toModelObject
if modelRel.isCovered:
val.modelXbrl.error(_("Variable set {0}, filter {1}, cannot be covered").format(
modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel),
"wrn", "arelle:variableSetFilterCovered")
modelRel._isCovered = False # block group filter from being able to covere
for depVar in varSetFilter.variableRefs():
if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable):
val.modelXbrl.error(_("Variable set {0}, filter {1}, cannot refer to variable {2}").format(
modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel, depVar),
"err", "xbrlve:factVariableReferenceNotAllowed")
# check aspects of formula
if isinstance(modelVariableSet, ModelFormula):
checkFormulaRules(val, modelVariableSet, nameVariables)
# determine instance dependency order
orderedInstancesSet = set()
stdInpInst = {XbrlConst.qnStandardInputInstance}
orderedInstancesList = []
orderedAnInstance = True
while (orderedAnInstance):
orderedAnInstance = False
for instqname, depInsts in instanceDependencies.items():
if instqname and instqname not in orderedInstancesSet and len(depInsts - stdInpInst - orderedInstancesSet) == 0:
orderedInstancesList.append(instqname)
orderedInstancesSet.add(instqname)
orderedAnInstance = True
orderedInstancesList.append(None) # assertions come after all formulas that produce outputs
# anything unresolved?
for instqname, depInsts in instanceDependencies.items():
if instqname not in orderedInstancesSet:
# can also be satisfied from an input DTS
missingDependentInstances = depInsts - stdInpInst
if val.parameters: missingDependentInstances -= val.parameters.keys()
if instqname:
if missingDependentInstances:
val.modelXbrl.error(
_("Cyclic dependencies of instance {0} produced by a formula, with variables consuming instances {1}").format(
instqname, missingDependentInstances ),
"err", "xbrlvarinste:instanceVariableRecursionCycle")
elif instqname == XbrlConst.qnStandardOutputInstance:
orderedInstancesSet.add(instqname)
orderedInstancesList.append(instqname) # standard output formula, all input dependencies in parameters
''' future check? if instance has no external input or producing formula
else:
val.modelXbrl.error(
_("Unresolved dependencies of an assertion's variables on instances {0}").format(
depInsts - stdInpInst ),
"err", "xbrlvarinste:instanceVariableRecursionCycle")
'''
if formulaOptions.traceVariablesOrder and len(orderedInstancesList) > 1:
val.modelXbrl.error(_("Variable instances processing order: {0}").format(
orderedInstancesList), "info", "formula:trace")
# linked consistency assertions
for modelRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).modelRelationships:
if modelRel.fromModelObject and modelRel.toModelObject and isinstance(modelRel.toModelObject,ModelFormula):
consisAsser = modelRel.fromModelObject
consisAsser.countSatisfied = 0
consisAsser.countNotSatisfied = 0
if consisAsser.hasProportionalAcceptanceRadius and consisAsser.hasAbsoluteAcceptanceRadius:
val.modelXbrl.error( _("Consistency assertion {0} has both absolute and proportional acceptance radii").format(
consisAsser.xlinkLabel),
"err", "xbrlcae:acceptanceRadiusConflict")
consisAsser.orderedVariableRelationships = []
for consisParamRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionParameter).fromModelObject(consisAsser):
if isinstance(consisParamRel.toModelObject, ModelVariable):
val.modelXbrl.error( _("Consistency assertion {0} has relationship to a {1} {2}").format(
consisAsser.xlinkLabel, consisParamRel.toModelObject.element.localName, consisParamRel.toModelObject.xlinkLabel),
"err", "xbrlcae:variablesNotAllowed")
else:
consisAsser.orderedVariableRelationships.append(consisParamRel)
consisAsser.compile()
modelRel.toModelObject.hasConsistencyAssertion = True
if initialErrorCount < val.modelXbrl.logCountErr:
return # don't try to execute
# formula output instances
if instanceQnames:
schemaRefs = [val.modelXbrl.modelDocument.relativeUri(referencedDoc.uri)
for referencedDoc in val.modelXbrl.modelDocument.referencesDocument.keys()
if referencedDoc.type == ModelDocument.Type.SCHEMA]
outputXbrlInstance = None
for instanceQname in instanceQnames:
if instanceQname == XbrlConst.qnStandardInputInstance:
continue # always present the standard way
if val.parameters and instanceQname in val.parameters:
namedInstance = val.parameters[instanceQname][1]
else: # empty intermediate instance
uri = val.modelXbrl.modelDocument.filepath[:-4] + "-output-XBRL-instance"
if instanceQname != XbrlConst.qnStandardOutputInstance:
uri = uri + "-" + instanceQname.localName
uri = uri + ".xml"
namedInstance = ModelXbrl.create(val.modelXbrl.modelManager,
newDocumentType=ModelDocument.Type.INSTANCE,
url=uri,
schemaRefs=schemaRefs,
isEntry=True)
xpathContext.inScopeVars[instanceQname] = namedInstance
if instanceQname == XbrlConst.qnStandardOutputInstance:
outputXbrlInstance = namedInstance
# evaluate consistency assertions
# evaluate variable sets not in consistency assertions
for instanceQname in orderedInstancesList:
for modelVariableSet in instanceProducingVariableSets[instanceQname]:
# produce variable evaluations
from arelle.FormulaEvaluator import evaluate
try:
evaluate(xpathContext, modelVariableSet)
except XPathContext.XPathException as err:
val.modelXbrl.error( _("Variable set \n{0} \nException: \n{1}").format( modelVariableSet, err.message),
"err", err.code)
# log assertion result counts
asserTests = {}
for exisValAsser in val.modelXbrl.modelVariableSets:
if isinstance(exisValAsser, ModelVariableSetAssertion):
asserTests[exisValAsser.id] = (exisValAsser.countSatisfied, exisValAsser.countNotSatisfied)
if formulaOptions.traceAssertionResultCounts:
val.modelXbrl.error( _("{0} Assertion {1} evaluations : {2} satisfied, {3} not satisfied").format(
"Existence" if isinstance(exisValAsser, ModelExistenceAssertion) else "Value",
exisValAsser.id, exisValAsser.countSatisfied, exisValAsser.countNotSatisfied),
"info", "formula:trace")
for modelRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).modelRelationships:
if modelRel.fromModelObject and modelRel.toModelObject and isinstance(modelRel.toModelObject,ModelFormula):
consisAsser = modelRel.fromModelObject
asserTests[consisAsser.id] = (consisAsser.countSatisfied, consisAsser.countNotSatisfied)
if formulaOptions.traceAssertionResultCounts:
val.modelXbrl.error( _("Consistency Assertion {0} evaluations : {1} satisfied, {2} not satisfied").format(
consisAsser.id, consisAsser.countSatisfied, consisAsser.countNotSatisfied),
"info", "formula:trace")
if asserTests:
val.modelXbrl.error( _("Assertion results {0}").format(asserTests),
"asrtNoLog", asserTests)
# display output instance
if outputXbrlInstance:
if val.modelXbrl.formulaOutputInstance:
# close prior instance, usually closed by caller to validate as it may affect UI on different thread
val.modelXbrl.formulaOutputInstance.close()
val.modelXbrl.formulaOutputInstance = outputXbrlInstance
def validateRssFeed(self):
self.modelXbrl.info("info", "RSS Feed", modelDocument=self.modelXbrl)
from arelle.FileSource import openFileSource
reloadCache = getattr(self.modelXbrl, "reloadCache", False)
for rssItem in self.modelXbrl.modelDocument.rssItems:
if getattr(rssItem, "skipRssItem", False):
self.modelXbrl.info(
"info",
_("skipping RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"
),
modelObject=rssItem,
accessionNumber=rssItem.accessionNumber,
formType=rssItem.formType,
companyName=rssItem.companyName,
period=rssItem.period)
continue
self.modelXbrl.info(
"info",
_("RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"
),
modelObject=rssItem,
accessionNumber=rssItem.accessionNumber,
formType=rssItem.formType,
companyName=rssItem.companyName,
period=rssItem.period)
modelXbrl = None
try:
modelXbrl = ModelXbrl.load(
self.modelXbrl.modelManager,
openFileSource(rssItem.zippedUrl,
self.modelXbrl.modelManager.cntlr,
reloadCache=reloadCache),
_("validating"),
rssItem=rssItem)
for pluginXbrlMethod in pluginClassMethods(
"RssItem.Xbrl.Loaded"):
pluginXbrlMethod(modelXbrl, {}, rssItem)
if getattr(rssItem, "doNotProcessRSSitem",
False) or modelXbrl.modelDocument is None:
modelXbrl.close()
continue # skip entry based on processing criteria
self.instValidator.validate(
modelXbrl,
self.modelXbrl.modelManager.formulaOptions.typedParameters(
))
self.instValidator.close()
rssItem.setResults(modelXbrl)
self.modelXbrl.modelManager.viewModelObject(
self.modelXbrl, rssItem.objectId())
for pluginXbrlMethod in pluginClassMethods("Validate.RssItem"):
pluginXbrlMethod(self, modelXbrl, rssItem)
modelXbrl.close()
except Exception as err:
self.modelXbrl.error(
"exception",
_("RSS item validation exception: %(error)s, instance: %(instance)s"
),
modelXbrl=(self.modelXbrl, modelXbrl),
instance=rssItem.zippedUrl,
error=err,
exc_info=True)
try:
self.instValidator.close()
if modelXbrl is not None:
modelXbrl.close()
except Exception as err:
pass
del modelXbrl # completely dereference
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None, *args, **kwargs):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
def addLocallyReferencedFile(elt,filingFiles):
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrValue):
attrValue = attrValue.partition('#')[0] # remove anchor
if attrValue: # ignore anchor references to base document
attrValue = os.path.normpath(attrValue) # change url path separators to host separators
file = os.path.join(sourceDir,attrValue)
if modelXbrl.fileSource.isInArchive(file, checkExistence=True) or os.path.exists(file):
filingFiles.add(file)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
rootElt = modelXbrl.modelDocument.xmlRootElement
# take baseXmlLang from <html> or <base>
baseXmlLang = rootElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang")
for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/1999/xhtml}body"):
baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang") or baseXmlLang
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True,
discover=False) # don't attempt to load DTS
if baseXmlLang:
targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in sorted(modelXbrl.contexts.values(), key=lambda c: elementChildSequence(c)):
ignore = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default
xmlLang = fact.xmlLang
if xmlLang is not None and xmlLang != baseXmlLang:
attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
# if fact.isFraction, create numerator and denominator
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references so that referenced files are included in the zip.
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter():
addLocallyReferencedFile(elt, filingFiles)
except (XMLSyntaxError, UnicodeDecodeError):
pass # TODO: Why ignore UnicodeDecodeError?
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
footnoteIdCount = {}
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and
any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype):
if HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
addChild(newLink, linkChild.qname,
attributes=attributes)
elif isinstance(linkChild, ArcPrototype):
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
newChild = addChild(newLink, linkChild.qname,
attributes=attributes)
xmlLang = linkChild.xmlLang
if xmlLang is not None and xmlLang != baseXmlLang: # default
newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang)
copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyIxFootnoteHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
addLocallyReferencedFile(elt,filingFiles)
targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip)
if getattr(modelXbrl, "isTestcaseVariation", False):
modelXbrl.extractedInlineInstance = True # for validation comparison
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
else: # need own file source, may need instance discovery
filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
if filesource and not filesource.selection and filesource.isArchive:
for _archiveFile in filesource.dir: # find instance document in archive
filesource.select(_archiveFile)
if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
break # use this selection
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel)
if modelXbrl.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelXbrl.close()
self.determineNotLoadedTestStatus(modelTestcaseVariation)
elif resultIsVersioningReport:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
self.modelXbrl.error("exception",
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
RenderingEvaluator.init(modelXbrl)
if modelXbrl.hasFormulae:
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
self.modelXbrl.error("exception",
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("formula:resultFactCounts",
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts))
else:
for fact in expectedInstance.facts:
unmatchedFactsStack = []
if formulaOutputInstance.matchFact(fact, unmatchedFactsStack, deemP0inf=True) is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = fact
formulaOutputInstance.error("formula:expectedFactMissing",
_("Formula output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname)
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in modelXbrl.contexts.values():
newCntx = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in modelXbrl.facts:
if fact.isItem:
attrs = {"contextRef": fact.contextID}
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
newFactForOldObjId[fact.objectIndex] = newFact
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
# footnote links
modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships"))
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
newLink = XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, linkqname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
if isinstance(linkChild, LocPrototype) and "{http://www.w3.org/1999/xlink}href" not in linkChild.attributes:
linkChild.attributes["{http://www.w3.org/1999/xlink}href"] = \
"#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
XmlUtil.addChild(newLink, linkChild.qname,
attributes=linkChild.attributes,
text=linkChild.textValue)
targetInstance.saveInstance(overrideFilepath=targetUrl)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def validateTestcase(self, testcase):
self.modelXbrl.error(_("testcase {0}").format(os.path.basename(testcase.uri)))
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation.element)
# try to load instance document
self.modelXbrl.error(_("variation {0} {1}: {2}").format(modelTestcaseVariation.id, modelTestcaseVariation.name, modelTestcaseVariation.expected))
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
ModelDocument.Type.DTSENTRIES,
self.modelXbrl.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = ModelDocument.load(modelXbrl, readMeFirstUri, base=baseForElement)
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
else: # not a multi-schemaRef versioning report
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement)
if modelXbrl.modelDocument is None:
self.modelXbrl.error(_("Testcase {0} {1} document not loaded: {2}").format(
modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(readMeFirstUri)))
modelTestcaseVariation.status = "not loadable"
modelXbrl.close()
elif resultIsVersioningReport:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl) \
.validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
modelXbrl.close()
elif testcase.type == ModelDocument.Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
self.modelXbrl.error(_("Testcase {0} {1} DTSes not loaded, unable to generate versioning report").format(
modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(readMeFirstUri)))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes:
inputDTS.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
parameters = modelTestcaseVariation.parameters.copy()
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName:
parameters[dtsName] = (None, inputDTS)
self.instValidator.validate(modelXbrl, parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
if modelXbrl.formulaOutputInstance and len(modelTestcaseVariation.actual) == 0:
# if an output instance is created, validate it
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance)
if len(modelTestcaseVariation.actual) == 0: # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement)
if expectedInstance.modelDocument is None:
self.modelXbrl.error(_("Testcase {0} {1} expected result instance not loaded: {2}").format(
modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(modelTestcaseVariation.resultXbrlInstance)))
modelTestcaseVariation.status = "result not loadable"
expectedInstance.close()
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error(
_("Formula output {0} facts, expected {1} facts").format(
len(formulaOutputInstance.facts),
len(expectedInstance.facts)),
"err", "formula:resultFactCounts")
else:
for fact in expectedInstance.facts:
if not formulaOutputInstance.matchFact(fact):
formulaOutputInstance.error(
_("Formula output missing expected fact {0}").format(
fact),
"err", "formula:expectedFactMissing")
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance)
formulaOutputInstance.close()
formulaOutputInstance = None
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def runFromExcel(self, options):
testGenFileName = options.excelfilename
#testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls"
testGenDir = os.path.dirname(testGenFileName)
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt'
modelTestcases = ModelXbrl.create(self.modelManager)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow))
# start index file
indexFiles = [testGenDir + os.sep + 'creationTestcasesIndex.xml',
testGenDir + os.sep + 'consumptionTestcasesIndex.xml']
indexDocs = [xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today)
) for purpose in ("Creation","Consumption")]
testcasesElements = [XmlUtil.child(indexDoc, None, "testcases") for indexDoc in indexDocs]
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
row = testcaseIndexSheet.row(iRow)
if row[0].ctype == xlrd.XL_CELL_EMPTY or row[1].ctype == xlrd.XL_CELL_EMPTY or row[2].ctype == xlrd.XL_CELL_EMPTY:
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
intention = row[3].value
if intention is None or len(intention) == 0:
continue # test not ready to run
reason = row[4].value
expectedEvent = row[5].value
base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and reason.lower() not in ("n.a.", "error") and expectedEvent != "N.A.":
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base)
else:
modelDTS = ModelXbrl.create(self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts",
base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS, uri.strip(), base=base)
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom and modelDTSto:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace("from.xsd","report.xml")
reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile,
base)
testcasesDir = os.path.dirname(os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml",
testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"]
for i,testcaseFile in enumerate(testcaseFiles):
XmlUtil.addChild(testcasesElements[i], None, "testcase",
("uri",
testcaseFile[len(testGenDir)+1:].replace("\\","/")) )
# start testcase file
testcaseDocs = [xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'<name>{0}</name>'
'<description>{0}</description>'
'</testcase>'.format(testcaseName,purpose,today)
) for purpose in ("Creation","Consumption")]
testcaseElements = [XmlUtil.child(testcaseDoc, conformanceNS, "testcase") for testcaseDoc in testcaseDocs]
priorTestcasesDir = testcasesDir
variationID = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(modelTestcases)
modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto)
# check for expected elements
if expectedEvent and expectedEvent not in (
"No change", "N.A."):
if len(modelVersReport.xmlDocument.getElementsByTagNameNS('*',expectedEvent)) == 0:
modelTestcases.error(
"Generated test case {0} missing expected event {1}".format(
reportName,
expectedEvent),
"wrn", "missingEvent")
modelVersReport.close([])
for i,testcaseElt in enumerate(testcaseElements):
variationElement = XmlUtil.addChild(testcaseElt, conformanceNS, "variation",
attributes=("id", "_{0:02n}".format(variationID)))
XmlUtil.addChild(variationElement, conformanceNS, "name", text=intention)
dataElement = XmlUtil.addChild(variationElement, conformanceNS, "data")
for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")):
for schemaURI in schemaURIs.split(","):
XmlUtil.addChild(dataElement, conformanceNS, "schema",
attributes=((("dts",dtsAttr),) +
((("readMeFirst","true"),) if i == 0 else ())),
text=os.path.basename(schemaURI.strip()))
resultElement = XmlUtil.addChild(variationElement, conformanceNS, "result")
XmlUtil.addChild(resultElement if i == 0 else dataElement,
conformanceNS, "versioningReport",
attributes=(("readMeFirst","true") if i == 1 else ()),
text="report/" + reportName)
variationID += 1
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i,indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def load(self, filesource, nextaction):
self.filesource = filesource
self.modelXbrl = ModelXbrl.load(self, filesource, nextaction)
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def _create_cache(val):
"""
Creates the caches needed for dqc_us_0018
:param val: ValidateXbrl needed in order to save the cache
:type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
:return: no explicit return but creates and saves a cache in
dqc_us_rule\resources\DQC_US_0018
:rtype: None
"""
val.ugtNamespace = None
cntlr = val.modelXbrl.modelManager.cntlr
year = _EARLIEST_US_GAAP_YEAR
for ugt in ugtDocs:
deprecations_json_file = os.path.join(
os.path.dirname(__file__),
'resources',
'DQC_US_0018',
'{}_deprecated-concepts.json'.format(str(year))
)
if not os.path.isfile(deprecations_json_file):
ugt_doc_lb = ugt["docLB"]
val.usgaapDeprecations = {}
disclosure_system = (
val.modelXbrl.modelManager.validateDisclosureSystem
)
prior_validate_disclosure_system = disclosure_system
val.modelXbrl.modelManager.validateDisclosureSystem = False
deprecations_instance = ModelXbrl.load(
val.modelXbrl.modelManager,
openFileSource(ugt_doc_lb, cntlr),
_("built deprecations table in cache") # noqa
)
val.modelXbrl.modelManager.validateDisclosureSystem = (
prior_validate_disclosure_system
)
if deprecations_instance is not None:
dep_label = 'http://www.xbrl.org/2009/role/deprecatedLabel'
dep_date_label = (
'http://www.xbrl.org/2009/role/deprecatedDateLabel'
)
concept_label = XbrlConst.conceptLabel
relationship_set = (
deprecations_instance.relationshipSet(concept_label)
)
model_relationships = relationship_set.modelRelationships
for labelRel in model_relationships:
model_documentation = labelRel.toModelObject
concept = labelRel.fromModelObject.name
if model_documentation.role == dep_label:
val.usgaapDeprecations[concept] = (
model_documentation.text,
val.usgaapDeprecations.get(concept, ('', ''))[0]
)
elif model_documentation.role == dep_date_label:
val.usgaapDeprecations[concept] = (
model_documentation.text,
val.usgaapDeprecations.get(concept, ('', ''))[1]
)
json_str = str(
json.dumps(
val.usgaapDeprecations,
ensure_ascii=False, indent=0
)
) # might not be unicode in 2.7
saveFile(cntlr, deprecations_json_file, json_str)
deprecations_instance.close()
del deprecations_instance # dereference closed modelXbrl
year += 1
def create(self, newDocumentType=None, url=None, schemaRefs=None, createModelDocument=True):
self.modelXbrl = ModelXbrl.create(self, newDocumentType, url, schemaRefs, createModelDocument)
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
elif resultIsTaxonomyPackage:
from arelle import PackageManager, PrototypeInstanceObject
dtsName = readMeFirstUri
modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
else: # need own file source, may need instance discovery
filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
if filesource and not filesource.selection and filesource.isArchive:
for _archiveFile in filesource.dir: # find instance document in archive
filesource.select(_archiveFile)
if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
break # use this selection
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel)
modelXbrl.isTestcaseVariation = True
if modelXbrl.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif resultIsVersioningReport or resultIsTaxonomyPackage:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif resultIsTaxonomyPackage:
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
RenderingEvaluator.init(modelXbrl)
if modelXbrl.hasFormulae:
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
compareIxResultInstance = getattr(modelXbrl, "extractedInlineInstance", False) and modelTestcaseVariation.resultXbrlInstanceUri
if compareIxResultInstance:
formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
errMsgPrefix = "ix"
else: # delete input instances before formula output comparision
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
errMsgPrefix = "formula"
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts),
messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
else:
formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
def factFootnotes(fact, footnotesRelSet):
footnotes = []
footnoteRels = footnotesRelSet.fromModelObject(fact)
if footnoteRels:
# most process rels in same order between two instances, use labels to sort
for i, footnoteRel in enumerate(sorted(footnoteRels,
key=lambda r: (r.fromLabel,r.toLabel))):
modelObject = footnoteRel.toModelObject
if isinstance(modelObject, ModelResource):
footnotes.append("Footnote {}: {}".format(
i+1, # compare footnote with normalize-space
re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))))
elif isinstance(modelObject, ModelFact):
footnotes.append("Footnoted fact {}: {} context: {} value: {}".format(
i+1,
modelObject.qname,
modelObject.contextID,
collapseWhitespace(modelObject.value)))
return footnotes
for expectedInstanceFact in expectedInstance.facts:
unmatchedFactsStack = []
formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True)
if formulaOutputFact is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = expectedInstanceFact
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else: # compare footnotes
expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
if expectedInstanceFactFootnotes != formulaOutputFactFootnotes:
formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
_("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=expectedInstanceFactFootnotes, footnotes2=formulaOutputFactFootnotes,
messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
if compareIxResultInstance:
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def create(self, newDocumentType=None, url=None, schemaRefs=None, createModelDocument=True, isEntry=False, errorCaptureLevel=None, initialXml=None, base=None):
self.modelXbrl = ModelXbrl.create(self, newDocumentType, url, schemaRefs, createModelDocument, isEntry, errorCaptureLevel, initialXml, base)
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY):
for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri):
self.validateTestcase(doc) # testcases doc's are sorted by their uri (file names), e.g., for formula
elif hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings:
errorCaptureLevel = logging._checkLevel("WARNING")
else:
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
elif resultIsTaxonomyPackage:
from arelle import PackageManager, PrototypeInstanceObject
dtsName = readMeFirstUri
modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
else: # need own file source, may need instance discovery
filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
if filesource and not filesource.selection and filesource.isArchive:
for _archiveFile in filesource.dir or (): # find instance document in archive
filesource.select(_archiveFile)
if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
break # use this selection
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel)
modelXbrl.isTestcaseVariation = True
if modelXbrl.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif resultIsVersioningReport or resultIsTaxonomyPackage:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif resultIsTaxonomyPackage:
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement)
for d in modelTestcaseVariation.dataUris.values() for uri in d
if not UrlUtil.isAbsolute(uri))
foundDataFiles = set()
variationBase = os.path.dirname(baseForElement)
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
for _inputDTS in inputDTS:
for docUrl, doc in _inputDTS.urlDocs.items():
if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET:
if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file
if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles:
docUrl = docUrl.replace("-formula.xml", ".xf")
foundDataFiles.add(docUrl)
if expectedDataFiles - foundDataFiles:
modelXbrl.info("arelle:testcaseDataNotUsed",
_("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"),
modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles)))
if foundDataFiles - expectedDataFiles:
modelXbrl.info("arelle:testcaseDataUnexpected",
_("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"),
modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles)))
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
try:
RenderingEvaluator.init(modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrlHasFormulae = modelXbrl.hasFormulae
if modelXbrlHasFormulae:
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
try:
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and
modelTestcaseVariation.resultXbrlInstanceUri is not None)
if compareIxResultInstance:
formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
errMsgPrefix = "ix"
else: # delete input instances before formula output comparision
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
errMsgPrefix = "formula"
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
_matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
_("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts),
messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
else:
formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
def factFootnotes(fact, footnotesRelSet):
footnotes = {}
footnoteRels = footnotesRelSet.fromModelObject(fact)
if footnoteRels:
# most process rels in same order between two instances, use labels to sort
for i, footnoteRel in enumerate(sorted(footnoteRels,
key=lambda r: (r.fromLabel,r.toLabel))):
modelObject = footnoteRel.toModelObject
if isinstance(modelObject, ModelResource):
xml = modelObject.viewText().strip()
footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))
elif isinstance(modelObject, ModelFact):
footnotes["Footnoted fact {}".format(i+1)] = \
"{} context: {} value: {}".format(
modelObject.qname,
modelObject.contextID,
collapseWhitespace(modelObject.value))
return footnotes
for expectedInstanceFact in expectedInstance.facts:
unmatchedFactsStack = []
formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False)
#formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True)
if formulaOutputFact is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = expectedInstanceFact
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else: # compare footnotes
expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or
set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())):
formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
_("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()),
messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
if compareIxResultInstance:
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
_statusCounts = OrderedDict((("pass",0),("fail",0)))
for tv in getattr(testcase, "testcaseVariations", ()):
_statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1
self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k))
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in modelXbrl.contexts.values():
newCntx = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem:
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)):
if elt.tag in ("a", "img") and not isHttpUrl(attrValue) and not os.path.isabs(attrvalue):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src"):
filingFiles.add(attrValue)
except (XMLSyntaxError, UnicodeDecodeError):
pass
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
# footnote links
footnoteIdCount = {}
modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and
any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype):
if HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
addChild(newLink, linkChild.qname,
attributes=attributes)
elif isinstance(linkChild, ArcPrototype):
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
newChild = addChild(newLink, linkChild.qname,
attributes=attributes)
copyIxFootnoteHtml(linkChild, newChild, withText=True)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyIxFootnoteHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrvalue):
filingFiles.add(attrValue)
targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def _make_cache(val, ugt, cntlr, ugt_default_dimensions_json_file):
"""
Creates a new caches for the Taxonomy default dimensions
:param val: ValidateXbrl to be validated
:type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
:param ugt: Taxonomy to check
:type ugt: str
:param ugt_default_dimensions_json_file: location to save json default
dimensions
:type ugt_default_dimensions_json_file: str
:return: no explicit return, but saves caches for dqc_us_0041
:rtype: None
"""
started_at = time.time()
ugt_entry_xsd = ugt["entryXsd"]
val.usgaapDefaultDimensions = {}
prior_validate_disclosure_system = (
val.modelXbrl.modelManager.validateDisclosureSystem
)
val.modelXbrl.modelManager.validateDisclosureSystem = False
ugt_entry_xsd_instance = (
ModelXbrl.load(
val.modelXbrl.modelManager,
openFileSource(ugt_entry_xsd, cntlr),
_("opened us-gaap entry xsd") # noqa
)
)
val.modelXbrl.modelManager.validateDisclosureSystem = (
prior_validate_disclosure_system
)
if ugt_entry_xsd_instance is None:
val.modelXbrl.error(
"arelle:notLoaded",
_("US-GAAP entry xsd not loaded: %(file)s"), # noqa
modelXbrl=val,
file=os.path.basename(ugt_entry_xsd)
)
else:
model_relationships = (
ugt_entry_xsd_instance.relationshipSet(
XbrlConst.dimensionDefault
).modelRelationships
)
for default_dim_rel in model_relationships:
if _default_dim_rel_is_instance(default_dim_rel):
from_name = default_dim_rel.fromModelObject.name
to_name = default_dim_rel.toModelObject.name
val.usgaapDefaultDimensions[from_name] = to_name
json_str = str(
json.dumps(
val.usgaapDefaultDimensions,
ensure_ascii=False,
indent=0
)
) # might not be unicode in 2.7
# 2.7 gets unicode this way
saveFile(cntlr, ugt_default_dimensions_json_file, json_str)
ugt_entry_xsd_instance.close()
del ugt_entry_xsd_instance # dereference closed modelXbrl
val.modelXbrl.profileStat(
_("build default dimensions cache"), # noqa
time.time() - started_at
)