def validateRssFeed(self):
     self.modelXbrl.info("info", "RSS Feed", modelDocument=self.modelXbrl)
     from arelle.FileSource import openFileSource
     for rssItem in self.modelXbrl.modelDocument.rssItems:
         self.modelXbrl.info("info", _("RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"),
             modelObject=rssItem, accessionNumber=rssItem.accessionNumber, formType=rssItem.formType, companyName=rssItem.companyName, period=rssItem.period)
         modelXbrl = None
         try:
             modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                        openFileSource(rssItem.zippedUrl, self.modelXbrl.modelManager.cntlr),
                                        _("validating"))
             self.instValidator.validate(modelXbrl, self.modelXbrl.modelManager.formulaOptions.typedParameters())
             self.instValidator.close()
             rssItem.setResults(modelXbrl)
             self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, rssItem.objectId())
             for pluginXbrlMethod in pluginClassMethods("Validate.RssItem"):
                 pluginXbrlMethod(self, modelXbrl, rssItem)
             modelXbrl.close()
         except Exception as err:
             self.modelXbrl.error("exception",
                 _("RSS item validation exception: %(error)s, instance: %(instance)s"),
                 modelXbrl=(self.modelXbrl, modelXbrl),
                 instance=rssItem.zippedUrl, error=err,
                 exc_info=True)
             try:
                 self.instValidator.close()
                 if modelXbrl is not None:
                     modelXbrl.close()
             except Exception as err:
                 pass
         del modelXbrl  # completely dereference
 def load(self, filesource, nextaction=None):
     """Load an entry point modelDocument object(s), which in turn load documents they discover 
     (for the case of instance, taxonomies, and versioning reports), but defer loading instances 
     for test case and RSS feeds.  
     
     The modelXbrl that is loaded is 'stacked', by this class, so that any modelXbrl operations such as validate, 
     and close, operate on the most recently loaded modelXbrl, and compareDTSes operates on the two 
     most recently loaded modelXbrl's.
     
     :param filesource: may be a FileSource object, with the entry point selected, or string file name (or web URL). 
     :type filesource: FileSource or str
     :param nextAction: status line text string, if any, to show upon completion
     :type nextAction: str
     """
     try:
         if filesource.url.startswith("urn:uuid:"): # request for an open modelXbrl
             for modelXbrl in self.loadedModelXbrls:
                 if not modelXbrl.isClosed and modelXbrl.uuid == filesource.url:
                     return modelXbrl
             raise IOError(_("Open file handle is not open: {0}").format(filesource.url))
     except AttributeError:
         pass # filesource may be a string, which has no url attribute
     self.filesource = filesource
     modelXbrl = None # loaded modelXbrl
     for customLoader in pluginClassMethods("ModelManager.Load"):
         modelXbrl = customLoader(self, filesource)
         if modelXbrl is not None:
             break # custom loader did the loading
     if modelXbrl is None:  # use default xbrl loader
         modelXbrl = ModelXbrl.load(self, filesource, nextaction)
     self.modelXbrl = modelXbrl
     self.loadedModelXbrls.append(self.modelXbrl)
     return self.modelXbrl
Exemple #3
0
 def runFromXml(self):
     testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\Arelle\roland test cases\1000-Concepts\index.xml"
     filesource = FileSource.FileSource(testGenFileName)
     startedAt = time.time()
     LogHandler(self) # start logger
     modelTestcases = self.modelManager.load(filesource, _("views loading"))
     self.addToLog(_("[info] loaded in {0:.2} secs").format(time.time() - startedAt))
     if modelTestcases.modelDocument.type == ModelDocument.Type.TESTCASESINDEX:
         for testcasesElement in modelTestcases.modelDocument.iter(tag="testcases"):
             rootAttr = testcasesElement.get("root")
             title = testcasesElement.get("title")
             self.addToLog(_("[info] testcases {0}").format(title))
             if rootAttr is not None:
                 base = os.path.join(os.path.dirname(modelTestcases.modelDocument.filepath),rootAttr) + os.sep
             else:
                 base = self.filepath
             for testcaseElement in testcasesElement.iterchildren(tag="testcase"):
                 uriFrom = testcaseElement.get("uriFrom")
                 uriTo = testcaseElement.get("uriTo")
                 modelDTSfrom = modelDTSto = None
                 self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
                 if uriFrom is not None and uriTo is not None:
                     modelDTSfrom = ModelXbrl.load(modelTestcases.modelManager, 
                                                uriFrom,
                                                _("loading from DTS"), 
                                                base=base)
                     modelDTSto = ModelXbrl.load(modelTestcases.modelManager, 
                                                uriTo,
                                                _("loading to DTS"), 
                                                base=base)
                     if modelDTSfrom is not None and modelDTSto is not None:
                         # generate differences report
                         reportName = os.path.basename(uriFrom).replace("from.xsd","report.xml")
                         reportFile = os.path.dirname(uriFrom) + "\\report\\" + reportName
                         reportFullPath = self.webCache.normalizeUrl(
                                             reportFile, 
                                             base)
                         try:
                             os.makedirs(os.path.dirname(reportFullPath))
                         except WindowsError:
                             pass # dir already exists
                         ModelVersReport.ModelVersReport(modelTestcases).diffDTSes(
                                       reportFullPath,
                                       modelDTSfrom, modelDTSto)
Exemple #4
0
 def load(self, filesource, nextaction=None):
     try:
         if filesource.url.startswith("urn:uuid:"): # request for an open modelXbrl
             for modelXbrl in self.loadedModelXbrls:
                 if not modelXbrl.isClosed and modelXbrl.uuid == filesource.url:
                     return modelXbrl
             raise IOError(_("Open file handle is not open: {0}".format(filesource.url)))
     except AttributeError:
         pass # filesource may be a string, which has no url attribute
     self.filesource = filesource
     self.modelXbrl = ModelXbrl.load(self, filesource, nextaction)
     self.loadedModelXbrls.append(self.modelXbrl)
     return self.modelXbrl
Exemple #5
0
    def __init__(self, cntlr, isCmdLine=False):
        self.cntlr = cntlr

        # setup tester
        xml = "<rootElement/>"
        self.modelXbrl = ModelXbrl.create(cntlr.modelManager, ModelDocument.Type.UnknownNonXML, initialXml=xml, isEntry=True)
        self.validator = ValidateXbrl.ValidateXbrl(self.modelXbrl)
        self.validator.validate(self.modelXbrl)  # required to set up
        cntlr.showStatus(_("Initializing Formula Grammar"))
        XPathParser.initializeParser(cntlr.modelManager)
        cntlr.showStatus(None)

        self.trRegs = sorted(ixtNamespaces.keys())
        self.trPrefixNSs = dict((qn.prefix, qn.namespaceURI) 
                                  for qn in self.modelXbrl.modelManager.customTransforms.keys())
        self.trRegs.extend(sorted(self.trPrefixNSs.keys()))
        self.trPrefixNSs.update(ixtNamespaces)
    def saveTargetDocument(self):
        targetUrl = self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(self.targetDocumentPreferredFilename, self.filepath)
        targetUrlParts = targetUrl.rpartition(".")
        targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
        self.modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
        targetInstance = ModelXbrl.create(self.modelXbrl.modelManager, 
                                          newDocumentType=Type.INSTANCE,
                                          url=targetUrl,
                                          schemaRefs=self.targetDocumentSchemaRefs,
                                          isEntry=True)
        ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults 
        for context in self.modelXbrl.contexts.values():
            newCntx = targetInstance.createContext(context.entityIdentifier[0],
                                                   context.entityIdentifier[1],
                                                   'instant' if context.isInstantPeriod else
                                                   'duration' if context.isStartEndPeriod
                                                   else 'forever',
                                                   context.startDatetime,
                                                   context.endDatetime,
                                                   None, 
                                                   context.qnameDims, [], [],
                                                   id=context.id)
        for unit in self.modelXbrl.units.values():
            measures = unit.measures
            newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id)

        self.modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
        for fact in self.modelXbrl.facts:
            if fact.isItem:
                attrs = [("contextRef", fact.contextID)]
                if fact.isNumeric:
                    attrs.append(("unitRef", fact.unitID))
                    if fact.get("decimals"):
                        attrs.append(("decimals", fact.get("decimals")))
                    if fact.get("precision"):
                        attrs.append(("precision", fact.get("precision")))
                if fact.isNil:
                    attrs.append(("{http://www.w3.org/2001/XMLSchema-instance}nil","true"))
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.elementText
                newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text)
        targetInstance.saveInstance(overrideFilepath=targetUrl)
        self.modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
Exemple #7
0
def deprecatedConceptDatesFile(modelManager, abbrNs, latestTaxonomyDoc):
    if latestTaxonomyDoc is None:
        return None
    if not abbrNs: # none for an unexpected namespace pattern
        return None
    cntlr = modelManager.cntlr
    _fileName = resourcesFilePath(modelManager, abbrNs.partition("/")[0] + "-deprecated-concepts.json")
    _deprecatedLabelRole = latestTaxonomyDoc["deprecatedLabelRole"]
    _deprecatedDateMatchPattern = latestTaxonomyDoc["deprecationDatePattern"]
    if os.path.exists(_fileName):
        return _fileName
    # load labels and store file name
    modelManager.addToLog(_("loading {} deprecated concepts into {}").format(abbrNs, _fileName), messageCode="info")
    deprecatedConceptDates = {}
    # load without SEC/EFM validation (doc file would not be acceptable)
    priorValidateDisclosureSystem = modelManager.validateDisclosureSystem
    modelManager.validateDisclosureSystem = False
    from arelle import ModelXbrl
    deprecationsInstance = ModelXbrl.load(modelManager, 
          # "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
          # load from zip (especially after caching) is incredibly faster
          openFileSource(latestTaxonomyDoc["deprecatedLabels"], cntlr), 
          _("built deprecations table in cache"))
    modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
    if deprecationsInstance is None:
        modelManager.addToLog(
            _("%(name)s documentation not loaded"),
            messageCode="arelle:notLoaded", messageArgs={"modelXbrl": val, "name":_abbrNs})
    else:   
        # load deprecations
        for labelRel in deprecationsInstance.relationshipSet(XbrlConst.conceptLabel).modelRelationships:
            modelLabel = labelRel.toModelObject
            conceptName = labelRel.fromModelObject.name
            if modelLabel.role == _deprecatedLabelRole:
                match = _deprecatedDateMatchPattern.match(modelLabel.text)
                if match is not None:
                    date = match.group(1)
                    if date:
                        deprecatedConceptDates[conceptName] = date
        jsonStr = _STR_UNICODE(json.dumps(deprecatedConceptDates, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
        saveFile(cntlr, _fileName, jsonStr)  # 2.7 gets unicode this way
        deprecationsInstance.close()
        del deprecationsInstance # dereference closed modelXbrl
Exemple #8
0
 def validateRssFeed(self):
     self.modelXbrl.info("info", "RSS Feed", modelDocument=self.modelXbrl)
     from arelle.FileSource import openFileSource
     reloadCache = getattr(self.modelXbrl, "reloadCache", False)
     for rssItem in self.modelXbrl.modelDocument.rssItems:
         if getattr(rssItem, "skipRssItem", False):
             self.modelXbrl.info("info", _("skipping RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"),
                 modelObject=rssItem, accessionNumber=rssItem.accessionNumber, formType=rssItem.formType, companyName=rssItem.companyName, period=rssItem.period)
             continue
         self.modelXbrl.info("info", _("RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"),
             modelObject=rssItem, accessionNumber=rssItem.accessionNumber, formType=rssItem.formType, companyName=rssItem.companyName, period=rssItem.period)
         modelXbrl = None
         try:
             modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                        openFileSource(rssItem.zippedUrl, self.modelXbrl.modelManager.cntlr, reloadCache=reloadCache),
                                        _("validating"), rssItem=rssItem)
             for pluginXbrlMethod in pluginClassMethods("RssItem.Xbrl.Loaded"):  
                 pluginXbrlMethod(modelXbrl, {}, rssItem)      
             if getattr(rssItem, "doNotProcessRSSitem", False) or modelXbrl.modelDocument is None:
                 modelXbrl.close()
                 continue # skip entry based on processing criteria
             self.instValidator.validate(modelXbrl, self.modelXbrl.modelManager.formulaOptions.typedParameters(self.modelXbrl.prefixedNamespaces))
             self.instValidator.close()
             rssItem.setResults(modelXbrl)
             self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, rssItem.objectId())
             for pluginXbrlMethod in pluginClassMethods("Validate.RssItem"):
                 pluginXbrlMethod(self, modelXbrl, rssItem)
             modelXbrl.close()
         except Exception as err:
             self.modelXbrl.error("exception:" + type(err).__name__,
                 _("RSS item validation exception: %(error)s, instance: %(instance)s"),
                 modelXbrl=(self.modelXbrl, modelXbrl),
                 instance=rssItem.zippedUrl, error=err,
                 exc_info=True)
             try:
                 self.instValidator.close()
                 if modelXbrl is not None:
                     modelXbrl.close()
             except Exception as err:
                 pass
         del modelXbrl  # completely dereference
Exemple #9
0
 def validateTestcase(self, testcase):
     self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
     self.modelXbrl.viewModelObject(testcase.objectId())
     if hasattr(testcase, "testcaseVariations"):
         for modelTestcaseVariation in testcase.testcaseVariations:
             # update ui thread via modelManager (running in background here)
             self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
             # is this a versioning report?
             resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
             resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
             formulaOutputInstance = None
             inputDTSes = defaultdict(list)
             baseForElement = testcase.baseForElement(modelTestcaseVariation)
             # try to load instance document
             self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s"),
                 modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected)
             for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
                 if isinstance(readMeFirstUri,tuple):
                     # dtsName is for formula instances, but is from/to dts if versioning
                     dtsName, readMeFirstUri = readMeFirstUri
                 elif resultIsVersioningReport:
                     if inputDTSes: dtsName = "to"
                     else: dtsName = "from"
                 else:
                     dtsName = None
                 if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
                     if dtsName in inputDTSes:
                         dtsName = inputDTSes[dtsName]
                     else:
                         modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, 
                                      ModelDocument.Type.DTSENTRIES,
                                      self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
                                      isEntry=True)
                     DTSdoc = modelXbrl.modelDocument
                     DTSdoc.inDTS = True
                     doc = ModelDocument.load(modelXbrl, readMeFirstUri, base=baseForElement)
                     if doc is not None:
                         DTSdoc.referencesDocument[doc] = "import"  #fake import
                         doc.inDTS = True
                 else: # not a multi-schemaRef versioning report
                     modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                readMeFirstUri,
                                                _("validating"), 
                                                base=baseForElement,
                                                useFileSource=self.useFileSource)
                 if modelXbrl.modelDocument is None:
                     self.modelXbrl.error("arelle:notLoaded",
                          _("Testcase %(id)s %(name)s document not loaded: %(file)s"),
                          modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
                     modelXbrl.close()
                     self.determineNotLoadedTestStatus(modelTestcaseVariation)
                 elif resultIsVersioningReport:
                     inputDTSes[dtsName] = modelXbrl
                 elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
                     ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
                     self.determineTestStatus(modelTestcaseVariation, modelXbrl)
                     modelXbrl.close()
                 elif testcase.type == ModelDocument.Type.REGISTRYTESTCASE:
                     self.instValidator.validate(modelXbrl)  # required to set up dimensions, etc
                     self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, 
                                modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
                     self.determineTestStatus(modelTestcaseVariation, modelXbrl)
                     self.instValidator.close()
                     modelXbrl.close()
                 else:
                     inputDTSes[dtsName].append(modelXbrl)
             if resultIsVersioningReport and modelXbrl.modelDocument:
                 versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
                     modelTestcaseVariation.versioningReportUri, baseForElement)
                 if os.path.exists(versReportFile): #validate existing
                     modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
                     if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
                         ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
                         self.determineTestStatus(modelTestcaseVariation, modelVersReport)
                         modelVersReport.close()
                 elif len(inputDTSes) == 2:
                     ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
                           versReportFile, inputDTSes["from"], inputDTSes["to"])
                     modelTestcaseVariation.status = "generated"
                 else:
                     self.modelXbrl.error("arelle:notLoaded",
                          _("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
                          modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
                     modelTestcaseVariation.status = "failed"
                 for inputDTS in inputDTSes.values():
                     inputDTS.close()
                 del inputDTSes # dereference
             elif inputDTSes:
                 # validate schema, linkbase, or instance
                 modelXbrl = inputDTSes[None][0]
                 parameters = modelTestcaseVariation.parameters.copy()
                 for dtsName, inputDTS in inputDTSes.items():  # input instances are also parameters
                     if dtsName:
                         parameters[dtsName] = (None, inputDTS)
                 self.instValidator.validate(modelXbrl, parameters)
                 if modelTestcaseVariation.resultIsInfoset:
                     infoset = ModelXbrl.load(self.modelXbrl.modelManager, 
                                              modelTestcaseVariation.resultInfosetUri,
                                                _("loading result infoset"), 
                                                base=baseForElement,
                                                useFileSource=self.useFileSource)
                     if infoset.modelDocument is None:
                         self.modelXbrl.error("arelle:notLoaded",
                             _("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
                             modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, 
                             file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
                         modelTestcaseVariation.status = "result infoset not loadable"
                     else:   # check infoset
                         ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
                     infoset.close()
                 self.determineTestStatus(modelTestcaseVariation, modelXbrl) # include infoset errors in status
                 self.instValidator.close()
                 if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): 
                     # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
                     modelXbrl.formulaOutputInstance.hasFormulae = False #  block formulae on output instance (so assertion of input is not lost)
                     self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
                     self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance)
                     if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
                         formulaOutputInstance = modelXbrl.formulaOutputInstance
                         modelXbrl.formulaOutputInstance = None # prevent it from being closed now
                     self.instValidator.close()
                 for inputDTSlist in inputDTSes.values():
                     for inputDTS in inputDTSlist:
                         inputDTS.close()
                 if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
                     expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                modelTestcaseVariation.resultXbrlInstanceUri,
                                                _("loading expected result XBRL instance"), 
                                                base=baseForElement,
                                                useFileSource=self.useFileSource)
                     if expectedInstance.modelDocument is None:
                         self.modelXbrl.error("arelle:notLoaded",
                             _("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
                             modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, 
                             file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
                         modelTestcaseVariation.status = "result not loadable"
                         expectedInstance.close()
                     else:   # compare facts
                         if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
                             formulaOutputInstance.error("formula:resultFactCounts",
                                 _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
                                 modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
                                      expectedFacts=len(expectedInstance.facts))
                         else:
                             for fact in expectedInstance.facts:
                                 if formulaOutputInstance.matchFact(fact) is None:
                                     formulaOutputInstance.error("formula:expectedFactMissing",
                                         _("Formula output missing expected fact %(fact)s"),
                                         modelXbrl=fact, fact=fact.qname)
                     self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance)
                     formulaOutputInstance.close()
                     formulaOutputInstance = None
             # update ui thread via modelManager (running in background here)
             self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
                 
         self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def setup(val):
    val.linroleDefinitionIsDisclosure = re.compile(r"-\s+Disclosure\s+-\s",
                                                   re.IGNORECASE)
    val.linkroleDefinitionStatementSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*", # no restriction to type of statement
                                                      re.IGNORECASE)
    val.ugtNamespace = None
    cntlr = val.modelXbrl.modelManager.cntlr
    # load deprecated concepts for filed year of us-gaap
    for ugt in ugtDocs:
        ugtNamespace = ugt["namespace"]
        if ugtNamespace in val.modelXbrl.namespaceDocs and len(val.modelXbrl.namespaceDocs[ugtNamespace]) > 0:
            val.ugtNamespace = ugtNamespace
            usgaapDoc = val.modelXbrl.namespaceDocs[ugtNamespace][0]
            deprecationsJsonFile = usgaapDoc.filepathdir + os.sep + "deprecated-concepts.json"
            file = None
            try:
                file = openFileStream(cntlr, deprecationsJsonFile, 'rt', encoding='utf-8')
                val.usgaapDeprecations = json.load(file)
                file.close()
            except Exception:
                if file:
                    file.close()
                val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} deprecated concepts into cache").format(ugt["year"]))
                startedAt = time.time()
                ugtDocLB = ugt["docLB"]
                val.usgaapDeprecations = {}
                # load without SEC/EFM validation (doc file would not be acceptable)
                priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem
                val.modelXbrl.modelManager.validateDisclosureSystem = False
                deprecationsInstance = ModelXbrl.load(val.modelXbrl.modelManager, 
                      # "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
                      # load from zip (especially after caching) is incredibly faster
                      openFileSource(ugtDocLB, cntlr), 
                      _("built deprecations table in cache"))
                val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
                if deprecationsInstance is None:
                    val.modelXbrl.error("arelle:notLoaded",
                        _("US-GAAP documentation not loaded: %(file)s"),
                        modelXbrl=val, file=os.path.basename(ugtDocLB))
                else:   
                    # load deprecations
                    for labelRel in deprecationsInstance.relationshipSet(XbrlConst.conceptLabel).modelRelationships:
                        modelDocumentation = labelRel.toModelObject
                        conceptName = labelRel.fromModelObject.name
                        if modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedLabel':
                            val.usgaapDeprecations[conceptName] = (val.usgaapDeprecations.get(conceptName, ('',''))[0], modelDocumentation.text)
                        elif modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedDateLabel':
                            val.usgaapDeprecations[conceptName] = (modelDocumentation.text, val.usgaapDeprecations.get(conceptName, ('',''))[1])
                    jsonStr = _STR_UNICODE(json.dumps(val.usgaapDeprecations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
                    saveFile(cntlr, deprecationsJsonFile, jsonStr)  # 2.7 gets unicode this way
                    deprecationsInstance.close()
                    del deprecationsInstance # dereference closed modelXbrl
                val.modelXbrl.profileStat(_("build us-gaap deprecated concepts cache"), time.time() - startedAt)
            ugtCalcsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-calculations.json"
            ugtDefaultDimensionsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-default-dimensions.json"
            file = None
            try:
                file = openFileStream(cntlr, ugtCalcsJsonFile, 'rt', encoding='utf-8')
                val.usgaapCalculations = json.load(file)
                file.close()
                file = openFileStream(cntlr, ugtDefaultDimensionsJsonFile, 'rt', encoding='utf-8')
                val.usgaapDefaultDimensions = json.load(file)
                file.close()
            except Exception:
                if file:
                    file.close()
                val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} calculations and default dimensions into cache").format(ugt["year"]))
                startedAt = time.time()
                ugtEntryXsd = ugt["entryXsd"]
                val.usgaapCalculations = {}
                val.usgaapDefaultDimensions = {}
                # load without SEC/EFM validation (doc file would not be acceptable)
                priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem
                val.modelXbrl.modelManager.validateDisclosureSystem = False
                calculationsInstance = ModelXbrl.load(val.modelXbrl.modelManager, 
                      # "http://xbrl.fasb.org/us-gaap/2012/entire/us-gaap-entryPoint-std-2012-01-31.xsd",
                      # load from zip (especially after caching) is incredibly faster
                      openFileSource(ugtEntryXsd, cntlr), 
                      _("built us-gaap calculations cache"))
                val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
                if calculationsInstance is None:
                    val.modelXbrl.error("arelle:notLoaded",
                        _("US-GAAP calculations not loaded: %(file)s"),
                        modelXbrl=val, file=os.path.basename(ugtEntryXsd))
                else:   
                    # load calculations
                    for ELR in calculationsInstance.relationshipSet(XbrlConst.summationItem).linkRoleUris:
                        elrRelSet = calculationsInstance.relationshipSet(XbrlConst.summationItem, ELR)
                        definition = ""
                        for roleType in calculationsInstance.roleTypes.get(ELR,()):
                            definition = roleType.definition
                            break
                        isStatementSheet = bool(val.linkroleDefinitionStatementSheet.match(definition))
                        elrUgtCalcs = {"#roots": [c.name for c in elrRelSet.rootConcepts],
                                       "#definition": definition,
                                       "#isStatementSheet": isStatementSheet}
                        for relFrom, rels in elrRelSet.fromModelObjects().items():
                            elrUgtCalcs[relFrom.name] = [rel.toModelObject.name for rel in rels]
                        val.usgaapCalculations[ELR] = elrUgtCalcs
                    jsonStr = _STR_UNICODE(json.dumps(val.usgaapCalculations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
                    saveFile(cntlr, ugtCalcsJsonFile, jsonStr)  # 2.7 gets unicode this way
                    # load default dimensions
                    for defaultDimRel in calculationsInstance.relationshipSet(XbrlConst.dimensionDefault).modelRelationships:
                        if defaultDimRel.fromModelObject is not None and defaultDimRel.toModelObject is not None:
                            val.usgaapDefaultDimensions[defaultDimRel.fromModelObject.name] = defaultDimRel.toModelObject.name
                    jsonStr = _STR_UNICODE(json.dumps(val.usgaapDefaultDimensions, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
                    saveFile(cntlr, ugtDefaultDimensionsJsonFile, jsonStr)  # 2.7 gets unicode this way
                    calculationsInstance.close()
                    del calculationsInstance # dereference closed modelXbrl
                val.modelXbrl.profileStat(_("build us-gaap calculations and default dimensions cache"), time.time() - startedAt)
            break
    val.deprecatedFactConcepts = defaultdict(list)
    val.deprecatedDimensions = defaultdict(list)
    val.deprecatedMembers = defaultdict(list)
Exemple #11
0
 def watchCycle(self):
     while not self.stopRequested:
         rssWatchOptions = self.rssModelXbrl.modelManager.rssWatchOptions
         
         # check rss expiration
         rssHeaders = self.cntlr.webCache.getheaders(self.rssModelXbrl.modelManager.rssWatchOptions.get("feedSourceUri"))
         expires = parseRfcDatetime(rssHeaders.get("expires"))
         reloadNow = True # texpires and expires > datetime.datetime.now()
         
         # reload rss feed
         self.rssModelXbrl.reload('checking RSS items', reloadCache=reloadNow)
         if self.stopRequested: break
         # setup validator
         postLoadActions = []
         if rssWatchOptions.get("validateDisclosureSystemRules"):
             self.instValidator = ValidateFiling.ValidateFiling(self.rssModelXbrl)
             postLoadActions.append(_("validating"))
         elif rssWatchOptions.get("validateXbrlRules") or rssWatchOptions.get("validateFormulaAssertions"):
             self.instValidator = ValidateXbrl.ValidateXbrl(self.rssModelXbrl)
             postLoadActions.append(_("validating"))
             if (rssWatchOptions.get("validateFormulaAssertions")):
                 postLoadActions.append(_("running formulas"))
         else:
             self.instValidator = None
            
         matchTextExpr = rssWatchOptions.get("matchTextExpr") 
         if matchTextExpr:
             matchPattern = re.compile(matchTextExpr)
             postLoadActions.append(_("matching text"))
         else:
             matchPattern= None
         postLoadAction = ', '.join(postLoadActions)
         
         # anything to check new filings for
         if (rssWatchOptions.get("validateDisclosureSystemRules") or
             rssWatchOptions.get("validateXbrlRules") or
             rssWatchOptions.get("validateCalcLinkbase") or
             rssWatchOptions.get("validateFormulaAssertions") or
             rssWatchOptions.get("alertMatchedFactText") or
             any(pluginXbrlMethod(rssWatchOptions)
                 for pluginXbrlMethod in pluginClassMethods("RssWatch.HasWatchAction"))
             ):
             # form keys in ascending order of pubdate
             pubDateRssItems = []
             for rssItem in self.rssModelXbrl.modelDocument.rssItems:
                 pubDateRssItems.append((rssItem.pubDate,rssItem.objectId()))
             
             for pubDate, rssItemObjectId in sorted(pubDateRssItems):
                 rssItem = self.rssModelXbrl.modelObject(rssItemObjectId)
                 # update ui thread via modelManager (running in background here)
                 self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId())
                 if self.stopRequested:
                     break
                 latestPubDate = XmlUtil.datetimeValue(rssWatchOptions.get("latestPubDate"))
                 if (latestPubDate and 
                     rssItem.pubDate < latestPubDate):
                     continue
                 try:
                     # try zipped URL if possible, else expanded instance document
                     modelXbrl = ModelXbrl.load(self.rssModelXbrl.modelManager, 
                                                openFileSource(rssItem.zippedUrl, self.cntlr),
                                                postLoadAction)
                     if self.stopRequested:
                         modelXbrl.close()
                         break
                     
                     emailAlert = False
                     if modelXbrl.modelDocument is None:
                         modelXbrl.error("arelle.rssWatch",
                                         _("RSS item %(company)s %(form)s document not loaded: %(date)s"),
                                         modelXbrl=modelXbrl, company=rssItem.companyName, 
                                         form=rssItem.formType, date=rssItem.filingDate)
                         rssItem.status = "not loadable"
                     else:
                         # validate schema, linkbase, or instance
                         if self.stopRequested:
                             modelXbrl.close()
                             break
                         if self.instValidator:
                             self.instValidator.validate(modelXbrl)
                             if modelXbrl.errors and rssWatchOptions.get("alertValiditionError"):
                                 emailAlert = True
                         for pluginXbrlMethod in pluginClassMethods("RssWatch.DoWatchAction"):  
                             pluginXbrlMethod(modelXbrl, rssWatchOptions, rssItem)      
                         # check match expression
                         if matchPattern:
                             for fact in modelXbrl.factsInInstance:
                                 v = fact.value
                                 if v is not None:
                                     m = matchPattern.search(v)
                                     if m:
                                         fr, to = m.span()
                                         msg = _("Fact Variable {0}\n context {1}\n matched text: {2}").format( 
                                                 fact.qname, fact.contextID, v[max(0,fr-20):to+20])
                                         modelXbrl.info("arelle.rssInfo",
                                                        msg,
                                                        modelXbrl=modelXbrl) # msg as code passes it through to the status
                                         if rssWatchOptions.get("alertMatchedFactText"):
                                             emailAlert = True
                                     
                         if (rssWatchOptions.get("formulaFileUri") and rssWatchOptions.get("validateFormulaAssertions") and
                             self.instValidator): 
                             # attach formulas
                             ModelDocument.load(modelXbrl, rssWatchOptions["formulaFileUri"])
                             ValidateFormula.validate(self.instValidator)
                             
                     rssItem.setResults(modelXbrl)
                     modelXbrl.close()
                     del modelXbrl  # completely dereference
                     self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId())
                     if rssItem.assertionUnsuccessful and rssWatchOptions.get("alertAssertionUnsuccessful"):
                         emailAlert = True
                     
                     msg = _("Filing CIK {0}\n "
                              "company {1}\n "
                              "published {2}\n "
                              "form type {3}\n "
                              "filing date {4}\n "
                              "period {5}\n "
                              "year end {6}\n "
                              "results: {7}").format(
                              rssItem.cikNumber,
                              rssItem.companyName,
                              rssItem.pubDate,
                              rssItem.formType,
                              rssItem.filingDate,
                              rssItem.period,
                              rssItem.fiscalYearEnd,
                              rssItem.status)
                     self.rssModelXbrl.info("arelle:rssWatch", msg, modelXbrl=self.rssModelXbrl)
                     emailAddress = rssWatchOptions.get("emailAddress")
                     if emailAlert and emailAddress:
                         self.rssModelXbrl.modelManager.showStatus(_("sending e-mail alert"))
                         import smtplib
                         from email.mime.text import MIMEText
                         emailMsg = MIMEText(msg)
                         emailMsg["Subject"] = _("Arelle RSS Watch alert on {0}").format(rssItem.companyName)
                         emailMsg["From"] = emailAddress
                         emailMsg["To"] = emailAddress
                         smtp = smtplib.SMTP()
                         smtp.sendmail(emailAddress, [emailAddress], emailMsg.as_string())
                         smtp.quit()
                     self.rssModelXbrl.modelManager.showStatus(_("RSS item {0}, {1} completed, status {2}").format(rssItem.companyName, rssItem.formType, rssItem.status), 3500)
                     self.rssModelXbrl.modelManager.cntlr.rssWatchUpdateOption(rssItem.pubDate.strftime('%Y-%m-%dT%H:%M:%S'))
                 except Exception as err:
                     self.rssModelXbrl.error("arelle.rssError",
                                             _("RSS item %(company)s, %(form)s, %(date)s, exception: %(error)s"),
                                             modelXbrl=self.rssModelXbrl, company=rssItem.companyName, 
                                             form=rssItem.formType, date=rssItem.filingDate, error=err,
                                             exc_info=True)
                 if self.stopRequested: break
         if self.stopRequested: 
             self.cntlr.showStatus(_("RSS watch, stop requested"), 10000)
         else:
             import time
             time.sleep(600)
         
     self.thread = None  # close thread
     self.stopRequested = False
     
             
def createTargetInstance(modelXbrl, targetUrl, targetDocumentSchemaRefs, filingFiles, baseXmlLang=None, defaultXmlLang=None):
    targetInstance = ModelXbrl.create(modelXbrl.modelManager,
                                      newDocumentType=Type.INSTANCE,
                                      url=targetUrl,
                                      schemaRefs=targetDocumentSchemaRefs,
                                      isEntry=True,
                                      discover=False) # don't attempt to load DTS
    if baseXmlLang:
        targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
        if defaultXmlLang is None:
            defaultXmlLang = baseXmlLang # allows facts/footnotes to override baseXmlLang
    ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults 
    # roleRef and arcroleRef (of each inline document)
    for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
        for roleRefElt in sourceRefs.values():
            addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
                     attributes=roleRefElt.items())
    
    # contexts
    for context in sorted(modelXbrl.contexts.values(), key=lambda c: c.objectIndex): # contexts may come from multiple IXDS files
        ignore = targetInstance.createContext(context.entityIdentifier[0],
                                               context.entityIdentifier[1],
                                               'instant' if context.isInstantPeriod else
                                               'duration' if context.isStartEndPeriod
                                               else 'forever',
                                               context.startDatetime,
                                               context.endDatetime,
                                               None,
                                               context.qnameDims, [], [],
                                               id=context.id)
    for unit in sorted(modelXbrl.units.values(), key=lambda u: u.objectIndex): # units may come from multiple IXDS files
        measures = unit.measures
        ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id)

    modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
    newFactForOldObjId = {}
    def createFacts(facts, parent):
        for fact in facts:
            if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior
                attrs = {"contextRef": fact.contextID}
                if fact.id:
                    attrs["id"] = fact.id
                if fact.isNumeric:
                    attrs["unitRef"] = fact.unitID
                    if fact.get("decimals"):
                        attrs["decimals"] = fact.get("decimals")
                    if fact.get("precision"):
                        attrs["precision"] = fact.get("precision")
                if fact.isNil:
                    attrs[XbrlConst.qnXsiNil] = "true"
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.textValue
                    if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default
                        xmlLang = fact.xmlLang
                        if xmlLang is not None and xmlLang != defaultXmlLang:
                            attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang
                newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
                # if fact.isFraction, create numerator and denominator
                newFactForOldObjId[fact.objectIndex] = newFact
                if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
                    # check for img and other filing references so that referenced files are included in the zip.
                    for xmltext in [text] + CDATApattern.findall(text):
                        try:
                            for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter():
                                addLocallyReferencedFile(elt, filingFiles)
                        except (XMLSyntaxError, UnicodeDecodeError):
                            pass  # TODO: Why ignore UnicodeDecodeError?
            elif fact.isTuple:
                newTuple = targetInstance.createFact(fact.qname, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newTuple
                createFacts(fact.modelTupleFacts, newTuple)
                
    createFacts(modelXbrl.facts, None)
    modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships"))
    HREF = "{http://www.w3.org/1999/xlink}href"
    footnoteLinks = defaultdict(list)
    footnoteIdCount = {}
    for linkKey, linkPrototypes in modelXbrl.baseSets.items():
        arcrole, linkrole, linkqname, arcqname = linkKey
        if (linkrole and linkqname and arcqname and  # fully specified roles
            arcrole != "XBRL-footnotes" and
            any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
            for linkPrototype in linkPrototypes:
                if linkPrototype not in footnoteLinks[linkrole]:
                    footnoteLinks[linkrole].append(linkPrototype)
    for linkrole in sorted(footnoteLinks.keys()):
        for linkPrototype in footnoteLinks[linkrole]:
            newLink = addChild(targetInstance.modelDocument.xmlRootElement, 
                               linkPrototype.qname, 
                               attributes=linkPrototype.attributes)
            for linkChild in linkPrototype:
                attributes = linkChild.attributes
                if isinstance(linkChild, LocPrototype):
                    if HREF not in linkChild.attributes:
                        linkChild.attributes[HREF] = \
                        "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
                    addChild(newLink, linkChild.qname, 
                             attributes=attributes)
                elif isinstance(linkChild, ArcPrototype):
                    addChild(newLink, linkChild.qname, attributes=attributes)
                elif isinstance(linkChild, ModelInlineFootnote):
                    idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
                    if idUseCount > 1: # if footnote with id in other links bump the id number
                        attributes = linkChild.attributes.copy()
                        attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
                    footnoteIdCount[linkChild.footnoteID] = idUseCount
                    newChild = addChild(newLink, linkChild.qname, 
                                        attributes=attributes)
                    xmlLang = linkChild.xmlLang
                    if xmlLang is not None and xmlLang != defaultXmlLang: # default
                        newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang)
                    copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True)

                    if filingFiles and linkChild.textValue:
                        footnoteHtml = XML("<body/>")
                        copyIxFootnoteHtml(linkChild, footnoteHtml)
                        for elt in footnoteHtml.iter():
                            addLocallyReferencedFile(elt,filingFiles)
    return targetInstance
Exemple #13
0
    def runFromExcel(self, options):
        #testGenFileName = options.excelfilename
        testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls"
        testGenDir = os.path.dirname(testGenFileName)
        schemaDir = os.path.dirname(testGenDir) + os.sep + "schema"
        timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
        if options.testfiledate:
            today = options.testfiledate
        else:
            today = XmlUtil.dateunionValue(datetime.date.today())
        startedAt = time.time()
        
        LogHandler(self) # start logger

        self.logMessages = []
        logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt'

        modelTestcases = ModelXbrl.create(self.modelManager, url=testGenFileName, isEntry=True)
        testcaseIndexBook = xlrd.open_workbook(testGenFileName)
        testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
        self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow))
        
        # start index file
        indexFiles = [testGenDir + os.sep + 'creation-testcases-index.xml',
                      testGenDir + os.sep + 'consumption-testcases-index.xml']
        indexDocs = []
        testcasesElements = []
        for purpose in ("Creation","Consumption"):
            file = io.StringIO(
                #'<?xml version="1.0" encoding="UTF-8"?>'
                '<!-- XBRL Versioning 1.0 {0} Tests -->'
                '<!-- Copyright 2011 XBRL International.  All Rights Reserved. -->'
                '<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
                '<testcases name="XBRL Versioning 1.0 {0} Tests" '
                ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
                ' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
                '</testcases>'.format(purpose, today)
                )
            doc = etree.parse(file)
            file.close()
            indexDocs.append(doc)
            testcasesElements.append(doc.getroot())
        priorTestcasesDir = None
        testcaseFiles = None
        testcaseDocs = None
        for iRow in range(1, testcaseIndexSheet.nrows):
            try:
                row = testcaseIndexSheet.row(iRow)
                if (row[0].ctype == xlrd.XL_CELL_EMPTY or # must have directory
                    row[1].ctype == xlrd.XL_CELL_EMPTY or # from
                    row[2].ctype == xlrd.XL_CELL_EMPTY):  # to
                    continue
                testDir = row[0].value
                uriFrom = row[1].value
                uriTo = row[2].value
                overrideReport = row[3].value
                description = row[4].value
                if description is None or len(description) == 0:
                    continue # test not ready to run
                assignment = row[5].value
                expectedEvents = row[6].value # comma space separated if multiple
                note = row[7].value
                useCase = row[8].value
                base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep
                self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
                if uriFrom and uriTo and assignment.lower() not in ("n.a.", "error") and expectedEvents != "N.A.":
                    modelDTSfrom = modelDTSto = None
                    for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)):
                        if ',' not in URIs:
                            modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base)
                        else:
                            modelDTS = ModelXbrl.create(self.modelManager, 
                                         ModelDocument.Type.DTSENTRIES,
                                         self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts", 
                                                                    base),
                                         isEntry=True)
                            DTSdoc = modelDTS.modelDocument
                            DTSdoc.inDTS = True
                            for uri in URIs.split(','):
                                doc = ModelDocument.load(modelDTS, uri.strip(), base=base)
                                if doc is not None:
                                    DTSdoc.referencesDocument[doc] = "import"  #fake import
                                    doc.inDTS = True
                        if isFrom: modelDTSfrom = modelDTS
                        else: modelDTSto = modelDTS
                    if modelDTSfrom is not None and modelDTSto is not None:
                        # generate differences report
                        reportUri = uriFrom.partition(',')[0]  # first file
                        reportDir = os.path.dirname(reportUri)
                        if reportDir: reportDir += os.sep
                        reportName = os.path.basename(reportUri).replace("from.xsd","report.xml")
                        reportFile = reportDir + "out" + os.sep + reportName
                        #reportFile = reportDir + "report" + os.sep + reportName
                        reportFullPath = self.webCache.normalizeUrl(
                                            reportFile, 
                                            base)
                        testcasesDir = os.path.dirname(os.path.dirname(reportFullPath))
                        if testcasesDir != priorTestcasesDir:
                            # close prior report
                            if priorTestcasesDir:
                                for i,testcaseFile in enumerate(testcaseFiles):
                                    with open(testcaseFile, "w", encoding="utf-8") as fh:
                                        XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
                            testcaseName = os.path.basename(testcasesDir)
                            testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml",
                                             testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"]
                            for i,testcaseFile in enumerate(testcaseFiles):
                                etree.SubElement(testcasesElements[i], "testcase", 
                                                 attrib={"uri": 
                                                         testcaseFile[len(testGenDir)+1:].replace("\\","/")} )
                            
                            # start testcase file
                            testcaseDocs = []
                            testcaseElements = []
                            testcaseNumber = testcaseName[0:4]
                            if testcaseNumber.isnumeric():
                                testcaseNumberElement = "<number>{0}</number>".format(testcaseNumber)
                                testcaseName = testcaseName[5:]
                            else:
                                testcaseNumberElement = ""
                            testDirSegments = testDir.split('/')
                            if len(testDirSegments) >= 2 and '-' in testDirSegments[1]:
                                testedModule = testDirSegments[1][testDirSegments[1].index('-') + 1:]
                            else:
                                testedModule = ''
                            for purpose in ("Creation","Consumption"):
                                file = io.StringIO(
                                    #'<?xml version="1.0" encoding="UTF-8"?>'
                                    '<!-- Copyright 2011 XBRL International.  All Rights Reserved. -->'
                                    '<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
                                    '<testcase '
                                    ' xmlns="http://xbrl.org/2008/conformance"'
                                    ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
                                    ' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
                                    '<creator>'
                                    '<name>Roland Hommes</name>'
                                    '<email>[email protected]</email>'
                                    '</creator>'
                                    '{0}'
                                    '<name>{1}</name>'
                                    # '<description>{0}</description>'
                                    '<reference>'
                                    '{2}'
                                    '{3}'
                                    '</reference>'
                                    '</testcase>'.format(testcaseNumberElement,
                                                         testcaseName,
                                                         '<name>{0}</name>'.format(testedModule) if testedModule else '',
                                                         '<id>{0}</id>'.format(useCase) if useCase else '')
                                    
                                    )
                                doc = etree.parse(file)
                                file.close()
                                testcaseDocs.append(doc)
                                testcaseElements.append(doc.getroot())
                            priorTestcasesDir = testcasesDir
                            variationSeq = 1
                        try:
                            os.makedirs(os.path.dirname(reportFullPath))
                        except WindowsError:
                            pass # dir already exists
                        modelVersReport = ModelVersReport.ModelVersReport(modelTestcases)
                        modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto, 
                                                  assignment=assignment,
                                                  schemaDir=schemaDir)
                        
                        # check for expected elements
                        if expectedEvents:
                            for expectedEvent in expectedEvents.split(","):
                                if expectedEvent not in ("No change", "N.A."):
                                    prefix, sep, localName = expectedEvent.partition(':')
                                    if sep and len(modelVersReport.xmlDocument.findall(
                                                        '//{{{0}}}{1}'.format(
                                                            XbrlConst.verPrefixNS.get(prefix),
                                                            localName))) == 0:
                                        modelTestcases.warning("warning",
                                            "Generated test case %(reportName)s missing expected event %(event)s",
                                            reportName=reportName, 
                                            event=expectedEvent)
                        
                        modelVersReport.close()
                        uriFromParts = uriFrom.split('_')
                        if len(uriFromParts) >= 2:
                            variationId = uriFromParts[1]
                        else:
                            variationId = "_{0:02n}".format(variationSeq)
                        for i,testcaseElt in enumerate(testcaseElements):
                            variationElement = etree.SubElement(testcaseElt, "{http://xbrl.org/2008/conformance}variation", 
                                                                attrib={"id": variationId})
                            nameElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
                            nameElement.text = description
                            ''' (removed per RH 2011/10/04
                            if note:
                                paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
                                paramElement.text = "Note: " + note
                            if useCase:
                                paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference")
                                paramElement.set("specification", "versioning-requirements")
                                paramElement.set("useCase", useCase)
                            '''
                            dataElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}data")
                            if i == 0:  # result is report
                                if expectedEvents:
                                    paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter",
                                                                    attrib={"name":"expectedEvent",
                                                                            "value":expectedEvents.replace(',',' ')},
                                                                    nsmap={"conf":"http://xbrl.org/2008/conformance",
                                                                           None:""})
                                if assignment:
                                    paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter",
                                                                    attrib={"name":"assignment",
                                                                            "value":assignment},
                                                                    nsmap={"conf":"http://xbrl.org/2008/conformance",
                                                                           None:""})
                            for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")):
                                for schemaURI in schemaURIs.split(","): 
                                    schemaElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}schema")
                                    schemaElement.set("dts",dtsAttr)
                                    if i == 0:
                                        schemaElement.set("readMeFirst","true")
                                    schemaElement.text=os.path.basename(schemaURI.strip())
                            resultElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}result")
                            reportElement = etree.SubElement(resultElement if i == 0 else dataElement, 
                                             "{http://xbrl.org/2008/conformance}versioningReport")
                            if i == 1:
                                reportElement.set("readMeFirst","true")
                            reportElement.text = "report/" + reportName
                        variationSeq += 1
            except Exception as err:
                modelTestcases.error("exception",
                    _("Exception: %(error)s, Excel row: %(excelRow)s"),
                    error=err,
                    excelRow=iRow, 
                    exc_info=True)
        
        # add tests-error-code index files to consumption
        for testcaseFile in self.testcaseFiles(testGenDir + os.sep + "tests-error-code"):
            etree.SubElement(testcasesElements[1], "testcase", 
                             attrib={"uri": 
                             testcaseFile[len(testGenDir)+1:].replace("\\","/")} )

        with open(logMessagesFile, "w") as fh:
            fh.writelines(self.logMessages)

        if priorTestcasesDir:
            for i,testcaseFile in enumerate(testcaseFiles):
                with open(testcaseFile, "w", encoding="utf-8") as fh:
                    XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
        for i,indexFile in enumerate(indexFiles):
            with open(indexFile, "w", encoding="utf-8") as fh:
                XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
Exemple #14
0
def validate(val):
    formulaOptions = val.modelXbrl.modelManager.formulaOptions
    XPathParser.initializeParser(val)
    val.modelXbrl.modelManager.showStatus(_("Compiling formulae"))
    initialErrorCount = val.modelXbrl.logCountErr
    
    # global parameter names
    parameterQnames = set()
    instanceQnames = set()
    parameterDependencies = {}
    instanceDependencies = defaultdict(set)  # None-key entries are non-formula dependencies
    dependencyResolvedParameters = set()
    orderedParameters = []
    orderedInstances = []
    for paramQname, modelParameter in val.modelXbrl.qnameParameters.items():
        if isinstance(modelParameter, ModelParameter):
            modelParameter.compile()
            parameterDependencies[paramQname] = modelParameter.variableRefs()
            parameterQnames.add(paramQname)
            if isinstance(modelParameter, ModelInstance):
                instanceQnames.add(paramQname)
            # duplicates checked on loading modelDocument
            
    #resolve dependencies
    resolvedAParameter = True
    while (resolvedAParameter):
        resolvedAParameter = False
        for paramQname in parameterQnames:
            if paramQname not in dependencyResolvedParameters and \
               len(parameterDependencies[paramQname] - dependencyResolvedParameters) == 0:
                dependencyResolvedParameters.add(paramQname)
                orderedParameters.append(paramQname)
                resolvedAParameter = True
    # anything unresolved?
    for paramQname in parameterQnames:
        if paramQname not in dependencyResolvedParameters:
            circularOrUndefDependencies = parameterDependencies[paramQname] - dependencyResolvedParameters
            undefinedVars = circularOrUndefDependencies - parameterQnames 
            paramsCircularDep = circularOrUndefDependencies - undefinedVars
            if len(undefinedVars) > 0:
                val.modelXbrl.error(
                    _("Undefined dependencies in parameter {0}, to names {1}").format(
                          paramQname, ", ".join((str(v) for v in undefinedVars))), 
                    "err", "xbrlve:unresolvedDependency")
            if len(paramsCircularDep) > 0:
                val.modelXbrl.error(
                    _("Cyclic dependencies in parameter {0}, to names {1}").format(
                          paramQname, ", ".join((str(d) for d in paramsCircularDep)) ), 
                    "err", "xbrlve:parameterCyclicDependencies")
            
    for custFnSig in val.modelXbrl.modelCustomFunctionSignatures.values():
        custFnQname = custFnSig.qname
        if custFnQname.namespaceURI == "XbrlConst.xfi":
            val.modelXbrl.error(
                _("Custom function {0} has namespace reserved for functions in the function registry {1}").format(
                      str(custFnQname), custFnQname.namespaceURI ), 
                "err", "xbrlve:noProhibitedNamespaceForCustomFunction")
        # any custom function implementations?
        for modelRel in val.modelXbrl.relationshipSet(XbrlConst.functionImplementation).fromModelObject(custFnSig):
            custFnImpl = modelRel.toModelObject
            custFnSig.customFunctionImplementation = custFnImpl
            if len(custFnImpl.inputNames) != len(custFnSig.inputTypes):
                val.modelXbrl.error(
                    _("Custom function {0} signature has {1} parameters but implementation has {2}, must be matching").format(
                          str(custFnQname), len(custFnSig.inputTypes), len(custFnImpl.inputNames) ), 
                    "err", "xbrlcfie:inputMismatch")
        
    for custFnImpl in val.modelXbrl.modelCustomFunctionImplementations:
        if not val.modelXbrl.relationshipSet(XbrlConst.functionImplementation).toModelObject(custFnImpl):
            val.modelXbrl.error(
                _("Custom function implementation {0} has no relationship from any custom function signature").format(
                      custFnImpl.xlinkLabel), 
                "err", "xbrlcfie:missingCFIRelationship")
        custFnImpl.compile()
            
    # xpathContext is needed for filter setup for expressions such as aspect cover filter
    # determine parameter values
    xpathContext = XPathContext.create(val.modelXbrl)
    for paramQname in orderedParameters:
        if not isinstance(modelParameter, ModelInstance):
            modelParameter = val.modelXbrl.qnameParameters[paramQname]
            asType = modelParameter.asType
            asLocalName = asType.localName if asType else "string"
            try:
                if val.parameters and paramQname in val.parameters:
                    paramDataType, paramValue = val.parameters[paramQname]
                    typeLocalName = paramDataType.localName if paramDataType else "string"
                    value = FunctionXs.call(xpathContext, None, typeLocalName, [paramValue])
                    result = FunctionXs.call(xpathContext, None, asLocalName, [value])
                    if formulaOptions.traceParameterInputValue:
                        val.modelXbrl.error( _("Parameter {0} input {1}").format( paramQname, result),
                            "info", "formula:trace")
                else:
                    result = modelParameter.evaluate(xpathContext, asType)
                    if formulaOptions.traceParameterExpressionResult:
                        val.modelXbrl.error( _("Parameter {0} result {1}").format( paramQname, result),
                            "info", "formula:trace")
                xpathContext.inScopeVars[paramQname] = result    # make visible to subsequent parameter expression 
            except XPathContext.XPathException as err:
                val.modelXbrl.error( _("Parameter \n{0} \nException: \n{1}").format( paramQname, err.message),
                    "err", "xbrlve:parameterTypeMismatch" if err.code == "err:FORG0001" else err.code)

    produceOutputXbrlInstance = False
    instanceProducingVariableSets = defaultdict(list)
        
    for modelVariableSet in val.modelXbrl.modelVariableSets:
        varSetInstanceDependencies = set()
        if isinstance(modelVariableSet, ModelFormula):
            instanceQname = None
            for modelRel in val.modelXbrl.relationshipSet(XbrlConst.formulaInstance).fromModelObject(modelVariableSet):
                instance = modelRel.toModelObject
                if isinstance(instance, ModelInstance):
                    if instanceQname is None:
                        instanceQname = instance.qname
                    else:
                        val.modelXbrl.error(
                            _("Multiple output instances for formula {0}, to names {1}, {2}").format(
                                  modelVariableSet.xlinkLabel, instanceQname, instance.qname ), 
                            "info", "arelle:multipleOutputInstances")
            if instanceQname is None: 
                instanceQname = XbrlConst.qnStandardOutputInstance
                instanceQnames.add(instanceQname)
            modelVariableSet.outputInstanceQname = instanceQname
            if val.validateSBRNL:
                val.modelXbrl.error(
                    _("Formula linkbase {0} formula:formula {1} is not allowed").format(
                        os.path.basename(modelVariableSet.modelDocument.uri), modelVariableSet.xlinkLabel), 
                    "err", "SBR.NL.2.3.9.03")
        else:
            instanceQname = None
            modelVariableSet.countSatisfied = 0
            modelVariableSet.countNotSatisfied = 0
            checkValidationMessages(val, modelVariableSet)
        instanceProducingVariableSets[instanceQname].append(modelVariableSet)
        modelVariableSet.outputInstanceQname = instanceQname
        if modelVariableSet.aspectModel not in ("non-dimensional", "dimensional"):
            val.modelXbrl.error(
                _("Variable set {0}, aspect model {1} not recognized").format(
                      modelVariableSet.xlinkLabel, modelVariableSet.aspectModel), 
                "err", "xbrlve:unknownAspectModel")
        modelVariableSet.compile()
        modelVariableSet.hasConsistencyAssertion = False
            
        #determine dependencies within variable sets
        nameVariables = {}
        qnameRels = {}
        definedNamesSet = set()
        for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSet).fromModelObject(modelVariableSet):
            varqname = modelRel.variableQname
            if varqname:
                qnameRels[varqname] = modelRel
                toVariable = modelRel.toModelObject
                if varqname not in definedNamesSet:
                    definedNamesSet.add(varqname)
                if varqname not in nameVariables:
                    nameVariables[varqname] = toVariable
                elif nameVariables[varqname] != toVariable:
                    val.modelXbrl.error(
                        _("Multiple variables named {1} in variable set {0}").format(
                              modelVariableSet.xlinkLabel, varqname ), 
                        "err", "xbrlve:duplicateVariableNames")
                fromInstanceQnames = None
                for instRel in val.modelXbrl.relationshipSet(XbrlConst.instanceVariable).toModelObject(toVariable):
                    fromInstance = instRel.fromModelObject
                    if isinstance(fromInstance, ModelInstance):
                        fromInstanceQname = fromInstance.qname
                        varSetInstanceDependencies.add(fromInstanceQname)
                        instanceDependencies[instanceQname].add(fromInstanceQname)
                        if fromInstanceQnames is None: fromInstanceQnames = set()
                        fromInstanceQnames.add(fromInstanceQname)
                if fromInstanceQnames is None:
                    varSetInstanceDependencies.add(XbrlConst.qnStandardInputInstance)
                    if instanceQname: instanceDependencies[instanceQname].add(XbrlConst.qnStandardInputInstance)
                toVariable.fromInstanceQnames = fromInstanceQnames
            else:
                val.modelXbrl.error(
                    _("Variables name {1} cannot be determined on arc from {0}").format(
                          modelVariableSet.xlinkLabel, modelRel.variablename ), 
                    "err", "xbrlve:variableNameResolutionFailure")
        definedNamesSet |= parameterQnames
                
        variableDependencies = {}
        for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSet).fromModelObject(modelVariableSet):
            variable = modelRel.toModelObject
            if isinstance(variable, (ModelParameter,ModelVariable)):    # ignore anything not parameter or variable
                varqname = modelRel.variableQname
                depVars = variable.variableRefs()
                variableDependencies[varqname] = depVars
                if len(depVars) > 0 and formulaOptions.traceVariablesDependencies:
                    val.modelXbrl.error(_("Variable set {0}, variable {1}, dependences {2}").format(
                                  modelVariableSet.xlinkLabel, varqname, depVars), 
                                  "info", "formula:trace") 
                definedNamesSet.add(varqname)
                # check for fallback value variable references
                if isinstance(variable, ModelFactVariable):
                    for depVar in XPathParser.variableReferencesSet(variable.fallbackValueProg, variable.element):
                        if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable):
                            val.modelXbrl.error(_("Variable set {0} fallbackValue '{1}' cannot refer to variable {2}").format(
                                          modelVariableSet.xlinkLabel, variable.fallbackValue, depVar),
                                          "err", "xbrlve:factVariableReferenceNotAllowed") 
                    # check for covering aspect not in variable set aspect model
                    checkFilterAspectModel(val, modelVariableSet, variable.filterRelationships, xpathContext)

        orderedNameSet = set()
        orderedNameList = []
        orderedAVariable = True
        while (orderedAVariable):
            orderedAVariable = False
            for varqname, depVars in variableDependencies.items():
                if varqname not in orderedNameSet and len(depVars - parameterQnames - orderedNameSet) == 0:
                    orderedNameList.append(varqname)
                    orderedNameSet.add(varqname)
                    orderedAVariable = True
                if varqname in instanceQnames:
                    varSetInstanceDependencies.add(varqname)
                    instanceDependencies[instanceQname].add(varqname)
                elif isinstance(nameVariables.get(varqname), ModelInstance):
                    instqname = nameVariables[varqname].qname
                    varSetInstanceDependencies.add(instqname)
                    instanceDependencies[instanceQname].add(instqname)
                    
        # anything unresolved?
        for varqname, depVars in variableDependencies.items():
            if varqname not in orderedNameSet:
                circularOrUndefVars = depVars - parameterQnames - orderedNameSet
                undefinedVars = circularOrUndefVars - definedNamesSet 
                varsCircularDep = circularOrUndefVars - undefinedVars
                if len(undefinedVars) > 0:
                    val.modelXbrl.error(
                        _("Undefined variable dependencies in variable st {0}, from variable {1} to {2}").format(
                              modelVariableSet.xlinkLabel, varqname, undefinedVars), 
                        "err", "xbrlve:unresolvedDependency")
                if len(varsCircularDep) > 0:
                    val.modelXbrl.error(
                        _("Cyclic dependencies in variable set {0}, from variable {1} to {2}").format(
                              modelVariableSet.xlinkLabel, varqname, varsCircularDep ), 
                        "err", "xbrlve:cyclicDependencies")
                    
        # check unresolved variable set dependencies
        for varSetDepVarQname in modelVariableSet.variableRefs():
            if varSetDepVarQname not in orderedNameSet and varSetDepVarQname not in parameterQnames:
                val.modelXbrl.error(
                    _("Undefined variable dependency in variable set {0}, {1}").format(
                          modelVariableSet.xlinkLabel, varSetDepVarQname), 
                    "err", "xbrlve:unresolvedDependency")
            if varSetDepVarQname in instanceQnames:
                varSetInstanceDependencies.add(varSetDepVarQname)
                instanceDependencies[instanceQname].add(varSetDepVarQname)
            elif isinstance(nameVariables.get(varSetDepVarQname), ModelInstance):
                instqname = nameVariables[varSetDepVarQname].qname
                varSetInstanceDependencies.add(instqname)
                instanceDependencies[instanceQname].add(instqname)
        
        if formulaOptions.traceVariablesOrder:
            val.modelXbrl.error(_("Variable set {0}, variables order: {1}").format(
                          modelVariableSet.xlinkLabel, orderedNameList), "info", "formula:trace") 
        
        if (formulaOptions.traceVariablesDependencies and len(varSetInstanceDependencies) > 0 and
            varSetInstanceDependencies != {XbrlConst.qnStandardInputInstance}):
            val.modelXbrl.error(_("Variable set {0}, instance dependences {1}").format(
                          modelVariableSet.xlinkLabel, varSetInstanceDependencies), 
                          "info", "formula:trace") 
            
        modelVariableSet.orderedVariableRelationships = []
        for varqname in orderedNameList:
            if varqname in qnameRels:
                modelVariableSet.orderedVariableRelationships.append(qnameRels[varqname])
                
        # check existence assertion variable dependencies
        if isinstance(modelVariableSet, ModelExistenceAssertion):
            for depVar in modelVariableSet.variableRefs():
                if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable):
                    val.modelXbrl.error(_("Existence Assertion {0}, cannot refer to variable {1}").format(
                                  modelVariableSet.xlinkLabel, depVar),
                                  "err", "xbrleae:variableReferenceNotAllowed") 
                    
        # check messages variable dependencies
        checkValidationMessageVariables(val, modelVariableSet, qnameRels)
                        
        # check preconditions
        modelVariableSet.preconditions = []
        for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSetPrecondition).fromModelObject(modelVariableSet):
            precondition = modelRel.toModelObject
            if isinstance(precondition, ModelPrecondition):
                modelVariableSet.preconditions.append(precondition)
                
        # check for variable sets referencing fact or general variables
        for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSetFilter).fromModelObject(modelVariableSet):
            varSetFilter = modelRel.toModelObject
            if modelRel.isCovered:
                val.modelXbrl.error(_("Variable set {0}, filter {1}, cannot be covered").format(
                              modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel),
                              "wrn", "arelle:variableSetFilterCovered") 
                modelRel._isCovered = False # block group filter from being able to covere
            for depVar in varSetFilter.variableRefs():
                if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable):
                    val.modelXbrl.error(_("Variable set {0}, filter {1}, cannot refer to variable {2}").format(
                                  modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel, depVar),
                                  "err", "xbrlve:factVariableReferenceNotAllowed") 
                    
        # check aspects of formula
        if isinstance(modelVariableSet, ModelFormula):
            checkFormulaRules(val, modelVariableSet, nameVariables)
            
    # determine instance dependency order
    orderedInstancesSet = set()
    stdInpInst = {XbrlConst.qnStandardInputInstance}
    orderedInstancesList = []
    orderedAnInstance = True
    while (orderedAnInstance):
        orderedAnInstance = False
        for instqname, depInsts in instanceDependencies.items():
            if instqname and instqname not in orderedInstancesSet and len(depInsts - stdInpInst - orderedInstancesSet) == 0:
                orderedInstancesList.append(instqname)
                orderedInstancesSet.add(instqname)
                orderedAnInstance = True
    orderedInstancesList.append(None)  # assertions come after all formulas that produce outputs

    # anything unresolved?
    for instqname, depInsts in instanceDependencies.items():
        if instqname not in orderedInstancesSet:
            # can also be satisfied from an input DTS
            missingDependentInstances = depInsts - stdInpInst
            if val.parameters: missingDependentInstances -= val.parameters.keys() 
            if instqname:
                if missingDependentInstances:
                    val.modelXbrl.error(
                        _("Cyclic dependencies of instance {0} produced by a formula, with variables consuming instances {1}").format(
                              instqname, missingDependentInstances ), 
                        "err", "xbrlvarinste:instanceVariableRecursionCycle")
                elif instqname == XbrlConst.qnStandardOutputInstance:
                    orderedInstancesSet.add(instqname)
                    orderedInstancesList.append(instqname) # standard output formula, all input dependencies in parameters
            ''' future check?  if instance has no external input or producing formula
            else:
                val.modelXbrl.error(
                    _("Unresolved dependencies of an assertion's variables on instances {0}").format(
                          depInsts - stdInpInst ), 
                    "err", "xbrlvarinste:instanceVariableRecursionCycle")
            '''

    if formulaOptions.traceVariablesOrder and len(orderedInstancesList) > 1:
        val.modelXbrl.error(_("Variable instances processing order: {0}").format(
                            orderedInstancesList), "info", "formula:trace") 

    # linked consistency assertions
    for modelRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).modelRelationships:
        if modelRel.fromModelObject and modelRel.toModelObject and isinstance(modelRel.toModelObject,ModelFormula):
            consisAsser = modelRel.fromModelObject
            consisAsser.countSatisfied = 0
            consisAsser.countNotSatisfied = 0
            if consisAsser.hasProportionalAcceptanceRadius and consisAsser.hasAbsoluteAcceptanceRadius:
                val.modelXbrl.error( _("Consistency assertion {0} has both absolute and proportional acceptance radii").format( 
                     consisAsser.xlinkLabel),
                    "err", "xbrlcae:acceptanceRadiusConflict")
            consisAsser.orderedVariableRelationships = []
            for consisParamRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionParameter).fromModelObject(consisAsser):
                if isinstance(consisParamRel.toModelObject, ModelVariable):
                    val.modelXbrl.error( _("Consistency assertion {0} has relationship to a {1} {2}").format( 
                         consisAsser.xlinkLabel, consisParamRel.toModelObject.element.localName, consisParamRel.toModelObject.xlinkLabel),
                        "err", "xbrlcae:variablesNotAllowed")
                else:
                    consisAsser.orderedVariableRelationships.append(consisParamRel)
            consisAsser.compile()
            modelRel.toModelObject.hasConsistencyAssertion = True

    if initialErrorCount < val.modelXbrl.logCountErr:
        return  # don't try to execute
        

    # formula output instances    
    if instanceQnames:      
        schemaRefs = [val.modelXbrl.modelDocument.relativeUri(referencedDoc.uri)
                        for referencedDoc in val.modelXbrl.modelDocument.referencesDocument.keys()
                            if referencedDoc.type == ModelDocument.Type.SCHEMA]
        
    outputXbrlInstance = None
    for instanceQname in instanceQnames:
        if instanceQname == XbrlConst.qnStandardInputInstance:
            continue    # always present the standard way
        if val.parameters and instanceQname in val.parameters:
            namedInstance = val.parameters[instanceQname][1]
        else:   # empty intermediate instance 
            uri = val.modelXbrl.modelDocument.filepath[:-4] + "-output-XBRL-instance"
            if instanceQname != XbrlConst.qnStandardOutputInstance:
                uri = uri + "-" + instanceQname.localName
            uri = uri + ".xml"
            namedInstance = ModelXbrl.create(val.modelXbrl.modelManager, 
                                             newDocumentType=ModelDocument.Type.INSTANCE,
                                             url=uri,
                                             schemaRefs=schemaRefs,
                                             isEntry=True)
        xpathContext.inScopeVars[instanceQname] = namedInstance
        if instanceQname == XbrlConst.qnStandardOutputInstance:
            outputXbrlInstance = namedInstance
        
    # evaluate consistency assertions
    
    # evaluate variable sets not in consistency assertions
    for instanceQname in orderedInstancesList:
        for modelVariableSet in instanceProducingVariableSets[instanceQname]:
            # produce variable evaluations
            from arelle.FormulaEvaluator import evaluate
            try:
                evaluate(xpathContext, modelVariableSet)
            except XPathContext.XPathException as err:
                val.modelXbrl.error( _("Variable set \n{0} \nException: \n{1}").format( modelVariableSet, err.message),
                    "err", err.code)
            
    # log assertion result counts
    asserTests = {}
    for exisValAsser in val.modelXbrl.modelVariableSets:
        if isinstance(exisValAsser, ModelVariableSetAssertion):
            asserTests[exisValAsser.id] = (exisValAsser.countSatisfied, exisValAsser.countNotSatisfied)
            if formulaOptions.traceAssertionResultCounts:
                val.modelXbrl.error( _("{0} Assertion {1} evaluations : {2} satisfied, {3} not satisfied").format(
                    "Existence" if isinstance(exisValAsser, ModelExistenceAssertion) else "Value", 
                    exisValAsser.id, exisValAsser.countSatisfied, exisValAsser.countNotSatisfied),
                    "info", "formula:trace")

    for modelRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).modelRelationships:
        if modelRel.fromModelObject and modelRel.toModelObject and isinstance(modelRel.toModelObject,ModelFormula):
            consisAsser = modelRel.fromModelObject
            asserTests[consisAsser.id] = (consisAsser.countSatisfied, consisAsser.countNotSatisfied)
            if formulaOptions.traceAssertionResultCounts:
                val.modelXbrl.error( _("Consistency Assertion {0} evaluations : {1} satisfied, {2} not satisfied").format(
                    consisAsser.id, consisAsser.countSatisfied, consisAsser.countNotSatisfied),
                    "info", "formula:trace")
            
    if asserTests:
        val.modelXbrl.error( _("Assertion results {0}").format(asserTests),
            "asrtNoLog", asserTests)

    # display output instance
    if outputXbrlInstance:
        if val.modelXbrl.formulaOutputInstance:
            # close prior instance, usually closed by caller to validate as it may affect UI on different thread
            val.modelXbrl.formulaOutputInstance.close()
        val.modelXbrl.formulaOutputInstance = outputXbrlInstance
Exemple #15
0
 def validateRssFeed(self):
     self.modelXbrl.info("info", "RSS Feed", modelDocument=self.modelXbrl)
     from arelle.FileSource import openFileSource
     reloadCache = getattr(self.modelXbrl, "reloadCache", False)
     for rssItem in self.modelXbrl.modelDocument.rssItems:
         if getattr(rssItem, "skipRssItem", False):
             self.modelXbrl.info(
                 "info",
                 _("skipping RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"
                   ),
                 modelObject=rssItem,
                 accessionNumber=rssItem.accessionNumber,
                 formType=rssItem.formType,
                 companyName=rssItem.companyName,
                 period=rssItem.period)
             continue
         self.modelXbrl.info(
             "info",
             _("RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"
               ),
             modelObject=rssItem,
             accessionNumber=rssItem.accessionNumber,
             formType=rssItem.formType,
             companyName=rssItem.companyName,
             period=rssItem.period)
         modelXbrl = None
         try:
             modelXbrl = ModelXbrl.load(
                 self.modelXbrl.modelManager,
                 openFileSource(rssItem.zippedUrl,
                                self.modelXbrl.modelManager.cntlr,
                                reloadCache=reloadCache),
                 _("validating"),
                 rssItem=rssItem)
             for pluginXbrlMethod in pluginClassMethods(
                     "RssItem.Xbrl.Loaded"):
                 pluginXbrlMethod(modelXbrl, {}, rssItem)
             if getattr(rssItem, "doNotProcessRSSitem",
                        False) or modelXbrl.modelDocument is None:
                 modelXbrl.close()
                 continue  # skip entry based on processing criteria
             self.instValidator.validate(
                 modelXbrl,
                 self.modelXbrl.modelManager.formulaOptions.typedParameters(
                 ))
             self.instValidator.close()
             rssItem.setResults(modelXbrl)
             self.modelXbrl.modelManager.viewModelObject(
                 self.modelXbrl, rssItem.objectId())
             for pluginXbrlMethod in pluginClassMethods("Validate.RssItem"):
                 pluginXbrlMethod(self, modelXbrl, rssItem)
             modelXbrl.close()
         except Exception as err:
             self.modelXbrl.error(
                 "exception",
                 _("RSS item validation exception: %(error)s, instance: %(instance)s"
                   ),
                 modelXbrl=(self.modelXbrl, modelXbrl),
                 instance=rssItem.zippedUrl,
                 error=err,
                 exc_info=True)
             try:
                 self.instValidator.close()
                 if modelXbrl is not None:
                     modelXbrl.close()
             except Exception as err:
                 pass
         del modelXbrl  # completely dereference
Exemple #16
0
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None, *args, **kwargs):
    targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
    def addLocallyReferencedFile(elt,filingFiles):
        if elt.tag in ("a", "img"):
            for attrTag, attrValue in elt.items():
                if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrValue):
                    attrValue = attrValue.partition('#')[0] # remove anchor
                    if attrValue: # ignore anchor references to base document
                        attrValue = os.path.normpath(attrValue) # change url path separators to host separators
                        file = os.path.join(sourceDir,attrValue)
                        if modelXbrl.fileSource.isInArchive(file, checkExistence=True) or os.path.exists(file):
                            filingFiles.add(file)
    targetUrlParts = targetUrl.rpartition(".")
    targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
    modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
    rootElt = modelXbrl.modelDocument.xmlRootElement
    # take baseXmlLang from <html> or <base>
    baseXmlLang = rootElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang")
    for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/1999/xhtml}body"):
        baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang") or baseXmlLang
    targetInstance = ModelXbrl.create(modelXbrl.modelManager, 
                                      newDocumentType=Type.INSTANCE,
                                      url=targetUrl,
                                      schemaRefs=targetDocumentSchemaRefs,
                                      isEntry=True,
                                      discover=False) # don't attempt to load DTS
    if baseXmlLang:
        targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
    ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults 
    # roleRef and arcroleRef (of each inline document)
    for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
        for roleRefElt in sourceRefs.values():
            addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, 
                     attributes=roleRefElt.items())
    
    # contexts
    for context in sorted(modelXbrl.contexts.values(), key=lambda c: elementChildSequence(c)):
        ignore = targetInstance.createContext(context.entityIdentifier[0],
                                               context.entityIdentifier[1],
                                               'instant' if context.isInstantPeriod else
                                               'duration' if context.isStartEndPeriod
                                               else 'forever',
                                               context.startDatetime,
                                               context.endDatetime,
                                               None, 
                                               context.qnameDims, [], [],
                                               id=context.id)
    for unit in modelXbrl.units.values():
        measures = unit.measures
        ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id)

    modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
    newFactForOldObjId = {}
    def createFacts(facts, parent):
        for fact in facts:
            if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior
                attrs = {"contextRef": fact.contextID}
                if fact.id:
                    attrs["id"] = fact.id
                if fact.isNumeric:
                    attrs["unitRef"] = fact.unitID
                    if fact.get("decimals"):
                        attrs["decimals"] = fact.get("decimals")
                    if fact.get("precision"):
                        attrs["precision"] = fact.get("precision")
                if fact.isNil:
                    attrs[XbrlConst.qnXsiNil] = "true"
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.textValue
                    if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default
                        xmlLang = fact.xmlLang
                        if xmlLang is not None and xmlLang != baseXmlLang:
                            attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang
                newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
                # if fact.isFraction, create numerator and denominator
                newFactForOldObjId[fact.objectIndex] = newFact
                if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
                    # check for img and other filing references so that referenced files are included in the zip.
                    for xmltext in [text] + CDATApattern.findall(text):
                        try:
                            for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter():
                                addLocallyReferencedFile(elt, filingFiles)
                        except (XMLSyntaxError, UnicodeDecodeError):
                            pass  # TODO: Why ignore UnicodeDecodeError?
            elif fact.isTuple:
                newTuple = targetInstance.createFact(fact.qname, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newTuple
                createFacts(fact.modelTupleFacts, newTuple)
                
    createFacts(modelXbrl.facts, None)
    modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships"))
    HREF = "{http://www.w3.org/1999/xlink}href"
    footnoteLinks = defaultdict(list)
    footnoteIdCount = {}
    for linkKey, linkPrototypes in modelXbrl.baseSets.items():
        arcrole, linkrole, linkqname, arcqname = linkKey
        if (linkrole and linkqname and arcqname and  # fully specified roles
            arcrole != "XBRL-footnotes" and
            any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
            for linkPrototype in linkPrototypes:
                if linkPrototype not in footnoteLinks[linkrole]:
                    footnoteLinks[linkrole].append(linkPrototype)
    for linkrole in sorted(footnoteLinks.keys()):
        for linkPrototype in footnoteLinks[linkrole]:
            newLink = addChild(targetInstance.modelDocument.xmlRootElement, 
                               linkPrototype.qname, 
                               attributes=linkPrototype.attributes)
            for linkChild in linkPrototype:
                attributes = linkChild.attributes
                if isinstance(linkChild, LocPrototype):
                    if HREF not in linkChild.attributes:
                        linkChild.attributes[HREF] = \
                        "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
                    addChild(newLink, linkChild.qname, 
                             attributes=attributes)
                elif isinstance(linkChild, ArcPrototype):
                    addChild(newLink, linkChild.qname, attributes=attributes)
                elif isinstance(linkChild, ModelInlineFootnote):
                    idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
                    if idUseCount > 1: # if footnote with id in other links bump the id number
                        attributes = linkChild.attributes.copy()
                        attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
                    footnoteIdCount[linkChild.footnoteID] = idUseCount
                    newChild = addChild(newLink, linkChild.qname, 
                                        attributes=attributes)
                    xmlLang = linkChild.xmlLang
                    if xmlLang is not None and xmlLang != baseXmlLang: # default
                        newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang)
                    copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True)

                    if filingFiles and linkChild.textValue:
                        footnoteHtml = XML("<body/>")
                        copyIxFootnoteHtml(linkChild, footnoteHtml)
                        for elt in footnoteHtml.iter():
                            addLocallyReferencedFile(elt,filingFiles)
    targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip)
    if getattr(modelXbrl, "isTestcaseVariation", False):
        modelXbrl.extractedInlineInstance = True # for validation comparison
    modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
Exemple #17
0
 def validateTestcase(self, testcase):
     self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
     self.modelXbrl.viewModelObject(testcase.objectId())
     if hasattr(testcase, "testcaseVariations"):
         for modelTestcaseVariation in testcase.testcaseVariations:
             # update ui thread via modelManager (running in background here)
             self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
             # is this a versioning report?
             resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
             resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
             formulaOutputInstance = None
             inputDTSes = defaultdict(list)
             baseForElement = testcase.baseForElement(modelTestcaseVariation)
             # try to load instance document
             self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
                                 modelObject=modelTestcaseVariation, 
                                 id=modelTestcaseVariation.id, 
                                 name=modelTestcaseVariation.name, 
                                 expected=modelTestcaseVariation.expected, 
                                 description=modelTestcaseVariation.description)
             errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
             parameters = modelTestcaseVariation.parameters.copy()
             for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
                 if isinstance(readMeFirstUri,tuple):
                     # dtsName is for formula instances, but is from/to dts if versioning
                     dtsName, readMeFirstUri = readMeFirstUri
                 elif resultIsVersioningReport:
                     if inputDTSes: dtsName = "to"
                     else: dtsName = "from"
                 else:
                     dtsName = None
                 if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
                     if dtsName in inputDTSes:
                         dtsName = inputDTSes[dtsName]
                     else:
                         modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, 
                                      Type.DTSENTRIES,
                                      self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
                                      isEntry=True,
                                      errorCaptureLevel=errorCaptureLevel)
                     DTSdoc = modelXbrl.modelDocument
                     DTSdoc.inDTS = True
                     doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
                     if doc is not None:
                         DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement)  #fake import
                         doc.inDTS = True
                 else: # not a multi-schemaRef versioning report
                     if self.useFileSource.isArchive:
                         modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                    readMeFirstUri,
                                                    _("validating"), 
                                                    base=baseForElement,
                                                    useFileSource=self.useFileSource,
                                                    errorCaptureLevel=errorCaptureLevel)
                     else: # need own file source, may need instance discovery
                         filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
                         if filesource and not filesource.selection and filesource.isArchive:
                             for _archiveFile in filesource.dir: # find instance document in archive
                                 filesource.select(_archiveFile)
                                 if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
                                     break # use this selection
                         modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                    filesource,
                                                    _("validating"), 
                                                    base=baseForElement,
                                                    errorCaptureLevel=errorCaptureLevel)
                 if modelXbrl.modelDocument is None:
                     self.modelXbrl.error("arelle:notLoaded",
                          _("Testcase %(id)s %(name)s document not loaded: %(file)s"),
                          modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
                     modelXbrl.close()
                     self.determineNotLoadedTestStatus(modelTestcaseVariation)
                 elif resultIsVersioningReport:
                     inputDTSes[dtsName] = modelXbrl
                 elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
                     ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
                     self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                     modelXbrl.close()
                 elif testcase.type == Type.REGISTRYTESTCASE:
                     self.instValidator.validate(modelXbrl)  # required to set up dimensions, etc
                     self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, 
                                modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
                     self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                     self.instValidator.close()
                     modelXbrl.close()
                 else:
                     inputDTSes[dtsName].append(modelXbrl)
                     # validate except for formulas
                     _hasFormulae = modelXbrl.hasFormulae
                     modelXbrl.hasFormulae = False
                     try:
                         for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
                             pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
                         self.instValidator.validate(modelXbrl, parameters)
                         for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
                             pluginXbrlMethod(self.modelXbrl, modelXbrl)
                     except Exception as err:
                         self.modelXbrl.error("exception",
                             _("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
                             modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                     modelXbrl.hasFormulae = _hasFormulae
             if resultIsVersioningReport and modelXbrl.modelDocument:
                 versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
                     modelTestcaseVariation.versioningReportUri, baseForElement)
                 if os.path.exists(versReportFile): #validate existing
                     modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
                     if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
                         ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
                         self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
                         modelVersReport.close()
                 elif len(inputDTSes) == 2:
                     ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
                           versReportFile, inputDTSes["from"], inputDTSes["to"])
                     modelTestcaseVariation.status = "generated"
                 else:
                     self.modelXbrl.error("arelle:notLoaded",
                          _("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
                          modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
                     modelTestcaseVariation.status = "failed"
                 for inputDTS in inputDTSes.values():
                     inputDTS.close()
                 del inputDTSes # dereference
             elif inputDTSes:
                 # validate schema, linkbase, or instance
                 modelXbrl = inputDTSes[None][0]
                 for dtsName, inputDTS in inputDTSes.items():  # input instances are also parameters
                     if dtsName: # named instance
                         parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
                     elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
                         parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
                 if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
                     RenderingEvaluator.init(modelXbrl)
                 if modelXbrl.hasFormulae:
                     try:
                         # validate only formulae
                         self.instValidator.parameters = parameters
                         ValidateFormula.validate(self.instValidator)
                     except Exception as err:
                         self.modelXbrl.error("exception",
                             _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
                             modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                 if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
                     for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
                         pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
                     infoset = ModelXbrl.load(self.modelXbrl.modelManager, 
                                              modelTestcaseVariation.resultInfosetUri,
                                                _("loading result infoset"), 
                                                base=baseForElement,
                                                useFileSource=self.useFileSource,
                                                errorCaptureLevel=errorCaptureLevel)
                     if infoset.modelDocument is None:
                         self.modelXbrl.error("arelle:notLoaded",
                             _("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
                             modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, 
                             file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
                         modelTestcaseVariation.status = "result infoset not loadable"
                     else:   # check infoset
                         ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
                     infoset.close()
                 if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
                     # diff (or generate) table infoset
                     resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
                     if not any(alternativeValidation(modelXbrl, resultTableUri)
                                for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
                         ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True)  # false to save infoset files
                 self.instValidator.close()
                 extraErrors = []
                 for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
                     pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
                 self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
                 if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): 
                     # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
                     modelXbrl.formulaOutputInstance.hasFormulae = False #  block formulae on output instance (so assertion of input is not lost)
                     self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
                     self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
                     if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
                         formulaOutputInstance = modelXbrl.formulaOutputInstance
                         modelXbrl.formulaOutputInstance = None # prevent it from being closed now
                     self.instValidator.close()
                 for inputDTSlist in inputDTSes.values():
                     for inputDTS in inputDTSlist:
                         inputDTS.close()
                 del inputDTSes # dereference
                 if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
                     expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                modelTestcaseVariation.resultXbrlInstanceUri,
                                                _("loading expected result XBRL instance"), 
                                                base=baseForElement,
                                                useFileSource=self.useFileSource,
                                                errorCaptureLevel=errorCaptureLevel)
                     if expectedInstance.modelDocument is None:
                         self.modelXbrl.error("arelle:notLoaded",
                             _("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
                             modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, 
                             file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
                         modelTestcaseVariation.status = "result not loadable"
                     else:   # compare facts
                         if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
                             formulaOutputInstance.error("formula:resultFactCounts",
                                 _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
                                 modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
                                      expectedFacts=len(expectedInstance.facts))
                         else:
                             for fact in expectedInstance.facts:
                                 unmatchedFactsStack = []
                                 if formulaOutputInstance.matchFact(fact, unmatchedFactsStack, deemP0inf=True) is None:
                                     if unmatchedFactsStack: # get missing nested tuple fact, if possible
                                         missingFact = unmatchedFactsStack[-1]
                                     else:
                                         missingFact = fact
                                     formulaOutputInstance.error("formula:expectedFactMissing",
                                         _("Formula output missing expected fact %(fact)s"),
                                         modelXbrl=missingFact, fact=missingFact.qname)
                         # for debugging uncomment next line to save generated instance document
                         # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
                     expectedInstance.close()
                     del expectedInstance # dereference
                     self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
                     formulaOutputInstance.close()
                     del formulaOutputInstance
             # update ui thread via modelManager (running in background here)
             self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
                 
         self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs):
    targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
    targetUrlParts = targetUrl.rpartition(".")
    targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
    modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
    targetInstance = ModelXbrl.create(modelXbrl.modelManager, 
                                      newDocumentType=Type.INSTANCE,
                                      url=targetUrl,
                                      schemaRefs=targetDocumentSchemaRefs,
                                      isEntry=True)
    ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults 
    # roleRef and arcroleRef (of each inline document)
    for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
        for roleRefElt in sourceRefs.values():
            XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, 
                             attributes=roleRefElt.items())
    
    # contexts
    for context in modelXbrl.contexts.values():
        newCntx = targetInstance.createContext(context.entityIdentifier[0],
                                               context.entityIdentifier[1],
                                               'instant' if context.isInstantPeriod else
                                               'duration' if context.isStartEndPeriod
                                               else 'forever',
                                               context.startDatetime,
                                               context.endDatetime,
                                               None, 
                                               context.qnameDims, [], [],
                                               id=context.id)
    for unit in modelXbrl.units.values():
        measures = unit.measures
        newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id)

    modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
    newFactForOldObjId = {}
    def createFacts(facts, parent):
        for fact in modelXbrl.facts:
            if fact.isItem:
                attrs = {"contextRef": fact.contextID}
                if fact.isNumeric:
                    attrs["unitRef"] = fact.unitID
                    if fact.get("decimals"):
                        attrs["decimals"] = fact.get("decimals")
                    if fact.get("precision"):
                        attrs["precision"] = fact.get("precision")
                if fact.isNil:
                    attrs[XbrlConst.qnXsiNil] = "true"
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.textValue
                newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newFact
            elif fact.isTuple:
                newTuple = targetInstance.createFact(fact.qname, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newTuple
                createFacts(fact.modelTupleFacts, newTuple)
                
    createFacts(modelXbrl.facts, None)
    # footnote links
    modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships"))
    for linkKey, linkPrototypes in modelXbrl.baseSets.items():
        arcrole, linkrole, linkqname, arcqname = linkKey
        if (linkrole and linkqname and arcqname and # fully specified roles
            any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
            for linkPrototype in linkPrototypes:
                newLink = XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, linkqname, 
                                           attributes=linkPrototype.attributes)
                for linkChild in linkPrototype:
                    if isinstance(linkChild, LocPrototype) and "{http://www.w3.org/1999/xlink}href" not in linkChild.attributes:
                        linkChild.attributes["{http://www.w3.org/1999/xlink}href"] = \
                        "#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
                    XmlUtil.addChild(newLink, linkChild.qname, 
                                     attributes=linkChild.attributes,
                                     text=linkChild.textValue)
            
    targetInstance.saveInstance(overrideFilepath=targetUrl)
    modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
Exemple #19
0
 def validateTestcase(self, testcase):
     self.modelXbrl.error(_("testcase {0}").format(os.path.basename(testcase.uri)))
     self.modelXbrl.viewModelObject(testcase.objectId())
     if hasattr(testcase, "testcaseVariations"):
         for modelTestcaseVariation in testcase.testcaseVariations:
             # update ui thread via modelManager (running in background here)
             self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
             # is this a versioning report?
             resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
             resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
             formulaOutputInstance = None
             inputDTSes = defaultdict(list)
             baseForElement = testcase.baseForElement(modelTestcaseVariation.element)
             # try to load instance document
             self.modelXbrl.error(_("variation {0} {1}: {2}").format(modelTestcaseVariation.id, modelTestcaseVariation.name, modelTestcaseVariation.expected))
             for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
                 if isinstance(readMeFirstUri,tuple):
                     # dtsName is for formula instances, but is from/to dts if versioning
                     dtsName, readMeFirstUri = readMeFirstUri
                 elif resultIsVersioningReport:
                     if inputDTSes: dtsName = "to"
                     else: dtsName = "from"
                 else:
                     dtsName = None
                 if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
                     if dtsName in inputDTSes:
                         dtsName = inputDTSes[dtsName]
                     else:
                         modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, 
                                      ModelDocument.Type.DTSENTRIES,
                                      self.modelXbrl.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
                                      isEntry=True)
                     DTSdoc = modelXbrl.modelDocument
                     DTSdoc.inDTS = True
                     doc = ModelDocument.load(modelXbrl, readMeFirstUri, base=baseForElement)
                     DTSdoc.referencesDocument[doc] = "import"  #fake import
                     doc.inDTS = True
                 else: # not a multi-schemaRef versioning report
                     modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                readMeFirstUri,
                                                _("validating"), 
                                                base=baseForElement)
                 if modelXbrl.modelDocument is None:
                     self.modelXbrl.error(_("Testcase {0} {1} document not loaded: {2}").format(
                         modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(readMeFirstUri)))
                     modelTestcaseVariation.status = "not loadable"
                     modelXbrl.close()
                 elif resultIsVersioningReport:
                     inputDTSes[dtsName] = modelXbrl
                 elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
                     ValidateVersReport.ValidateVersReport(self.modelXbrl) \
                         .validate(modelXbrl)
                     self.determineTestStatus(modelTestcaseVariation, modelXbrl)
                     modelXbrl.close()
                 elif testcase.type == ModelDocument.Type.REGISTRYTESTCASE:
                     self.instValidator.validate(modelXbrl)  # required to set up dimensions, etc
                     self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, 
                                modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
                     self.determineTestStatus(modelTestcaseVariation, modelXbrl)
                     modelXbrl.close()
                 else:
                     inputDTSes[dtsName].append(modelXbrl)
             if resultIsVersioningReport and modelXbrl.modelDocument:
                 versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
                     modelTestcaseVariation.versioningReportUri, baseForElement)
                 if os.path.exists(versReportFile): #validate existing
                     modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
                     if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
                         ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
                         self.determineTestStatus(modelTestcaseVariation, modelVersReport)
                         modelVersReport.close()
                 elif len(inputDTSes) == 2:
                     ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
                           versReportFile, inputDTSes["from"], inputDTSes["to"])
                     modelTestcaseVariation.status = "generated"
                 else:
                     self.modelXbrl.error(_("Testcase {0} {1} DTSes not loaded, unable to generate versioning report").format(
                         modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(readMeFirstUri)))
                     modelTestcaseVariation.status = "failed"
                 for inputDTS in inputDTSes:
                     inputDTS.close()
             elif inputDTSes:
                 # validate schema, linkbase, or instance
                 modelXbrl = inputDTSes[None][0]
                 parameters = modelTestcaseVariation.parameters.copy()
                 for dtsName, inputDTS in inputDTSes.items():  # input instances are also parameters
                     if dtsName:
                         parameters[dtsName] = (None, inputDTS)
                 self.instValidator.validate(modelXbrl, parameters)
                 self.determineTestStatus(modelTestcaseVariation, modelXbrl)
                 if modelXbrl.formulaOutputInstance and len(modelTestcaseVariation.actual) == 0: 
                     # if an output instance is created, validate it
                     self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
                     self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance)
                     if len(modelTestcaseVariation.actual) == 0: # if still 'clean' pass it forward for comparison to expected result instance
                         formulaOutputInstance = modelXbrl.formulaOutputInstance
                         modelXbrl.formulaOutputInstance = None # prevent it from being closed now
                 for inputDTSlist in inputDTSes.values():
                     for inputDTS in inputDTSlist:
                         inputDTS.close()
                 if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
                     expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                modelTestcaseVariation.resultXbrlInstanceUri,
                                                _("loading expected result XBRL instance"), 
                                                base=baseForElement)
                     if expectedInstance.modelDocument is None:
                         self.modelXbrl.error(_("Testcase {0} {1} expected result instance not loaded: {2}").format(
                             modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(modelTestcaseVariation.resultXbrlInstance)))
                         modelTestcaseVariation.status = "result not loadable"
                         expectedInstance.close()
                     else:   # compare facts
                         if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
                             formulaOutputInstance.error(
                                 _("Formula output {0} facts, expected {1} facts").format(
                                      len(formulaOutputInstance.facts),
                                      len(expectedInstance.facts)), 
                                 "err", "formula:resultFactCounts")
                         else:
                             for fact in expectedInstance.facts:
                                 if not formulaOutputInstance.matchFact(fact):
                                     formulaOutputInstance.error(
                                         _("Formula output missing expected fact {0}").format(
                                              fact), 
                                         "err", "formula:expectedFactMissing")
                     self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance)
                     formulaOutputInstance.close()
                     formulaOutputInstance = None
             # update ui thread via modelManager (running in background here)
             self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
                 
         self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
Exemple #20
0
    def runFromExcel(self, options):
        testGenFileName = options.excelfilename
        #testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls"
        testGenDir = os.path.dirname(testGenFileName)
        timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
        if options.testfiledate:
            today = options.testfiledate
        else:
            today = XmlUtil.dateunionValue(datetime.date.today())
        startedAt = time.time()
        
        self.logMessages = []
        logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt'

        modelTestcases = ModelXbrl.create(self.modelManager)
        testcaseIndexBook = xlrd.open_workbook(testGenFileName)
        testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
        self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow))
        
        # start index file
        indexFiles = [testGenDir + os.sep + 'creationTestcasesIndex.xml',
                      testGenDir + os.sep + 'consumptionTestcasesIndex.xml']
        indexDocs = [xml.dom.minidom.parseString(
            '<?xml version="1.0" encoding="UTF-8"?>'
            '<!-- XBRL Versioning 1.0 {0} Tests -->'
            '<!-- Copyright 2011 XBRL International.  All Rights Reserved. -->'
            '<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
            '<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" '
            ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
            ' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
            '</testcases>'.format(purpose, today)
            ) for purpose in ("Creation","Consumption")]
        testcasesElements = [XmlUtil.child(indexDoc, None, "testcases") for indexDoc in indexDocs]
        priorTestcasesDir = None
        testcaseFiles = None
        testcaseDocs = None
        for iRow in range(1, testcaseIndexSheet.nrows):
            row = testcaseIndexSheet.row(iRow)
            if row[0].ctype == xlrd.XL_CELL_EMPTY or row[1].ctype == xlrd.XL_CELL_EMPTY or row[2].ctype == xlrd.XL_CELL_EMPTY:
                continue
            testDir = row[0].value
            uriFrom = row[1].value
            uriTo = row[2].value
            intention = row[3].value
            if intention is None or len(intention) == 0:
                continue # test not ready to run
            reason = row[4].value
            expectedEvent = row[5].value
            base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep
            self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
            if uriFrom and uriTo and reason.lower() not in ("n.a.", "error") and expectedEvent != "N.A.":
                for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)):
                    if ',' not in URIs:
                        modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base)
                    else:
                        modelDTS = ModelXbrl.create(self.modelManager, 
                                     ModelDocument.Type.DTSENTRIES,
                                     self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts", 
                                                                base),
                                     isEntry=True)
                        DTSdoc = modelDTS.modelDocument
                        DTSdoc.inDTS = True
                        for uri in URIs.split(','):
                            doc = ModelDocument.load(modelDTS, uri.strip(), base=base)
                            DTSdoc.referencesDocument[doc] = "import"  #fake import
                            doc.inDTS = True
                    if isFrom: modelDTSfrom = modelDTS
                    else: modelDTSto = modelDTS
                if modelDTSfrom and modelDTSto:
                    # generate differences report
                    reportUri = uriFrom.partition(',')[0]  # first file
                    reportDir = os.path.dirname(reportUri)
                    if reportDir: reportDir += os.sep
                    reportName = os.path.basename(reportUri).replace("from.xsd","report.xml")
                    reportFile = reportDir + "report" + os.sep + reportName
                    reportFullPath = self.webCache.normalizeUrl(
                                        reportFile, 
                                        base)
                    testcasesDir = os.path.dirname(os.path.dirname(reportFullPath))
                    if testcasesDir != priorTestcasesDir:
                        # close prior report
                        if priorTestcasesDir:
                            for i,testcaseFile in enumerate(testcaseFiles):
                                with open(testcaseFile, "w", encoding="utf-8") as fh:
                                    XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
                        testcaseName = os.path.basename(testcasesDir)
                        testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml",
                                         testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"]
                        for i,testcaseFile in enumerate(testcaseFiles):
                            XmlUtil.addChild(testcasesElements[i], None, "testcase", 
                                             ("uri", 
                                              testcaseFile[len(testGenDir)+1:].replace("\\","/")) )
                        
                        # start testcase file
                        testcaseDocs = [xml.dom.minidom.parseString(
                            '<?xml version="1.0" encoding="UTF-8"?>'
                            '<!-- Copyright 2011 XBRL International.  All Rights Reserved. -->'
                            '<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
                            '<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" '
                            ' xmlns="http://xbrl.org/2008/conformance"'
                            ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
                            ' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
                            '<creator>'
                            '<name>Roland Hommes</name>'
                            '<email>[email protected]</email>'
                            '</creator>'
                            '<name>{0}</name>'
                            '<description>{0}</description>'
                            '</testcase>'.format(testcaseName,purpose,today)
                            ) for purpose in ("Creation","Consumption")]
                        testcaseElements = [XmlUtil.child(testcaseDoc, conformanceNS, "testcase") for testcaseDoc in testcaseDocs]
                        priorTestcasesDir = testcasesDir
                        variationID = 1
                    try:
                        os.makedirs(os.path.dirname(reportFullPath))
                    except WindowsError:
                        pass # dir already exists
                    modelVersReport = ModelVersReport.ModelVersReport(modelTestcases)
                    modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto)
                    
                    # check for expected elements
                    if expectedEvent and expectedEvent not in (
                           "No change", "N.A."):
                        if len(modelVersReport.xmlDocument.getElementsByTagNameNS('*',expectedEvent)) == 0:
                            modelTestcases.error(
                                "Generated test case {0} missing expected event {1}".format(
                                           reportName, 
                                           expectedEvent), 
                                "wrn", "missingEvent")
                    
                    modelVersReport.close([])
                    for i,testcaseElt in enumerate(testcaseElements):
                        variationElement = XmlUtil.addChild(testcaseElt, conformanceNS, "variation", 
                                      attributes=("id", "_{0:02n}".format(variationID)))
                        XmlUtil.addChild(variationElement, conformanceNS, "name", text=intention)
                        dataElement = XmlUtil.addChild(variationElement, conformanceNS, "data")
                        for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")):
                            for schemaURI in schemaURIs.split(","): 
                                XmlUtil.addChild(dataElement, conformanceNS, "schema", 
                                                 attributes=((("dts",dtsAttr),) +
                                                             ((("readMeFirst","true"),) if i == 0 else ())), 
                                                 text=os.path.basename(schemaURI.strip()))
                        resultElement = XmlUtil.addChild(variationElement, conformanceNS, "result")
                        XmlUtil.addChild(resultElement if i == 0 else dataElement, 
                                         conformanceNS, "versioningReport", 
                                         attributes=(("readMeFirst","true") if i == 1 else ()), 
                                         text="report/" + reportName)
                    variationID += 1
        
        with open(logMessagesFile, "w") as fh:
            fh.writelines(self.logMessages)

        if priorTestcasesDir:
            for i,testcaseFile in enumerate(testcaseFiles):
                with open(testcaseFile, "w", encoding="utf-8") as fh:
                    XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
        for i,indexFile in enumerate(indexFiles):
            with open(indexFile, "w", encoding="utf-8") as fh:
                XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
Exemple #21
0
 def load(self, filesource, nextaction):
     self.filesource = filesource
     self.modelXbrl = ModelXbrl.load(self, filesource, nextaction)
     self.loadedModelXbrls.append(self.modelXbrl)
     return self.modelXbrl
def _create_cache(val):
    """
    Creates the caches needed for dqc_us_0018

    :param val: ValidateXbrl needed in order to save the cache
    :type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
    :return: no explicit return but creates and saves a cache in
        dqc_us_rule\resources\DQC_US_0018
    :rtype: None
    """
    val.ugtNamespace = None
    cntlr = val.modelXbrl.modelManager.cntlr
    year = _EARLIEST_US_GAAP_YEAR

    for ugt in ugtDocs:
        deprecations_json_file = os.path.join(
            os.path.dirname(__file__),
            'resources',
            'DQC_US_0018',
            '{}_deprecated-concepts.json'.format(str(year))
        )

        if not os.path.isfile(deprecations_json_file):
            ugt_doc_lb = ugt["docLB"]
            val.usgaapDeprecations = {}
            disclosure_system = (
                val.modelXbrl.modelManager.validateDisclosureSystem
            )

            prior_validate_disclosure_system = disclosure_system
            val.modelXbrl.modelManager.validateDisclosureSystem = False
            deprecations_instance = ModelXbrl.load(
                val.modelXbrl.modelManager,
                openFileSource(ugt_doc_lb, cntlr),
                _("built deprecations table in cache")  # noqa
            )
            val.modelXbrl.modelManager.validateDisclosureSystem = (
                prior_validate_disclosure_system
            )

            if deprecations_instance is not None:
                dep_label = 'http://www.xbrl.org/2009/role/deprecatedLabel'
                dep_date_label = (
                    'http://www.xbrl.org/2009/role/deprecatedDateLabel'
                )
                concept_label = XbrlConst.conceptLabel
                relationship_set = (
                    deprecations_instance.relationshipSet(concept_label)
                )
                model_relationships = relationship_set.modelRelationships

                for labelRel in model_relationships:
                    model_documentation = labelRel.toModelObject
                    concept = labelRel.fromModelObject.name

                    if model_documentation.role == dep_label:
                        val.usgaapDeprecations[concept] = (
                            model_documentation.text,
                            val.usgaapDeprecations.get(concept, ('', ''))[0]
                        )
                    elif model_documentation.role == dep_date_label:
                        val.usgaapDeprecations[concept] = (
                            model_documentation.text,
                            val.usgaapDeprecations.get(concept, ('', ''))[1]
                        )
                json_str = str(
                    json.dumps(
                        val.usgaapDeprecations,
                        ensure_ascii=False, indent=0
                    )
                )  # might not be unicode in 2.7
                saveFile(cntlr, deprecations_json_file, json_str)
                deprecations_instance.close()
                del deprecations_instance  # dereference closed modelXbrl
        year += 1
 def create(self, newDocumentType=None, url=None, schemaRefs=None, createModelDocument=True):
     self.modelXbrl = ModelXbrl.create(self, newDocumentType, url, schemaRefs, createModelDocument)
     self.loadedModelXbrls.append(self.modelXbrl)
     return self.modelXbrl
Exemple #24
0
    def validateTestcase(self, testcase):
        self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
        self.modelXbrl.viewModelObject(testcase.objectId())
        if hasattr(testcase, "testcaseVariations"):
            for modelTestcaseVariation in testcase.testcaseVariations:
                # update ui thread via modelManager (running in background here)
                self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
                # is this a versioning report?
                resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
                resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
                resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
                formulaOutputInstance = None
                inputDTSes = defaultdict(list)
                baseForElement = testcase.baseForElement(modelTestcaseVariation)
                # try to load instance document
                self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
                                    modelObject=modelTestcaseVariation, 
                                    id=modelTestcaseVariation.id, 
                                    name=modelTestcaseVariation.name, 
                                    expected=modelTestcaseVariation.expected, 
                                    description=modelTestcaseVariation.description)
                errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
                parameters = modelTestcaseVariation.parameters.copy()
                for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
                    if isinstance(readMeFirstUri,tuple):
                        # dtsName is for formula instances, but is from/to dts if versioning
                        dtsName, readMeFirstUri = readMeFirstUri
                    elif resultIsVersioningReport:
                        if inputDTSes: dtsName = "to"
                        else: dtsName = "from"
                    else:
                        dtsName = None
                    if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
                        if dtsName in inputDTSes:
                            dtsName = inputDTSes[dtsName]
                        else:
                            modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, 
                                         Type.DTSENTRIES,
                                         self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
                                         isEntry=True,
                                         errorCaptureLevel=errorCaptureLevel)
                        DTSdoc = modelXbrl.modelDocument
                        DTSdoc.inDTS = True
                        doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
                        if doc is not None:
                            DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement)  #fake import
                            doc.inDTS = True
                    elif resultIsTaxonomyPackage:
                        from arelle import PackageManager, PrototypeInstanceObject
                        dtsName = readMeFirstUri
                        modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
                        PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
                    else: # not a multi-schemaRef versioning report
                        if self.useFileSource.isArchive:
                            modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                       readMeFirstUri,
                                                       _("validating"), 
                                                       base=baseForElement,
                                                       useFileSource=self.useFileSource,
                                                       errorCaptureLevel=errorCaptureLevel)
                        else: # need own file source, may need instance discovery
                            filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
                            if filesource and not filesource.selection and filesource.isArchive:
                                for _archiveFile in filesource.dir: # find instance document in archive
                                    filesource.select(_archiveFile)
                                    if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
                                        break # use this selection
                            modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                       filesource,
                                                       _("validating"), 
                                                       base=baseForElement,
                                                       errorCaptureLevel=errorCaptureLevel)
                        modelXbrl.isTestcaseVariation = True
                    if modelXbrl.modelDocument is None:
                        modelXbrl.error("arelle:notLoaded",
                             _("Testcase %(id)s %(name)s document not loaded: %(file)s"),
                             modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
                        self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors)
                        modelXbrl.close()
                    elif resultIsVersioningReport or resultIsTaxonomyPackage:
                        inputDTSes[dtsName] = modelXbrl
                    elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
                        ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
                        self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                        modelXbrl.close()
                    elif testcase.type == Type.REGISTRYTESTCASE:
                        self.instValidator.validate(modelXbrl)  # required to set up dimensions, etc
                        self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, 
                                   modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
                        self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                        self.instValidator.close()
                        modelXbrl.close()
                    else:
                        inputDTSes[dtsName].append(modelXbrl)
                        # validate except for formulas
                        _hasFormulae = modelXbrl.hasFormulae
                        modelXbrl.hasFormulae = False
                        try:
                            for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
                                pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
                            self.instValidator.validate(modelXbrl, parameters)
                            for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
                                pluginXbrlMethod(self.modelXbrl, modelXbrl)
                        except Exception as err:
                            modelXbrl.error("exception:" + type(err).__name__,
                                _("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
                                modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                        modelXbrl.hasFormulae = _hasFormulae
                if resultIsVersioningReport and modelXbrl.modelDocument:
                    versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
                        modelTestcaseVariation.versioningReportUri, baseForElement)
                    if os.path.exists(versReportFile): #validate existing
                        modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
                        if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
                            ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
                            self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
                            modelVersReport.close()
                    elif len(inputDTSes) == 2:
                        ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
                              versReportFile, inputDTSes["from"], inputDTSes["to"])
                        modelTestcaseVariation.status = "generated"
                    else:
                        modelXbrl.error("arelle:notLoaded",
                             _("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
                             modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
                        modelTestcaseVariation.status = "failed"
                    for inputDTS in inputDTSes.values():
                        inputDTS.close()
                    del inputDTSes # dereference
                elif resultIsTaxonomyPackage:
                    self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                    modelXbrl.close()
                elif inputDTSes:
                    # validate schema, linkbase, or instance
                    modelXbrl = inputDTSes[None][0]
                    for dtsName, inputDTS in inputDTSes.items():  # input instances are also parameters
                        if dtsName: # named instance
                            parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
                        elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
                            parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
                    if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
                        RenderingEvaluator.init(modelXbrl)
                    if modelXbrl.hasFormulae:
                        try:
                            # validate only formulae
                            self.instValidator.parameters = parameters
                            ValidateFormula.validate(self.instValidator)
                        except Exception as err:
                            modelXbrl.error("exception:" + type(err).__name__,
                                _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
                                modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                    if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
                        for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
                            pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
                        infoset = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                 modelTestcaseVariation.resultInfosetUri,
                                                   _("loading result infoset"), 
                                                   base=baseForElement,
                                                   useFileSource=self.useFileSource,
                                                   errorCaptureLevel=errorCaptureLevel)
                        if infoset.modelDocument is None:
                            modelXbrl.error("arelle:notLoaded",
                                _("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
                                modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, 
                                file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
                            modelTestcaseVariation.status = "result infoset not loadable"
                        else:   # check infoset
                            ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
                        infoset.close()
                    if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
                        # diff (or generate) table infoset
                        resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
                        if not any(alternativeValidation(modelXbrl, resultTableUri)
                                   for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
                            ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True)  # false to save infoset files
                    self.instValidator.close()
                    extraErrors = []
                    for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
                        pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
                    self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
                    if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): 
                        # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
                        modelXbrl.formulaOutputInstance.hasFormulae = False #  block formulae on output instance (so assertion of input is not lost)
                        self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
                        self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
                        if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
                            formulaOutputInstance = modelXbrl.formulaOutputInstance
                            modelXbrl.formulaOutputInstance = None # prevent it from being closed now
                        self.instValidator.close()
                    compareIxResultInstance = getattr(modelXbrl, "extractedInlineInstance", False) and modelTestcaseVariation.resultXbrlInstanceUri
                    if compareIxResultInstance:
                        formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
                        errMsgPrefix = "ix"
                    else: # delete input instances before formula output comparision
                        for inputDTSlist in inputDTSes.values():
                            for inputDTS in inputDTSlist:
                                inputDTS.close()
                        del inputDTSes # dereference
                        errMsgPrefix = "formula"
                    if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
                        expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                   modelTestcaseVariation.resultXbrlInstanceUri,
                                                   _("loading expected result XBRL instance"), 
                                                   base=baseForElement,
                                                   useFileSource=self.useFileSource,
                                                   errorCaptureLevel=errorCaptureLevel)
                        if expectedInstance.modelDocument is None:
                            self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
                                _("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
                                modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, 
                                file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
                                messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
                            modelTestcaseVariation.status = "result not loadable"
                        else:   # compare facts
                            if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
                                formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
                                    _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
                                    modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
                                    expectedFacts=len(expectedInstance.facts),
                                    messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
                            else:
                                formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
                                expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
                                def factFootnotes(fact, footnotesRelSet):
                                    footnotes = []
                                    footnoteRels = footnotesRelSet.fromModelObject(fact)
                                    if footnoteRels:
                                        # most process rels in same order between two instances, use labels to sort
                                        for i, footnoteRel in enumerate(sorted(footnoteRels,
                                                                               key=lambda r: (r.fromLabel,r.toLabel))):
                                            modelObject = footnoteRel.toModelObject
                                            if isinstance(modelObject, ModelResource):
                                                footnotes.append("Footnote {}: {}".format(
                                                   i+1, # compare footnote with normalize-space
                                                   re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))))
                                            elif isinstance(modelObject, ModelFact):
                                                footnotes.append("Footnoted fact {}: {} context: {} value: {}".format(
                                                    i+1,
                                                    modelObject.qname,
                                                    modelObject.contextID,
                                                    collapseWhitespace(modelObject.value)))
                                    return footnotes
                                for expectedInstanceFact in expectedInstance.facts:
                                    unmatchedFactsStack = []
                                    formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True)
                                    if formulaOutputFact is None:
                                        if unmatchedFactsStack: # get missing nested tuple fact, if possible
                                            missingFact = unmatchedFactsStack[-1]
                                        else:
                                            missingFact = expectedInstanceFact
                                        formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
                                            _("Output missing expected fact %(fact)s"),
                                            modelXbrl=missingFact, fact=missingFact.qname,
                                            messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
                                    else: # compare footnotes
                                        expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
                                        formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
                                        if expectedInstanceFactFootnotes != formulaOutputFactFootnotes:
                                            formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
                                                _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
                                                modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=expectedInstanceFactFootnotes, footnotes2=formulaOutputFactFootnotes,
                                                messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))

                            # for debugging uncomment next line to save generated instance document
                            # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
                        expectedInstance.close()
                        del expectedInstance # dereference
                        self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
                        formulaOutputInstance.close()
                        del formulaOutputInstance
                    if compareIxResultInstance:
                        for inputDTSlist in inputDTSes.values():
                            for inputDTS in inputDTSlist:
                                inputDTS.close()
                        del inputDTSes # dereference
                # update ui thread via modelManager (running in background here)
                self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
                    
            self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
Exemple #25
0
 def create(self, newDocumentType=None, url=None, schemaRefs=None, createModelDocument=True, isEntry=False, errorCaptureLevel=None, initialXml=None, base=None):
     self.modelXbrl = ModelXbrl.create(self, newDocumentType, url, schemaRefs, createModelDocument, isEntry, errorCaptureLevel, initialXml, base)
     self.loadedModelXbrls.append(self.modelXbrl)
     return self.modelXbrl
Exemple #26
0
    def validateTestcase(self, testcase):
        self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
        self.modelXbrl.viewModelObject(testcase.objectId())
        if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY):
            for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri):
                self.validateTestcase(doc)  # testcases doc's are sorted by their uri (file names), e.g., for formula
        elif hasattr(testcase, "testcaseVariations"):
            for modelTestcaseVariation in testcase.testcaseVariations:
                # update ui thread via modelManager (running in background here)
                self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
                # is this a versioning report?
                resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
                resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
                resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
                formulaOutputInstance = None
                inputDTSes = defaultdict(list)
                baseForElement = testcase.baseForElement(modelTestcaseVariation)
                # try to load instance document
                self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
                                    modelObject=modelTestcaseVariation, 
                                    id=modelTestcaseVariation.id, 
                                    name=modelTestcaseVariation.name, 
                                    expected=modelTestcaseVariation.expected, 
                                    description=modelTestcaseVariation.description)
                if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings:
                    errorCaptureLevel = logging._checkLevel("WARNING")
                else:
                    errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
                parameters = modelTestcaseVariation.parameters.copy()
                for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
                    if isinstance(readMeFirstUri,tuple):
                        # dtsName is for formula instances, but is from/to dts if versioning
                        dtsName, readMeFirstUri = readMeFirstUri
                    elif resultIsVersioningReport:
                        if inputDTSes: dtsName = "to"
                        else: dtsName = "from"
                    else:
                        dtsName = None
                    if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
                        if dtsName in inputDTSes:
                            dtsName = inputDTSes[dtsName]
                        else:
                            modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, 
                                         Type.DTSENTRIES,
                                         self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
                                         isEntry=True,
                                         errorCaptureLevel=errorCaptureLevel)
                        DTSdoc = modelXbrl.modelDocument
                        DTSdoc.inDTS = True
                        doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
                        if doc is not None:
                            DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement)  #fake import
                            doc.inDTS = True
                    elif resultIsTaxonomyPackage:
                        from arelle import PackageManager, PrototypeInstanceObject
                        dtsName = readMeFirstUri
                        modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
                        PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
                    else: # not a multi-schemaRef versioning report
                        if self.useFileSource.isArchive:
                            modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                       readMeFirstUri,
                                                       _("validating"), 
                                                       base=baseForElement,
                                                       useFileSource=self.useFileSource,
                                                       errorCaptureLevel=errorCaptureLevel)
                        else: # need own file source, may need instance discovery
                            filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
                            if filesource and not filesource.selection and filesource.isArchive:
                                for _archiveFile in filesource.dir or (): # find instance document in archive
                                    filesource.select(_archiveFile)
                                    if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
                                        break # use this selection
                            modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                       filesource,
                                                       _("validating"), 
                                                       base=baseForElement,
                                                       errorCaptureLevel=errorCaptureLevel)
                        modelXbrl.isTestcaseVariation = True
                    if modelXbrl.modelDocument is None:
                        modelXbrl.error("arelle:notLoaded",
                             _("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"),
                             modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
                        self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors)
                        modelXbrl.close()
                    elif resultIsVersioningReport or resultIsTaxonomyPackage:
                        inputDTSes[dtsName] = modelXbrl
                    elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
                        ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
                        self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                        modelXbrl.close()
                    elif testcase.type == Type.REGISTRYTESTCASE:
                        self.instValidator.validate(modelXbrl)  # required to set up dimensions, etc
                        self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, 
                                   modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
                        self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                        self.instValidator.close()
                        modelXbrl.close()
                    else:
                        inputDTSes[dtsName].append(modelXbrl)
                        # validate except for formulas
                        _hasFormulae = modelXbrl.hasFormulae
                        modelXbrl.hasFormulae = False
                        try:
                            for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
                                pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
                            self.instValidator.validate(modelXbrl, parameters)
                            for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
                                pluginXbrlMethod(self.modelXbrl, modelXbrl)
                        except Exception as err:
                            modelXbrl.error("exception:" + type(err).__name__,
                                _("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
                                modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                        modelXbrl.hasFormulae = _hasFormulae
                if resultIsVersioningReport and modelXbrl.modelDocument:
                    versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
                        modelTestcaseVariation.versioningReportUri, baseForElement)
                    if os.path.exists(versReportFile): #validate existing
                        modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
                        if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
                            ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
                            self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
                            modelVersReport.close()
                    elif len(inputDTSes) == 2:
                        ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
                              versReportFile, inputDTSes["from"], inputDTSes["to"])
                        modelTestcaseVariation.status = "generated"
                    else:
                        modelXbrl.error("arelle:notLoaded",
                             _("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"),
                             modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
                        modelTestcaseVariation.status = "failed"
                    for inputDTS in inputDTSes.values():
                        inputDTS.close()
                    del inputDTSes # dereference
                elif resultIsTaxonomyPackage:
                    self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                    modelXbrl.close()
                elif inputDTSes:
                    # validate schema, linkbase, or instance
                    modelXbrl = inputDTSes[None][0]
                    expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement)
                                            for d in modelTestcaseVariation.dataUris.values() for uri in d
                                            if not UrlUtil.isAbsolute(uri))
                    foundDataFiles = set()
                    variationBase = os.path.dirname(baseForElement)
                    for dtsName, inputDTS in inputDTSes.items():  # input instances are also parameters
                        if dtsName: # named instance
                            parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
                        elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
                            parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
                        for _inputDTS in inputDTS:
                            for docUrl, doc in _inputDTS.urlDocs.items():
                                if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET:
                                    if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file
                                        if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles:
                                            docUrl = docUrl.replace("-formula.xml", ".xf")
                                    foundDataFiles.add(docUrl)
                    if expectedDataFiles - foundDataFiles:
                        modelXbrl.info("arelle:testcaseDataNotUsed",
                            _("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"),
                            modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, 
                            missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles)))
                    if foundDataFiles - expectedDataFiles:
                        modelXbrl.info("arelle:testcaseDataUnexpected",
                            _("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"),
                            modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
                            unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles)))
                    if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
                        try:
                            RenderingEvaluator.init(modelXbrl)
                        except Exception as err:
                            modelXbrl.error("exception:" + type(err).__name__,
                                _("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"),
                                modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                    modelXbrlHasFormulae = modelXbrl.hasFormulae
                    if modelXbrlHasFormulae:
                        try:
                            # validate only formulae
                            self.instValidator.parameters = parameters
                            ValidateFormula.validate(self.instValidator)
                        except Exception as err:
                            modelXbrl.error("exception:" + type(err).__name__,
                                _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
                                modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                    if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
                        for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
                            pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
                        infoset = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                 modelTestcaseVariation.resultInfosetUri,
                                                   _("loading result infoset"), 
                                                   base=baseForElement,
                                                   useFileSource=self.useFileSource,
                                                   errorCaptureLevel=errorCaptureLevel)
                        if infoset.modelDocument is None:
                            modelXbrl.error("arelle:notLoaded",
                                _("Variation %(id)s %(name)s result infoset not loaded: %(file)s"),
                                modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, 
                                file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
                            modelTestcaseVariation.status = "result infoset not loadable"
                        else:   # check infoset
                            ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
                        infoset.close()
                    if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
                        # diff (or generate) table infoset
                        resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
                        if not any(alternativeValidation(modelXbrl, resultTableUri)
                                   for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
                            try:
                                ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True)  # false to save infoset files
                            except Exception as err:
                                modelXbrl.error("exception:" + type(err).__name__,
                                    _("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"),
                                    modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                    self.instValidator.close()
                    extraErrors = []
                    for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
                        pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
                    self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
                    if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): 
                        # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
                        modelXbrl.formulaOutputInstance.hasFormulae = False #  block formulae on output instance (so assertion of input is not lost)
                        self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
                        self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
                        if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
                            formulaOutputInstance = modelXbrl.formulaOutputInstance
                            modelXbrl.formulaOutputInstance = None # prevent it from being closed now
                        self.instValidator.close()
                    compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and 
                                               modelTestcaseVariation.resultXbrlInstanceUri is not None)
                    if compareIxResultInstance:
                        formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
                        errMsgPrefix = "ix"
                    else: # delete input instances before formula output comparision
                        for inputDTSlist in inputDTSes.values():
                            for inputDTS in inputDTSlist:
                                inputDTS.close()
                        del inputDTSes # dereference
                        errMsgPrefix = "formula"
                    if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
                        _matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs
                        expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                   modelTestcaseVariation.resultXbrlInstanceUri,
                                                   _("loading expected result XBRL instance"), 
                                                   base=baseForElement,
                                                   useFileSource=self.useFileSource,
                                                   errorCaptureLevel=errorCaptureLevel)
                        if expectedInstance.modelDocument is None:
                            self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
                                _("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"),
                                modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, 
                                file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
                                messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
                            modelTestcaseVariation.status = "result not loadable"
                        else:   # compare facts
                            if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
                                formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
                                    _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
                                    modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
                                    expectedFacts=len(expectedInstance.facts),
                                    messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
                            else:
                                formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
                                expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
                                def factFootnotes(fact, footnotesRelSet):
                                    footnotes = {}
                                    footnoteRels = footnotesRelSet.fromModelObject(fact)
                                    if footnoteRels:
                                        # most process rels in same order between two instances, use labels to sort
                                        for i, footnoteRel in enumerate(sorted(footnoteRels,
                                                                               key=lambda r: (r.fromLabel,r.toLabel))):
                                            modelObject = footnoteRel.toModelObject
                                            if isinstance(modelObject, ModelResource):
                                                xml = modelObject.viewText().strip()
                                                footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))
                                            elif isinstance(modelObject, ModelFact):
                                                footnotes["Footnoted fact {}".format(i+1)] = \
                                                    "{} context: {} value: {}".format(
                                                    modelObject.qname,
                                                    modelObject.contextID,
                                                    collapseWhitespace(modelObject.value))
                                    return footnotes
                                for expectedInstanceFact in expectedInstance.facts:
                                    unmatchedFactsStack = []
                                    formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False)
                                    #formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True)
                                    if formulaOutputFact is None:
                                        if unmatchedFactsStack: # get missing nested tuple fact, if possible
                                            missingFact = unmatchedFactsStack[-1]
                                        else:
                                            missingFact = expectedInstanceFact
                                        formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
                                            _("Output missing expected fact %(fact)s"),
                                            modelXbrl=missingFact, fact=missingFact.qname,
                                            messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
                                    else: # compare footnotes
                                        expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
                                        formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
                                        if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or
                                            set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())):
                                            formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
                                                _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
                                                modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()),
                                                messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))

                            # for debugging uncomment next line to save generated instance document
                            # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
                        expectedInstance.close()
                        del expectedInstance # dereference
                        self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
                        formulaOutputInstance.close()
                        del formulaOutputInstance
                    if compareIxResultInstance:
                        for inputDTSlist in inputDTSes.values():
                            for inputDTS in inputDTSlist:
                                inputDTS.close()
                        del inputDTSes # dereference
                # update ui thread via modelManager (running in background here)
                self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
                    
            _statusCounts = OrderedDict((("pass",0),("fail",0)))
            for tv in getattr(testcase, "testcaseVariations", ()):
                _statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1    
            self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k))
            
            self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None):
    targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
    targetUrlParts = targetUrl.rpartition(".")
    targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
    modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
    targetInstance = ModelXbrl.create(modelXbrl.modelManager, 
                                      newDocumentType=Type.INSTANCE,
                                      url=targetUrl,
                                      schemaRefs=targetDocumentSchemaRefs,
                                      isEntry=True)
    ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults 
    # roleRef and arcroleRef (of each inline document)
    for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
        for roleRefElt in sourceRefs.values():
            addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, 
                     attributes=roleRefElt.items())
    
    # contexts
    for context in modelXbrl.contexts.values():
        newCntx = targetInstance.createContext(context.entityIdentifier[0],
                                               context.entityIdentifier[1],
                                               'instant' if context.isInstantPeriod else
                                               'duration' if context.isStartEndPeriod
                                               else 'forever',
                                               context.startDatetime,
                                               context.endDatetime,
                                               None, 
                                               context.qnameDims, [], [],
                                               id=context.id)
    for unit in modelXbrl.units.values():
        measures = unit.measures
        newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id)

    modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
    newFactForOldObjId = {}
    def createFacts(facts, parent):
        for fact in facts:
            if fact.isItem:
                attrs = {"contextRef": fact.contextID}
                if fact.id:
                    attrs["id"] = fact.id
                if fact.isNumeric:
                    attrs["unitRef"] = fact.unitID
                    if fact.get("decimals"):
                        attrs["decimals"] = fact.get("decimals")
                    if fact.get("precision"):
                        attrs["precision"] = fact.get("precision")
                if fact.isNil:
                    attrs[XbrlConst.qnXsiNil] = "true"
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.textValue
                newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newFact
                if filingFiles and fact.concept is not None and fact.concept.isTextBlock:
                    # check for img and other filing references
                    for xmltext in [text] + CDATApattern.findall(text):
                        try:
                            for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)):
                                if elt.tag in ("a", "img") and not isHttpUrl(attrValue) and not os.path.isabs(attrvalue):
                                    for attrTag, attrValue in elt.items():
                                        if attrTag in ("href", "src"):
                                            filingFiles.add(attrValue)
                        except (XMLSyntaxError, UnicodeDecodeError):
                            pass
            elif fact.isTuple:
                newTuple = targetInstance.createFact(fact.qname, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newTuple
                createFacts(fact.modelTupleFacts, newTuple)
                
    createFacts(modelXbrl.facts, None)
    # footnote links
    footnoteIdCount = {}
    modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships"))
    HREF = "{http://www.w3.org/1999/xlink}href"
    footnoteLinks = defaultdict(list)
    for linkKey, linkPrototypes in modelXbrl.baseSets.items():
        arcrole, linkrole, linkqname, arcqname = linkKey
        if (linkrole and linkqname and arcqname and # fully specified roles
            arcrole != "XBRL-footnotes" and
            any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
            for linkPrototype in linkPrototypes:
                if linkPrototype not in footnoteLinks[linkrole]:
                    footnoteLinks[linkrole].append(linkPrototype)
    for linkrole in sorted(footnoteLinks.keys()):
        for linkPrototype in footnoteLinks[linkrole]:
            newLink = addChild(targetInstance.modelDocument.xmlRootElement, 
                               linkPrototype.qname, 
                               attributes=linkPrototype.attributes)
            for linkChild in linkPrototype:
                attributes = linkChild.attributes
                if isinstance(linkChild, LocPrototype):
                    if HREF not in linkChild.attributes:
                        linkChild.attributes[HREF] = \
                        "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
                    addChild(newLink, linkChild.qname, 
                             attributes=attributes)
                elif isinstance(linkChild, ArcPrototype):
                    addChild(newLink, linkChild.qname, attributes=attributes)
                elif isinstance(linkChild, ModelInlineFootnote):
                    idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
                    if idUseCount > 1: # if footnote with id in other links bump the id number
                        attributes = linkChild.attributes.copy()
                        attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
                    footnoteIdCount[linkChild.footnoteID] = idUseCount
                    newChild = addChild(newLink, linkChild.qname, 
                                        attributes=attributes)
                    copyIxFootnoteHtml(linkChild, newChild, withText=True)
                    if filingFiles and linkChild.textValue:
                        footnoteHtml = XML("<body/>")
                        copyIxFootnoteHtml(linkChild, footnoteHtml)
                        for elt in footnoteHtml.iter():
                            if elt.tag in ("a", "img"):
                                for attrTag, attrValue in elt.items():
                                    if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrvalue):
                                        filingFiles.add(attrValue)
        
    targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip)
    modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def _make_cache(val, ugt, cntlr, ugt_default_dimensions_json_file):
    """
    Creates a new caches for the Taxonomy default dimensions

    :param val: ValidateXbrl to be validated
    :type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
    :param ugt: Taxonomy to check
    :type ugt: str
    :param ugt_default_dimensions_json_file: location to save json default
        dimensions
    :type ugt_default_dimensions_json_file: str
    :return: no explicit return, but saves caches for dqc_us_0041
    :rtype: None
    """
    started_at = time.time()
    ugt_entry_xsd = ugt["entryXsd"]
    val.usgaapDefaultDimensions = {}
    prior_validate_disclosure_system = (
        val.modelXbrl.modelManager.validateDisclosureSystem
    )
    val.modelXbrl.modelManager.validateDisclosureSystem = False
    ugt_entry_xsd_instance = (
        ModelXbrl.load(
            val.modelXbrl.modelManager,
            openFileSource(ugt_entry_xsd, cntlr),
            _("opened us-gaap entry xsd")  # noqa
        )
    )
    val.modelXbrl.modelManager.validateDisclosureSystem = (
        prior_validate_disclosure_system
    )

    if ugt_entry_xsd_instance is None:
        val.modelXbrl.error(
            "arelle:notLoaded",
            _("US-GAAP entry xsd not loaded: %(file)s"),  # noqa
            modelXbrl=val,
            file=os.path.basename(ugt_entry_xsd)
        )

    else:
        model_relationships = (
            ugt_entry_xsd_instance.relationshipSet(
                XbrlConst.dimensionDefault
            ).modelRelationships
        )
        for default_dim_rel in model_relationships:
            if _default_dim_rel_is_instance(default_dim_rel):
                from_name = default_dim_rel.fromModelObject.name
                to_name = default_dim_rel.toModelObject.name
                val.usgaapDefaultDimensions[from_name] = to_name
        json_str = str(
            json.dumps(
                val.usgaapDefaultDimensions,
                ensure_ascii=False,
                indent=0
            )
        )  # might not be unicode in 2.7
        # 2.7 gets unicode this way
        saveFile(cntlr, ugt_default_dimensions_json_file, json_str)
        ugt_entry_xsd_instance.close()
        del ugt_entry_xsd_instance  # dereference closed modelXbrl
    val.modelXbrl.profileStat(
        _("build default dimensions cache"),  # noqa
        time.time() - started_at
    )