Example #1
0
def backgroundProfileFormula(cntlr, profileReportFile, maxRunTime, excludeCompileTime):
    from arelle import Locale, XPathParser, ValidateXbrlDimensions, ValidateFormula

    # build grammar before profiling (if this is the first pass, so it doesn't count in profile statistics)
    XPathParser.initializeParser(cntlr.modelManager)
    
    # load dimension defaults
    ValidateXbrlDimensions.loadDimensionDefaults(cntlr.modelManager)
    
    import cProfile, pstats, sys, time
    
    # a minimal validation class for formula validator parameters that are needed
    class Validate:
        def __init__(self, modelXbrl, maxRunTime):
            self.modelXbrl = modelXbrl
            self.parameters = None
            self.validateSBRNL = False
            self.maxFormulaRunTime = maxRunTime
        def close(self):
            self.__dict__.clear()
            
    val = Validate(cntlr.modelManager.modelXbrl, maxRunTime)
    formulaOptions = val.modelXbrl.modelManager.formulaOptions
    if excludeCompileTime:
        startedAt = time.time()
        cntlr.addToLog(_("pre-compiling formulas before profiling"))
        val.validateFormulaCompileOnly = True
        ValidateFormula.validate(val)
        del val.validateFormulaCompileOnly
        cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale, 
                                            _("formula pre-compiling completed in %.2f secs"), 
                                            time.time() - startedAt))
        cntlr.addToLog(_("executing formulas for profiling"))
    else:
        cntlr.addToLog(_("compiling and executing formulas for profiling"))
    startedAt = time.time()
            
    statsFile = profileReportFile + ".bin"
    cProfile.runctx("ValidateFormula.validate(val)", globals(), locals(), statsFile)
    cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale, 
                                        _("formula profiling completed in %.2f secs"), 
                                        time.time() - startedAt))
    # dereference val
    val.close()
    
    # specify a file for log
    priorStdOut = sys.stdout
    sys.stdout = open(profileReportFile, "w")

    statObj = pstats.Stats(statsFile)
    statObj.strip_dirs()
    statObj.sort_stats("time")
    statObj.print_stats()
    statObj.print_callees()
    statObj.print_callers()
    sys.stdout.flush()
    sys.stdout.close()
    del statObj
    sys.stdout = priorStdOut
    os.remove(statsFile)
Example #2
0
    def insertXbrl(self, rssItem):
        try:
            # must also have default dimensions loaded
            from arelle import ValidateXbrlDimensions
            ValidateXbrlDimensions.loadDimensionDefaults(self.modelXbrl)
            
            # must have a valid XBRL instance or document
            if self.modelXbrl.modelDocument is None:
                raise XPDBException("xpgDB:MissingXbrlDocument",
                                    _("No XBRL instance or schema loaded for this filing.")) 
            
            # at this point we determine what's in the database and provide new tables
            # requires locking most of the table structure
            self.lockTables(("dAvailableTable", "dInstance",  "dFact", "dFilingIndicator",
                             "dProcessingContext", "dProcessingFact"))
            
            self.dropTemporaryTable()
 
            startedAt = time.time()
            self.insertInstance()
            self.insertDataPoints()
            self.modelXbrl.profileStat(_("XbrlSqlDB: instance insertion"), time.time() - startedAt)
            
            startedAt = time.time()
            self.showStatus("Committing entries")
            self.commit()
            self.modelXbrl.profileStat(_("XbrlSqlDB: insertion committed"), time.time() - startedAt)
            self.showStatus("DB insertion completed", clearAfter=5000)
        except Exception as ex:
            self.showStatus("DB insertion failed due to exception", clearAfter=5000)
            raise
Example #3
0
def viewRenderedGrid(modelXbrl, tabWin, lang=None):
    modelXbrl.modelManager.showStatus(_("viewing rendering"))
    view = ViewRenderedGrid(modelXbrl, tabWin, lang)
    
    # dimension defaults required in advance of validation
    from arelle import ValidateXbrlDimensions
    ValidateXbrlDimensions.loadDimensionDefaults(view)
    
    # context menu
    menu = view.contextMenu()
    optionsMenu = Menu(view.viewFrame, tearoff=0)
    view.ignoreDimValidity = BooleanVar(value=True)
    view.ignoreDimValidity.trace("w", view.viewReloadDueToMenuAction)
    optionsMenu.add_checkbutton(label=_("Ignore Dimensional Validity"), underline=0, variable=view.ignoreDimValidity, onvalue=True, offvalue=False)
    view.xAxisChildrenFirst = BooleanVar(value=True)
    view.xAxisChildrenFirst.trace("w", view.viewReloadDueToMenuAction)
    optionsMenu.add_checkbutton(label=_("X-Axis Children First"), underline=0, variable=view.xAxisChildrenFirst, onvalue=True, offvalue=False)
    view.yAxisChildrenFirst = BooleanVar(value=False)
    view.yAxisChildrenFirst.trace("w", view.viewReloadDueToMenuAction)
    optionsMenu.add_checkbutton(label=_("Y-Axis Children First"), underline=0, variable=view.yAxisChildrenFirst, onvalue=True, offvalue=False)
    menu.add_cascade(label=_("Options"), menu=optionsMenu, underline=0)
    view.tablesMenu = Menu(view.viewFrame, tearoff=0)
    menu.add_cascade(label=_("Tables"), menu=view.tablesMenu, underline=0)
    view.tablesMenuLength = 0
    view.menuAddLangs()
    view.view()
    view.blockSelectEvent = 1
    view.blockViewModelObject = 0
    view.viewFrame.bind("<Enter>", view.cellEnter, '+')
    view.viewFrame.bind("<Leave>", view.cellLeave, '+')
Example #4
0
    def insertXbrl(self, rssItem):
        try:
            # must also have default dimensions loaded
            from arelle import ValidateXbrlDimensions
            ValidateXbrlDimensions.loadDimensionDefaults(self.modelXbrl)
            
            # must have a valid XBRL instance or document
            if self.modelXbrl.modelDocument is None:
                raise XPDBException("xpgDB:MissingXbrlDocument",
                                    _("No XBRL instance or schema loaded for this filing.")) 
            
            # at this point we determine what's in the database and provide new tables
            # requires locking most of the table structure
            self.lockTables(("dAvailableTable", "dInstance",  "dFact", "dFilingIndicator",
                             # "dProcessingContext", "dProcessingFact"
                             ))
            
            self.dropTemporaryTable()
 
            startedAt = time.time()
            self.insertInstance()
            self.insertDataPoints()
            self.modelXbrl.profileStat(_("XbrlSqlDB: instance insertion"), time.time() - startedAt)
            
            startedAt = time.time()
            self.showStatus("Committing entries")
            self.commit()
            self.modelXbrl.profileStat(_("XbrlSqlDB: insertion committed"), time.time() - startedAt)
            self.showStatus("DB insertion completed", clearAfter=5000)
        except Exception as ex:
            self.showStatus("DB insertion failed due to exception", clearAfter=5000)
            raise
Example #5
0
 def insertXbrl(self, rssItem):
     try:
         # must also have default dimensions loaded
         from arelle import ValidateXbrlDimensions
         ValidateXbrlDimensions.loadDimensionDefaults(self.modelXbrl)
         
         startedAt = time.time()
         # self.load()  this done in the verify step
         self.insertAccession(rssItem)
         self.insertDocuments()
         self.insertDataDictionary() # XML namespaces types aspects
         #self.insertRelationshipTypeSets()
         #self.insertResourceRoleSets()
         #self.insertAspectValues()
         #self.insertResources()
         self.modelXbrl.profileStat(_("XbrlPublicDB: DTS insertion"), time.time() - startedAt)
         startedAt = time.time()
         self.insertDataPoints()
         self.modelXbrl.profileStat(_("XbrlPublicDB: data points insertion"), time.time() - startedAt)
         startedAt = time.time()
         self.insertRelationshipSets()
         self.modelXbrl.profileStat(_("XbrlPublicDB: Relationships insertion"), time.time() - startedAt)
         #startedAt = time.time()
         #self.insertValidCombinations()
         #self.modelXbrl.profileStat(_("XbrlPublicDB: Valid Combinations insertion"), time.time() - startedAt)
         self.showStatus("Committing entries")
         self.commit()
         self.modelXbrl.profileStat(_("XbrlPublicDB: insertion committed"), time.time() - startedAt)
         self.showStatus("DB insertion completed", clearAfter=5000)
     except Exception as ex:
         self.showStatus("DB insertion failed due to exception", clearAfter=5000)
         raise
Example #6
0
 def start(self, tag, attrib, nsmap=None):
     mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
     mdlObj.sourceline = 1
     if self.newTree:
         self.newTree = False
         self.currentMdlObj = mdlObj
         modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
         modelXbrl.modelDocument = modelDocument # needed for incremental validation
         mdlObj.init(modelDocument)
         modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement 
         modelDocument.parserLookupName = _parserLookupName
         modelDocument.parserLookupClass = _parserLookupClass
         modelDocument.xmlRootElement = mdlObj
         modelDocument.schemaLocationElements.add(mdlObj)
         modelDocument.documentEncoding = _encoding
         modelDocument._creationSoftwareComment = creationSoftwareComment
         modelXbrl.info("streamingExtensions:streaming",
                        _("Stream processing this instance."),
                        modelObject = modelDocument)    
     else:
         self.currentMdlObj.append(mdlObj)
         self.currentMdlObj = mdlObj
         mdlObj._init()
         ns = mdlObj.namespaceURI
         ln = mdlObj.localName
         if (self.beforeInstanceStream and (
             (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
             (ns == XbrlConst.xbrli and ln in ("context", "unit")) or
             (ns not in (XbrlConst.link, XbrlConst.xbrli)))):
             self.beforeInstanceStream = False
             if _streamingExtensionsValidate:
                 instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters())
             else: # need default dimensions
                 ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
     return mdlObj
Example #7
0
def viewRenderedGrid(modelXbrl, tabWin, lang=None):
    modelXbrl.modelManager.showStatus(_("viewing rendering"))
    view = ViewRenderedGrid(modelXbrl, tabWin, lang)
    
    # dimension defaults required in advance of validation
    from arelle import ValidateXbrlDimensions
    ValidateXbrlDimensions.loadDimensionDefaults(view)
    
    # context menu
    setDefaults(view)
    menu = view.contextMenu()
    optionsMenu = Menu(view.viewFrame, tearoff=0)
    view.ignoreDimValidity.trace("w", view.viewReloadDueToMenuAction)
    optionsMenu.add_checkbutton(label=_("Ignore Dimensional Validity"), underline=0, variable=view.ignoreDimValidity, onvalue=True, offvalue=False)
    view.xAxisChildrenFirst.trace("w", view.viewReloadDueToMenuAction)
    optionsMenu.add_checkbutton(label=_("X-Axis Children First"), underline=0, variable=view.xAxisChildrenFirst, onvalue=True, offvalue=False)
    view.yAxisChildrenFirst.trace("w", view.viewReloadDueToMenuAction)
    optionsMenu.add_checkbutton(label=_("Y-Axis Children First"), underline=0, variable=view.yAxisChildrenFirst, onvalue=True, offvalue=False)
    menu.add_cascade(label=_("Options"), menu=optionsMenu, underline=0)
    view.tablesMenu = Menu(view.viewFrame, tearoff=0)
    menu.add_cascade(label=_("Tables"), menu=view.tablesMenu, underline=0)
    view.tablesMenuLength = 0
    view.menuAddLangs()
    view.menu.add_command(label=_("Save html file"), underline=0, command=lambda: view.modelXbrl.modelManager.cntlr.fileSave(view=view))
    view.view()
    view.blockSelectEvent = 1
    view.blockViewModelObject = 0
    view.viewFrame.bind("<Enter>", view.cellEnter, '+')
    view.viewFrame.bind("<Leave>", view.cellLeave, '+')
Example #8
0
 def insertXbrl(self, rssItem):
     try:
         # must also have default dimensions loaded
         from arelle import ValidateXbrlDimensions
         ValidateXbrlDimensions.loadDimensionDefaults(self.modelXbrl)
                     
         # find pre-existing documents in server database
         self.identifyPreexistingDocuments()
         self.identifyConceptsUsed()
         
         startedAt = time.time()
         self.insertAccession(rssItem)
         self.insertUris()
         self.insertQnames()
         self.insertNamespaces()
         self.insertDocuments()
         self.insertCustomArcroles()
         self.insertCustomRoles()
         self.insertElements()
         self.insertResources()
         self.insertNetworks()
         self.modelXbrl.profileStat(_("XbrlPublicDB: DTS insertion"), time.time() - startedAt)
         startedAt = time.time()
         self.insertFacts()
         self.modelXbrl.profileStat(_("XbrlPublicDB: instance insertion"), time.time() - startedAt)
         startedAt = time.time()
         self.showStatus("Committing entries")
         self.commit()
         self.modelXbrl.profileStat(_("XbrlPublicDB: insertion committed"), time.time() - startedAt)
         self.showStatus("DB insertion completed", clearAfter=5000)
     except Exception as ex:
         self.showStatus("DB insertion failed due to exception", clearAfter=5000)
         raise
 def start(self, tag, attrib, nsmap=None):
     mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
     mdlObj.sourceline = 1
     if self.newTree:
         self.newTree = False
         self.currentMdlObj = mdlObj
         modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
         modelXbrl.modelDocument = modelDocument # needed for incremental validation
         mdlObj.init(modelDocument)
         modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement 
         modelDocument.parserLookupName = _parserLookupName
         modelDocument.parserLookupClass = _parserLookupClass
         modelDocument.xmlRootElement = mdlObj
         modelDocument.schemaLocationElements.add(mdlObj)
         modelDocument.documentEncoding = _encoding
         modelDocument._creationSoftwareComment = creationSoftwareComment
         modelXbrl.info("streamingExtensions:streaming",
                        _("Stream processing this instance."),
                        modelObject = modelDocument)    
     else:
         self.currentMdlObj.append(mdlObj)
         self.currentMdlObj = mdlObj
         mdlObj._init()
         ns = mdlObj.namespaceURI
         ln = mdlObj.localName
         if (self.beforeInstanceStream and (
             (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
             (ns == XbrlConst.xbrli and ln in ("context", "unit")) or
             (ns not in (XbrlConst.link, XbrlConst.xbrli)))):
             self.beforeInstanceStream = False
             if _streamingExtensionsValidate:
                 instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters())
             else: # need default dimensions
                 ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
     return mdlObj
Example #10
0
def backgroundProfileFormula(cntlr, profileReportFile, maxRunTime, excludeCompileTime):
    from arelle import Locale, XPathParser, ValidateXbrlDimensions, ValidateFormula

    # build grammar before profiling (if this is the first pass, so it doesn't count in profile statistics)
    XPathParser.initializeParser(cntlr.modelManager)
    
    # load dimension defaults
    ValidateXbrlDimensions.loadDimensionDefaults(cntlr.modelManager)
    
    import cProfile, pstats, sys, time
    
    # a minimal validation class for formula validator parameters that are needed
    class Validate:
        def __init__(self, modelXbrl, maxRunTime):
            self.modelXbrl = modelXbrl
            self.parameters = None
            self.validateSBRNL = False
            self.maxFormulaRunTime = maxRunTime
        def close(self):
            self.__dict__.clear()
            
    val = Validate(cntlr.modelManager.modelXbrl, maxRunTime)
    formulaOptions = val.modelXbrl.modelManager.formulaOptions
    if excludeCompileTime:
        startedAt = time.time()
        cntlr.addToLog(_("pre-compiling formulas before profiling"))
        val.validateFormulaCompileOnly = True
        ValidateFormula.validate(val)
        del val.validateFormulaCompileOnly
        cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale, 
                                            _("formula pre-compiling completed in %.2f secs"), 
                                            time.time() - startedAt))
        cntlr.addToLog(_("executing formulas for profiling"))
    else:
        cntlr.addToLog(_("compiling and executing formulas for profiling"))
    startedAt = time.time()
            
    statsFile = profileReportFile + ".bin"
    cProfile.runctx("ValidateFormula.validate(val)", globals(), locals(), statsFile)
    cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale, 
                                        _("formula profiling completed in %.2f secs"), 
                                        time.time() - startedAt))
    # dereference val
    val.close()
    
    # specify a file for log
    priorStdOut = sys.stdout
    sys.stdout = open(profileReportFile, "w")

    statObj = pstats.Stats(statsFile)
    statObj.strip_dirs()
    statObj.sort_stats("time")
    statObj.print_stats()
    statObj.print_callees()
    statObj.print_callers()
    sys.stdout.flush()
    sys.stdout.close()
    del statObj
    sys.stdout = priorStdOut
    os.remove(statsFile)
    def saveTargetDocument(self):
        targetUrl = self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
            self.targetDocumentPreferredFilename, self.filepath)
        targetUrlParts = targetUrl.rpartition(".")
        targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
        self.modelXbrl.modelManager.showStatus(
            _("Extracting instance ") + os.path.basename(targetUrl))
        targetInstance = ModelXbrl.create(
            self.modelXbrl.modelManager,
            newDocumentType=Type.INSTANCE,
            url=targetUrl,
            schemaRefs=self.targetDocumentSchemaRefs,
            isEntry=True)
        ValidateXbrlDimensions.loadDimensionDefaults(
            targetInstance)  # need dimension defaults
        for context in self.modelXbrl.contexts.values():
            newCntx = targetInstance.createContext(
                context.entityIdentifier[0],
                context.entityIdentifier[1],
                'instant' if context.isInstantPeriod else
                'duration' if context.isStartEndPeriod else 'forever',
                context.startDatetime,
                context.endDatetime,
                None,
                context.qnameDims, [], [],
                id=context.id)
        for unit in self.modelXbrl.units.values():
            measures = unit.measures
            newUnit = targetInstance.createUnit(measures[0],
                                                measures[1],
                                                id=unit.id)

        self.modelXbrl.modelManager.showStatus(
            _("Creating and validating facts"))
        for fact in self.modelXbrl.facts:
            if fact.isItem:
                attrs = [("contextRef", fact.contextID)]
                if fact.isNumeric:
                    attrs.append(("unitRef", fact.unitID))
                    if fact.get("decimals"):
                        attrs.append(("decimals", fact.get("decimals")))
                    if fact.get("precision"):
                        attrs.append(("precision", fact.get("precision")))
                if fact.isNil:
                    attrs.append(
                        ("{http://www.w3.org/2001/XMLSchema-instance}nil",
                         "true"))
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.elementText
                newFact = targetInstance.createFact(fact.qname,
                                                    attributes=attrs,
                                                    text=text)
        targetInstance.saveInstance(overrideFilepath=targetUrl)
        self.modelXbrl.modelManager.showStatus(_("Saved extracted instance"),
                                               5000)
Example #12
0
def viewRenderedGrid(modelXbrl, tabWin, lang=None):
    modelXbrl.modelManager.showStatus(_("viewing rendering"))
    view = ViewRenderedGrid(modelXbrl, tabWin, lang)

    # dimension defaults required in advance of validation
    from arelle import ValidateXbrlDimensions
    ValidateXbrlDimensions.loadDimensionDefaults(view)

    # context menu
    setDefaults(view)
    menu = view.contextMenu()
    optionsMenu = Menu(view.viewFrame, tearoff=0)
    optionsMenu.add_command(
        label=_("New fact item options"),
        underline=0,
        command=lambda: getNewFactItemOptions(modelXbrl.modelManager.cntlr,
                                              view.newFactItemOptions))
    view.ignoreDimValidity.trace("w", view.viewReloadDueToMenuAction)
    optionsMenu.add_checkbutton(label=_("Ignore Dimensional Validity"),
                                underline=0,
                                variable=view.ignoreDimValidity,
                                onvalue=True,
                                offvalue=False)
    view.xAxisChildrenFirst.trace("w", view.viewReloadDueToMenuAction)
    optionsMenu.add_checkbutton(label=_("X-Axis Children First"),
                                underline=0,
                                variable=view.xAxisChildrenFirst,
                                onvalue=True,
                                offvalue=False)
    view.yAxisChildrenFirst.trace("w", view.viewReloadDueToMenuAction)
    optionsMenu.add_checkbutton(label=_("Y-Axis Children First"),
                                underline=0,
                                variable=view.yAxisChildrenFirst,
                                onvalue=True,
                                offvalue=False)
    menu.add_cascade(label=_("Options"), menu=optionsMenu, underline=0)
    view.tablesMenu = Menu(view.viewFrame, tearoff=0)
    menu.add_cascade(label=_("Tables"), menu=view.tablesMenu, underline=0)
    view.tablesMenuLength = 0
    view.menuAddLangs()
    saveMenu = Menu(view.viewFrame, tearoff=0)
    saveMenu.add_command(label=_("HTML file"),
                         underline=0,
                         command=lambda: view.modelXbrl.modelManager.cntlr.
                         fileSave(view=view, fileType="html"))
    saveMenu.add_command(label=_("XBRL instance"),
                         underline=0,
                         command=view.saveInstance)
    menu.add_cascade(label=_("Save"), menu=saveMenu, underline=0)
    view.view()
    view.blockSelectEvent = 1
    view.blockViewModelObject = 0
    view.viewFrame.bind("<Enter>", view.cellEnter, '+')
    view.viewFrame.bind("<Leave>", view.cellLeave, '+')
Example #13
0
def viewRenderedGrid(modelXbrl, outfile, lang=None, viewTblELR=None, sourceView=None):
    modelXbrl.modelManager.showStatus(_("viewing rendering"))
    view = ViewRenderedGrid(modelXbrl, outfile, lang)
    
    # dimension defaults required in advance of validation
    from arelle import ValidateXbrlDimensions
    ValidateXbrlDimensions.loadDimensionDefaults(view)
    setDefaults(view)
    if sourceView is not None:
        viewTblELR = sourceView.tblELR
        view.ignoreDimValidity.set(sourceView.ignoreDimValidity.get())
        view.xAxisChildrenFirst.set(sourceView.xAxisChildrenFirst.get())
        view.yAxisChildrenFirst.set(sourceView.yAxisChildrenFirst.get())
    view.view(viewTblELR)    
    view.close()
Example #14
0
    def insertXbrl(self, rssItem):
        try:
            # must also have default dimensions loaded
            from arelle import ValidateXbrlDimensions
            ValidateXbrlDimensions.loadDimensionDefaults(self.modelXbrl)

            # obtain supplementaion entity information
            self.entityInformation = loadEntityInformation(
                self.modelXbrl, rssItem)
            # identify table facts (table datapoints) (prior to locked database transaction
            self.tableFacts = tableFacts(
                self.modelXbrl
            )  # for EFM & HMRC this is ( (roleType, table_code, fact) )
            loadPrimaryDocumentFacts(
                self.modelXbrl, rssItem, self.entityInformation
            )  # load primary document facts for SEC filing

            # find pre-existing documents in server database
            self.identifyPreexistingDocuments()
            self.identifyConceptsUsed()

            startedAt = time.time()
            self.insertAccession(rssItem)
            self.insertUris()
            self.insertQnames()
            self.insertNamespaces()
            self.insertDocuments()
            self.insertCustomArcroles()
            self.insertCustomRoles()
            self.insertElements()
            self.insertResources()
            self.insertNetworks()
            self.modelXbrl.profileStat(_("XbrlPublicDB: DTS insertion"),
                                       time.time() - startedAt)
            startedAt = time.time()
            self.insertFacts()
            self.modelXbrl.profileStat(_("XbrlPublicDB: instance insertion"),
                                       time.time() - startedAt)
            startedAt = time.time()
            self.showStatus("Committing entries")
            self.commit()
            self.modelXbrl.profileStat(_("XbrlPublicDB: insertion committed"),
                                       time.time() - startedAt)
            self.showStatus("DB insertion completed", clearAfter=5000)
        except Exception as ex:
            self.showStatus("DB insertion failed due to exception",
                            clearAfter=5000)
            raise
Example #15
0
 def insertXbrl(self, rssItem):
     try:
         # must also have default dimensions loaded
         from arelle import ValidateXbrlDimensions
         ValidateXbrlDimensions.loadDimensionDefaults(self.modelXbrl)
         
         #initialVcount, initialEcount = self.getDBsize() # don't include in timing, very slow
         startedAt = time.time()
         
         # find pre-existing documents in server database
         self.identifyPreexistingDocuments()
         
         g = {FILINGS:{},
              DOCUMENTS:{}}
         self.insertSchema(g)
         
         # self.load()  this done in the verify step
         self.insertFiling(rssItem,g)
         self.insertDocuments(g)
         self.insertDataDictionary() # XML namespaces types aspects
         #self.insertRelationshipTypeSets()
         #self.insertResourceRoleSets()
         #self.insertAspectValues()
         self.modelXbrl.profileStat(_("XbrlSemanticJsonDB: DTS insertion"), time.time() - startedAt)
         startedAt = time.time()
         self.insertDataPoints()
         self.modelXbrl.profileStat(_("XbrlSemanticJsonDB: data points insertion"), time.time() - startedAt)
         startedAt = time.time()
         self.insertRelationshipSets()
         self.modelXbrl.profileStat(_("XbrlSemanticJsonDB: Relationships insertion"), time.time() - startedAt)
         self.insertValidationResults()
         self.modelXbrl.profileStat(_("XbrlSemanticJsonDB: Validation results insertion"), time.time() - startedAt)
         #startedAt = time.time()
         #self.insertValidCombinations()
         #self.modelXbrl.profileStat(_("XbrlSemanticJsonDB: Valid Combinations insertion"), time.time() - startedAt)
         self.showStatus("Committing entries")
         self.commit(g)
         self.modelXbrl.profileStat(_("XbrlSemanticJsonDB: insertion committed"), time.time() - startedAt)
         #finalVcount, finalEcount = self.getDBsize()
         #self.modelXbrl.modelManager.addToLog("added vertices: {0}, edges: {1}, total vertices: {2}, edges: {3}".format(
         #              finalVcount - initialVcount, finalEcount - initialEcount, finalVcount, finalEcount))
         self.showStatus("DB insertion completed", clearAfter=5000)
     except Exception as ex:
         self.showStatus("DB insertion failed due to exception", clearAfter=5000)
         raise
    def saveTargetDocument(self):
        targetUrl = self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(self.targetDocumentPreferredFilename, self.filepath)
        targetUrlParts = targetUrl.rpartition(".")
        targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
        self.modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
        targetInstance = ModelXbrl.create(self.modelXbrl.modelManager, 
                                          newDocumentType=Type.INSTANCE,
                                          url=targetUrl,
                                          schemaRefs=self.targetDocumentSchemaRefs,
                                          isEntry=True)
        ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults 
        for context in self.modelXbrl.contexts.values():
            newCntx = targetInstance.createContext(context.entityIdentifier[0],
                                                   context.entityIdentifier[1],
                                                   'instant' if context.isInstantPeriod else
                                                   'duration' if context.isStartEndPeriod
                                                   else 'forever',
                                                   context.startDatetime,
                                                   context.endDatetime,
                                                   None, 
                                                   context.qnameDims, [], [],
                                                   id=context.id)
        for unit in self.modelXbrl.units.values():
            measures = unit.measures
            newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id)

        self.modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
        for fact in self.modelXbrl.facts:
            if fact.isItem:
                attrs = [("contextRef", fact.contextID)]
                if fact.isNumeric:
                    attrs.append(("unitRef", fact.unitID))
                    if fact.get("decimals"):
                        attrs.append(("decimals", fact.get("decimals")))
                    if fact.get("precision"):
                        attrs.append(("precision", fact.get("precision")))
                if fact.isNil:
                    attrs.append(("{http://www.w3.org/2001/XMLSchema-instance}nil","true"))
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.elementText
                newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text)
        targetInstance.saveInstance(overrideFilepath=targetUrl)
        self.modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
Example #17
0
def viewRenderedGrid(modelXbrl,
                     outfile,
                     lang=None,
                     viewTblELR=None,
                     sourceView=None):
    modelXbrl.modelManager.showStatus(_("viewing rendering"))
    view = ViewRenderedGrid(modelXbrl, outfile, lang)

    # dimension defaults required in advance of validation
    from arelle import ValidateXbrlDimensions
    ValidateXbrlDimensions.loadDimensionDefaults(view)
    setDefaults(view)
    if sourceView is not None:
        viewTblELR = sourceView.tblELR
        view.ignoreDimValidity.set(sourceView.ignoreDimValidity.get())
        view.xAxisChildrenFirst.set(sourceView.xAxisChildrenFirst.get())
        view.yAxisChildrenFirst.set(sourceView.yAxisChildrenFirst.get())
    view.view(viewTblELR)
    view.close()
 def insertXbrl(self, rssItem):
     try:
         # must also have default dimensions loaded
         from arelle import ValidateXbrlDimensions
         ValidateXbrlDimensions.loadDimensionDefaults(self.modelXbrl)
         
         # get logging entries (needed to find which aspects to identify)
         self.loggingEntries = []
         for handler in logging.getLogger("arelle").handlers:
             if hasattr(handler, "dbHandlerLogEntries"):
                 self.loggingEntries = handler.dbHandlerLogEntries()
                 break
                     
         # find pre-existing documents in server database
         self.identifyPreexistingDocuments()
         self.identifyAspectsUsed()
         
         startedAt = time.time()
         self.dropTemporaryTable()
         self.insertFiling(rssItem)
         self.insertDocuments()
         self.insertAspects()
         self.insertArcroleTypes()
         self.insertRoleTypes()
         self.insertResources()
         self.insertRelationships()
         self.modelXbrl.profileStat(_("XbrlSqlDB: DTS insertion"), time.time() - startedAt)
         startedAt = time.time()
         self.insertDataPoints()
         self.modelXbrl.profileStat(_("XbrlSqlDB: instance insertion"), time.time() - startedAt)
         self.insertValidationResults()
         self.modelXbrl.profileStat(_("XbrlSqlDB: Validation results insertion"), time.time() - startedAt)
         startedAt = time.time()
         self.showStatus("Committing entries")
         self.commit()
         self.modelXbrl.profileStat(_("XbrlSqlDB: insertion committed"), time.time() - startedAt)
         self.showStatus("DB insertion completed", clearAfter=5000)
     except Exception as ex:
         self.showStatus("DB insertion failed due to exception", clearAfter=5000)
         raise
 def insertXbrl(self, rssItem):
     try:
         # must also have default dimensions loaded
         from arelle import ValidateXbrlDimensions
         ValidateXbrlDimensions.loadDimensionDefaults(self.modelXbrl)
                     
         # obtain supplementaion entity information
         self.entityInformation = loadEntityInformation(self.modelXbrl, rssItem)
         # identify table facts (table datapoints) (prior to locked database transaction
         self.tableFacts = tableFacts(self.modelXbrl)  # for EFM & HMRC this is ( (roleType, table_code, fact) )
         loadPrimaryDocumentFacts(self.modelXbrl, rssItem, self.entityInformation) # load primary document facts for SEC filing
         
         # find pre-existing documents in server database
         self.identifyPreexistingDocuments()
         self.identifyConceptsUsed()
         
         startedAt = time.time()
         self.insertAccession(rssItem)
         self.insertUris()
         self.insertQnames()
         self.insertNamespaces()
         self.insertDocuments()
         self.insertCustomArcroles()
         self.insertCustomRoles()
         self.insertElements()
         self.insertResources()
         self.insertNetworks()
         self.modelXbrl.profileStat(_("XbrlPublicDB: DTS insertion"), time.time() - startedAt)
         startedAt = time.time()
         self.insertFacts()
         self.modelXbrl.profileStat(_("XbrlPublicDB: instance insertion"), time.time() - startedAt)
         startedAt = time.time()
         self.showStatus("Committing entries")
         self.commit()
         self.modelXbrl.profileStat(_("XbrlPublicDB: insertion committed"), time.time() - startedAt)
         self.showStatus("DB insertion completed", clearAfter=5000)
     except Exception as ex:
         self.showStatus("DB insertion failed due to exception", clearAfter=5000)
         raise
Example #20
0
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
    from openpyxl import load_workbook
    from arelle import ModelDocument, ModelXbrl, XmlUtil
    from arelle.ModelDocument import ModelDocumentReference
    from arelle.ModelValue import qname

    try:
        currentAction = "initializing"
        startingErrorCount = len(modelXbrl.errors)
        startedAt = time.time()

        if os.path.isabs(oimFile):
            # allow relative filenames to loading directory
            priorCWD = os.getcwd()
            os.chdir(os.path.dirname(oimFile))
        else:
            priorCWD = None

        currentAction = "determining file type"
        isJSON = oimFile.endswith(
            ".json") and not oimFile.endswith("-metadata.json")
        isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
        isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
        isCSVorXL = isCSV or isXL
        instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"

        if isJSON:
            currentAction = "loading and parsing JSON OIM file"

            def loadDict(keyValuePairs):
                _dict = OrderedDict(
                )  # preserve fact order in resulting instance
                for key, value in keyValuePairs:
                    if isinstance(value, dict):
                        if DUPJSONKEY in value:
                            for _errKey, _errValue, _otherValue in value[
                                    DUPJSONKEY]:
                                if key == "prefixes":
                                    modelXbrl.error(
                                        "oime:duplicatedPrefix",
                                        _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"
                                          ),
                                        modelObject=modelXbrl,
                                        prefix=_errKey,
                                        uri1=_errValue,
                                        uri2=_otherValue)
                            del value[DUPJSONKEY]
                    if key in _dict:
                        if DUPJSONKEY not in _dict:
                            _dict[DUPJSONKEY] = []
                        _dict[DUPJSONKEY].append((key, value, _dict[key]))
                    else:
                        _dict[key] = value
                return _dict

            with io.open(oimFile, 'rt', encoding='utf-8') as f:
                oimObject = json.load(f, object_pairs_hook=loadDict)
            missing = [
                t for t in ("dtsReferences", "prefixes", "facts")
                if t not in oimObject
            ]
            if missing:
                raise OIMException(
                    "oime:missingJSONElements",
                    _("Required element(s) are missing from JSON input: %(missing)s"
                      ),
                    missing=", ".join(missing))
            currentAction = "identifying JSON objects"
            dtsReferences = oimObject["dtsReferences"]
            prefixesList = oimObject["prefixes"].items()
            facts = oimObject["facts"]
            footnotes = oimObject["facts"]  # shares this object
        elif isCSV:
            currentAction = "identifying CSV input tables"
            if sys.version[0] >= '3':
                csvOpenMode = 'w'
                csvOpenNewline = ''
            else:
                csvOpenMode = 'wb'  # for 2.7
                csvOpenNewline = None

            oimFileBase = None
            if "-facts" in oimFile:
                oimFileBase = oimFile.partition("-facts")[0]
            else:
                for suffix in ("-dtsReferences.csv", "-defaults.csv",
                               "-prefixes.csv", "-footnotes.csv",
                               "-metadata.json"):
                    if oimFile.endswith(suffix):
                        oimFileBase = oimFile[:-len(suffix)]
                        break
            if oimFileBase is None:
                raise OIMException(
                    "oime:missingCSVTables",
                    _("Unable to identify CSV tables file name pattern"))
            if (not os.path.exists(oimFileBase + "-dtsReferences.csv")
                    or not os.path.exists(oimFileBase + "-prefixes.csv")):
                raise OIMException(
                    "oime:missingCSVTables",
                    _("Unable to identify CSV tables for dtsReferences or prefixes"
                      ))
            instanceFileName = oimFileBase + ".xbrl"
            currentAction = "loading CSV dtsReferences table"
            dtsReferences = []
            with io.open(oimFileBase + "-dtsReferences.csv",
                         'rt',
                         encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = row
                    else:
                        dtsReferences.append(
                            dict(
                                (header[j], col) for j, col in enumerate(row)))
            currentAction = "loading CSV prefixes table"
            prefixesList = []
            with io.open(oimFileBase + "-prefixes.csv",
                         'rt',
                         encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = dict((col, i) for i, col in enumerate(row))
                    else:
                        prefixesList.append(
                            (row[header["prefix"]], row[header["URI"]]))
            defaults = {}
            if os.path.exists(oimFileBase + "-defaults.csv"):
                currentAction = "loading CSV defaults table"
                with io.open(oimFileBase + "-defaults.csv",
                             'rt',
                             encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                            fileCol = row.index("file")
                        else:
                            defaults[row[fileCol]] = dict(
                                (header[j], col) for j, col in enumerate(row)
                                if j != fileCol)
            currentAction = "loading CSV facts tables"
            facts = []
            _dir = os.path.dirname(oimFileBase)
            factsFileBasename = os.path.basename(oimFileBase) + "-facts"
            for filename in os.listdir(_dir):
                filepath = os.path.join(_dir, filename)
                if filename.startswith(factsFileBasename):
                    currentAction = "loading CSV facts table {}".format(
                        filename)
                    tableDefaults = defaults.get(filename, {})
                    with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
                        csvReader = csv.reader(f)
                        for i, row in enumerate(csvReader):
                            if i == 0:
                                header = row
                            else:
                                fact = {}
                                fact.update(tableDefaults)
                                for j, col in enumerate(row):
                                    if col is not None:
                                        if header[
                                                j]:  # skip cols with no header
                                            if header[j].endswith("Value"):
                                                if col:  # ignore empty columns (= null CSV value)
                                                    fact["value"] = col
                                            else:
                                                fact[header[j]] = col
                                facts.append(fact)
            footnotes = []
            if os.path.exists(oimFileBase + "-footnotes.csv"):
                currentAction = "loading CSV footnotes table"
                with io.open(oimFileBase + "-footnotes.csv",
                             'rt',
                             encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                        else:
                            footnotes.append(
                                dict((header[j], col)
                                     for j, col in enumerate(row) if col))
        elif isXL:
            currentAction = "identifying workbook input worksheets"
            oimWb = load_workbook(oimFile, read_only=True, data_only=True)
            sheetNames = oimWb.get_sheet_names()
            if (not any(sheetName == "prefixes" for sheetName in sheetNames)
                    or not any(sheetName == "dtsReferences"
                               for sheetName in sheetNames)
                    or not any("facts" in sheetName
                               for sheetName in sheetNames)):
                raise OIMException(
                    "oime:missingWorkbookWorksheets",
                    _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts"
                      ))
            currentAction = "loading worksheet: dtsReferences"
            dtsReferences = []
            for i, row in enumerate(oimWb["dtsReferences"]):
                if i == 0:
                    header = [col.value for col in row]
                else:
                    dtsReferences.append(
                        dict((header[j], col.value)
                             for j, col in enumerate(row)))
            currentAction = "loading worksheet: prefixes"
            prefixesList = []
            for i, row in enumerate(oimWb["prefixes"]):
                if i == 0:
                    header = dict((col.value, i) for i, col in enumerate(row))
                else:
                    prefixesList.append((row[header["prefix"]].value,
                                         row[header["URI"]].value))
            defaults = {}
            if "defaults" in sheetNames:
                currentAction = "loading worksheet: defaults"
                for i, row in enumerate(oimWb["defaults"]):
                    if i == 0:
                        header = dict(
                            (col.value, i) for i, col in enumerate(row))
                        fileCol = header["file"]
                    else:
                        defaults[row[fileCol].value] = dict(
                            (header[j], col.value) for j, col in enumerate(row)
                            if j != fileCol)
            facts = []
            for sheetName in sheetNames:
                if sheetName == "facts" or "-facts" in sheetName:
                    currentAction = "loading worksheet: {}".format(sheetName)
                    tableDefaults = defaults.get(sheetName, {})
                    for i, row in enumerate(oimWb[sheetName]):
                        if i == 0:
                            header = [col.value for col in row]
                        else:
                            fact = {}
                            fact.update(tableDefaults)
                            for j, col in enumerate(row):
                                if col.value is not None:
                                    if header[j]:  # skip cols with no header
                                        if header[j].endswith("Value"):
                                            fact["value"] = str(col.value)
                                        else:
                                            fact[header[j]] = str(col.value)
                            facts.append(fact)
            footnotes = []
            if "footnotes" in sheetNames:
                currentAction = "loading worksheet: footnotes"
                for i, row in enumerate(oimWb["footnotes"]):
                    if i == 0:
                        header = dict((col.value, i)
                                      for i, col in enumerate(row)
                                      if col.value)
                    else:
                        footnotes.append(
                            dict((header[j], col.value)
                                 for j, col in enumerate(row) if col.value))

        currentAction = "identifying default dimensions"
        ValidateXbrlDimensions.loadDimensionDefaults(
            modelXbrl)  # needs dimension defaults

        currentAction = "validating OIM"
        prefixes = {}
        prefixedUris = {}
        for _prefix, _uri in prefixesList:
            if not _prefix:
                modelXbrl.error(
                    "oime:emptyPrefix",
                    _("The empty string must not be used as a prefix, uri %(uri)s"
                      ),
                    modelObject=modelXbrl,
                    uri=_uri)
            elif not NCNamePattern.match(_prefix):
                modelXbrl.error(
                    "oime:prefixPattern",
                    _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"
                      ),
                    modelObject=modelXbrl,
                    prefix=_prefix,
                    uri=_uri)
            elif _prefix in prefixes:
                modelXbrl.error(
                    "oime:duplicatedPrefix",
                    _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"
                      ),
                    modelObject=modelXbrl,
                    prefix=_prefix,
                    uri1=prefixes[_prefix],
                    uri2=_uri)
            elif _uri in prefixedUris:
                modelXbrl.error(
                    "oime:duplicatedUri",
                    _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"
                      ),
                    modelObject=modelXbrl,
                    uri=_uri,
                    prefix1=prefixedUris[_uri],
                    prefix2=_prefix)
            else:
                prefixes[_prefix] = _uri
                prefixedUris[_uri] = _prefix

        oimPrefix = None
        for _nsOim in nsOim:
            if _nsOim in prefixedUris:
                oimPrefix = prefixedUris[_nsOim]
        if not oimPrefix:
            raise OIMException(
                "oime:noOimPrefix",
                _("The oim namespace must have a declared prefix"))
        oimConcept = "{}:concept".format(oimPrefix)
        oimEntity = "{}:entity".format(oimPrefix)
        oimPeriod = "{}:period".format(oimPrefix)
        oimPeriodStart = "{}:periodStart".format(oimPrefix)
        oimPeriodEnd = "{}:periodEnd".format(oimPrefix)
        oimUnit = "{}:unit".format(oimPrefix)
        oimPrefix = "{}:".format(oimPrefix)

        # create the instance document
        currentAction = "creating instance document"
        modelXbrl.blockDpmDBrecursion = True
        modelXbrl.modelDocument = createModelDocument(
            modelXbrl,
            Type.INSTANCE,
            instanceFileName,
            schemaRefs=[
                dtsRef["href"] for dtsRef in dtsReferences
                if dtsRef["type"] == "schema"
            ],
            isEntry=True,
            initialComment="extracted from OIM {}".format(mappedUri),
            documentEncoding="utf-8")
        cntxTbl = {}
        unitTbl = {}
        currentAction = "creating facts"
        for fact in facts:
            conceptQn = qname(fact[oimConcept], prefixes)
            concept = modelXbrl.qnameConcepts.get(conceptQn)
            entityAsQn = qname(fact[oimEntity], prefixes)
            if oimPeriod in fact:
                periodStart = fact[oimPeriod]["start"]
                periodEnd = fact[oimPeriod]["end"]
            else:
                periodStart = fact[oimPeriodStart]
                periodEnd = fact[oimPeriodEnd]
            cntxKey = (  # hashable context key
                ("periodType", concept.periodType), ("entity", entityAsQn),
                ("periodStart", periodStart),
                ("periodEnd", periodEnd)) + tuple(
                    sorted(
                        (dimName, dimVal)
                        for dimName, dimVal in fact.items() if ":" in dimName
                        and not dimName.startswith(oimPrefix)))
            if cntxKey in cntxTbl:
                _cntx = cntxTbl[cntxKey]
            else:
                cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                qnameDims = {}
                for dimName, dimVal in fact.items():
                    if ":" in dimName and not dimName.startswith(
                            oimPrefix) and dimVal:
                        dimQname = qname(dimName, prefixes)
                        dimConcept = modelXbrl.qnameConcepts.get(dimQname)
                        if ":" in dimVal and dimVal.partition(
                                ':')[0] in prefixes:
                            mem = qname(dimVal, prefixes)  # explicit dim
                        elif dimConcept.isTypedDimension:
                            # a modelObject xml element is needed for all of the instance functions to manage the typed dim
                            mem = addChild(modelXbrl.modelDocument,
                                           dimConcept.typedDomainElement.qname,
                                           text=dimVal,
                                           appendChild=False)
                        qnameDims[dimQname] = DimValuePrototype(
                            modelXbrl, None, dimQname, mem, "segment")
                _cntx = modelXbrl.createContext(
                    entityAsQn.namespaceURI,
                    entityAsQn.localName,
                    concept.periodType,
                    None if concept.periodType == "instant" else dateTime(
                        periodStart, type=DATETIME),
                    dateTime(periodEnd, type=DATETIME),
                    None,  # no dimensional validity checking (like formula does)
                    qnameDims,
                    [],
                    [],
                    id=cntxId)
                cntxTbl[cntxKey] = _cntx
            if oimUnit in fact:
                unitKey = fact[oimUnit]
                if unitKey in unitTbl:
                    _unit = unitTbl[unitKey]
                else:
                    _unit = None
                    # validate unit
                    unitKeySub = PrefixedQName.sub(
                        UnitPrefixedQNameSubstitutionChar, unitKey)
                    if not UnitPattern.match(unitKeySub):
                        modelXbrl.error(
                            "oime:unitStringRepresentation",
                            _("Unit string representation is lexically invalid, %(unit)s"
                              ),
                            modelObject=modelXbrl,
                            unit=unitKey)
                    else:
                        _mul, _sep, _div = unitKey.partition('/')
                        if _mul.startswith('('):
                            _mul = _mul[1:-1]
                        _muls = [u for u in _mul.split('*') if u]
                        if _div.startswith('('):
                            _div = _div[1:-1]
                        _divs = [u for u in _div.split('*') if u]
                        if _muls != sorted(_muls) or _divs != sorted(_divs):
                            modelXbrl.error(
                                "oime:unitStringRepresentation",
                                _("Unit string representation measures are not in alphabetical order, %(unit)s"
                                  ),
                                modelObject=modelXbrl,
                                unit=unitKey)
                        try:
                            mulQns = [
                                qname(
                                    u,
                                    prefixes,
                                    prefixException=OIMException(
                                        "oime:unitPrefix",
                                        _("Unit prefix is not declared: %(unit)s"
                                          ),
                                        unit=u)) for u in _muls
                            ]
                            divQns = [
                                qname(
                                    u,
                                    prefixes,
                                    prefixException=OIMException(
                                        "oime:unitPrefix",
                                        _("Unit prefix is not declared: %(unit)s"
                                          ),
                                        unit=u)) for u in _divs
                            ]
                            unitId = 'u-{:02}'.format(len(unitTbl) + 1)
                            for _measures in mulQns, divQns:
                                for _measure in _measures:
                                    addQnameValue(modelXbrl.modelDocument,
                                                  _measure)
                            _unit = modelXbrl.createUnit(mulQns,
                                                         divQns,
                                                         id=unitId)
                        except OIMException as ex:
                            modelXbrl.error(ex.code,
                                            ex.message,
                                            modelObject=modelXbrl,
                                            **ex.msgArgs)
                    unitTbl[unitKey] = _unit
            else:
                _unit = None

            attrs = {"contextRef": _cntx.id}

            if fact.get("value") is None:
                attrs[XbrlConst.qnXsiNil] = "true"
                text = None
            else:
                text = fact["value"]

            if fact.get("id"):
                attrs["id"] = fact["id"]

            if concept.isNumeric:
                if _unit is None:
                    continue  # skip creating fact because unit was invalid
                attrs["unitRef"] = _unit.id
                if "accuracy" in fact:
                    attrs["decimals"] = fact["accuracy"]

            # is value a QName?
            if concept.baseXbrliType == "QName":
                addQnameValue(modelXbrl.modelDocument,
                              qname(text.strip(), prefixes))

            f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)

        currentAction = "creating footnotes"
        footnoteLinks = {}  # ELR elements
        factLocs = {}  # index by (linkrole, factId)
        footnoteNbr = 0
        locNbr = 0
        for factOrFootnote in footnotes:
            if "factId" in factOrFootnote:
                factId = factOrFootnote["factId"]
                factFootnotes = (factOrFootnote, )  # CSV or XL
            elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
                factId = factOrFootnote["id"]
                factFootnotes = factOrFootnote["footnotes"]
            else:
                factFootnotes = ()
            for footnote in factFootnotes:
                linkrole = footnote.get("group")
                arcrole = footnote.get("footnoteType")
                if not factId or not linkrole or not arcrole or not (
                        footnote.get("factRef") or footnote.get("footnote")):
                    # invalid footnote
                    continue
                if linkrole not in footnoteLinks:
                    footnoteLinks[linkrole] = addChild(
                        modelXbrl.modelDocument.xmlRootElement,
                        XbrlConst.qnLinkFootnoteLink,
                        attributes={
                            "{http://www.w3.org/1999/xlink}type": "extended",
                            "{http://www.w3.org/1999/xlink}role": linkrole
                        })
                footnoteLink = footnoteLinks[linkrole]
                if (linkrole, factId) not in factLocs:
                    locNbr += 1
                    locLabel = "l_{:02}".format(locNbr)
                    factLocs[(linkrole, factId)] = locLabel
                    addChild(footnoteLink,
                             XbrlConst.qnLinkLoc,
                             attributes={
                                 XLINKTYPE: "locator",
                                 XLINKHREF: "#" + factId,
                                 XLINKLABEL: locLabel
                             })
                locLabel = factLocs[(linkrole, factId)]
                if footnote.get("footnote"):
                    footnoteNbr += 1
                    footnoteLabel = "f_{:02}".format(footnoteNbr)
                    attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel}
                    if footnote.get("language"):
                        attrs[XMLLANG] = footnote["language"]
                    # note, for HTML will need to build an element structure
                    addChild(footnoteLink,
                             XbrlConst.qnLinkFootnote,
                             attributes=attrs,
                             text=footnote["footnote"])
                elif footnote.get("factRef"):
                    factRef = footnote.get("factRef")
                    if (linkrole, factRef) not in factLocs:
                        locNbr += 1
                        locLabel = "f_{:02}".format(footnoteNbr)
                        factLoc[(linkrole, factRef)] = locLabel
                        addChild(footnoteLink,
                                 XbrlConst.qnLinkLoc,
                                 attributes={
                                     XLINKTYPE: "locator",
                                     XLINKHREF: "#" + factRef,
                                     XLINKLABEL: locLabel
                                 })
                    footnoteLabel = factLoc[(linkrole, factId)]
                footnoteArc = addChild(footnoteLink,
                                       XbrlConst.qnLinkFootnoteArc,
                                       attributes={
                                           XLINKTYPE: "arc",
                                           XLINKARCROLE: arcrole,
                                           XLINKFROM: locLabel,
                                           XLINKTO: footnoteLabel
                                       })

        currentAction = "done loading facts and footnotes"

        #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
        #               messageCode="loadFromExcel:info")
    except Exception as ex:
        if isinstance(ex, OIMException):
            modelXbrl.error(ex.code,
                            ex.message,
                            modelObject=modelXbrl,
                            **ex.msgArgs)
        else:
            modelXbrl.error(
                "arelleOIMloader:error",
                "Error while %(action)s, error %(error)s\ntraceback %(traceback)s",
                modelObject=modelXbrl,
                action=currentAction,
                error=ex,
                traceback=traceback.format_tb(sys.exc_info()[2]))

    if priorCWD:
        os.chdir(
            priorCWD
        )  # restore prior current working directory            startingErrorCount = len(modelXbrl.errors)

    if startingErrorCount < len(modelXbrl.errors):
        # had errors, don't allow ModelDocument.load to continue
        return OIMException("arelleOIMloader:unableToLoad",
                            "Unable to load due to reported errors")

    return getattr(modelXbrl, "modelDocument",
                   None)  # none if returning from exception
Example #21
0
def evaluateTableIndex(modelXbrl):
    disclosureSystem = modelXbrl.modelManager.disclosureSystem
    if disclosureSystem.EFM:
        COVER    = "1Cover"
        STMTS    = "2Financial Statements"
        NOTES    = "3Notes to Financial Statements"
        POLICIES = "4Accounting Policies"
        TABLES   = "5Notes Tables"
        DETAILS  = "6Notes Details"
        UNCATEG  = "7Uncategorized"
        roleDefinitionPattern = re.compile(r"([0-9]+) - (Statement|Disclosure|Schedule|Document) - (.+)")
        # build EFM rendering-compatible index
        definitionElrs = dict((roleType.definition, roleType)
                              for roleURI in modelXbrl.relationshipSet(XbrlConst.parentChild).linkRoleUris
                              for roleType in modelXbrl.roleTypes.get(roleURI,()))
        isRR = any(ns.startswith("http://xbrl.sec.gov/rr/") for ns in modelXbrl.namespaceDocs.keys())
        tableGroup = None
        firstTableLinkroleURI = None
        firstDocumentLinkroleURI = None
        sortedRoleTypes = sorted(definitionElrs.items(), key=lambda item: item[0])
        for roleDefinition, roleType in sortedRoleTypes:
            roleType._tableChildren = []
            match = roleDefinitionPattern.match(roleDefinition) if roleDefinition else None
            if not match: 
                roleType._tableIndex = (UNCATEG, "", roleType.roleURI)
                continue
            seq, tblType, tblName = match.groups()
            if isRR:
                tableGroup = COVER
            elif not tableGroup:
                tableGroup = ("Paren" in tblName and COVER or tblType == "Statement" and STMTS or
                              "(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or
                              "(Detail" in tblName and DETAILS or COVER)
            elif tableGroup == COVER:
                tableGroup = (tblType == "Statement" and STMTS or "Paren" in tblName and COVER or
                              "(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or
                              "(Detail" in tblName and DETAILS or NOTES)
            elif tableGroup == STMTS:
                tableGroup = ((tblType == "Statement" or "Paren" in tblName) and STMTS or
                              "(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or
                              "(Detail" in tblName and DETAILS or NOTES)
            elif tableGroup == NOTES:
                tableGroup = ("(Polic" in tblName and POLICIES or "(Table" in tblName and TABLES or 
                              "(Detail" in tblName and DETAILS or tblType == "Disclosure" and NOTES or UNCATEG)
            elif tableGroup == POLICIES:
                tableGroup = ("(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or 
                              ("Paren" in tblName or "(Polic" in tblName) and POLICIES or UNCATEG)
            elif tableGroup == TABLES:
                tableGroup = ("(Detail" in tblName and DETAILS or 
                              ("Paren" in tblName or "(Table" in tblName) and TABLES or UNCATEG)
            elif tableGroup == DETAILS:
                tableGroup = (("Paren" in tblName or "(Detail" in tblName) and DETAILS or UNCATEG)
            else:
                tableGroup = UNCATEG
            if firstTableLinkroleURI is None and tableGroup == COVER:
                firstTableLinkroleURI = roleType.roleURI
            if tblType == "Document" and not firstDocumentLinkroleURI:
                firstDocumentLinkroleURI = roleType.roleURI
            roleType._tableIndex = (tableGroup, seq, tblName)

        # flow allocate facts to roles (SEC presentation groups)
        if not modelXbrl.qnameDimensionDefaults: # may not have run validatino yet
            from arelle import ValidateXbrlDimensions
            ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
        reportedFacts = set() # facts which were shown in a higher-numbered ELR table
        factsByQname = modelXbrl.factsByQname
        reportingPeriods = set()
        nextEnd = None
        deiFact = {}
        for conceptName in ("DocumentPeriodEndDate", "DocumentType", "CurrentFiscalPeriodEndDate"):
            for concept in modelXbrl.nameConcepts[conceptName]:
                for fact in factsByQname[concept.qname]:
                    deiFact[conceptName] = fact
                    if fact.context is not None:
                        reportingPeriods.add((None, fact.context.endDatetime)) # for instant
                        reportingPeriods.add((fact.context.startDatetime, fact.context.endDatetime)) # for startEnd
                        nextEnd = fact.context.startDatetime
                        duration = (fact.context.endDatetime - fact.context.startDatetime).days + 1
                        break
        if "DocumentType" in deiFact:
            fact = deiFact["DocumentType"]
            if "-Q" in fact.xValue:
                # need quarterly and yr to date durations
                endDatetime = fact.context.endDatetime
                # if within 2 days of end of month use last day of month
                endDatetimeMonth = endDatetime.month
                if (endDatetime + timedelta(2)).month != endDatetimeMonth:
                    # near end of month
                    endOfMonth = True
                    while endDatetime.month == endDatetimeMonth:
                        endDatetime += timedelta(1) # go forward to next month
                else:
                    endOfMonth = False
                startYr = endDatetime.year
                startMo = endDatetime.month - 3
                if startMo <= 0:
                    startMo += 12
                    startYr -= 1
                startDatetime = datetime(startYr, startMo, endDatetime.day, endDatetime.hour, endDatetime.minute, endDatetime.second)
                if endOfMonth:
                    startDatetime -= timedelta(1)
                    endDatetime -= timedelta(1)
                reportingPeriods.add((startDatetime, endDatetime))
                duration = 91
        # find preceding compatible default context periods
        while (nextEnd is not None):
            thisEnd = nextEnd
            prevMaxStart = thisEnd - timedelta(duration * .9)
            prevMinStart = thisEnd - timedelta(duration * 1.1)
            nextEnd = None
            for cntx in modelXbrl.contexts.values():
                if (cntx.isStartEndPeriod and not cntx.qnameDims and thisEnd == cntx.endDatetime and
                    prevMinStart <= cntx.startDatetime <= prevMaxStart):
                    reportingPeriods.add((None, cntx.endDatetime))
                    reportingPeriods.add((cntx.startDatetime, cntx.endDatetime))
                    nextEnd = cntx.startDatetime
                    break
                elif (cntx.isInstantPeriod and not cntx.qnameDims and thisEnd == cntx.endDatetime):
                    reportingPeriods.add((None, cntx.endDatetime))
        stmtReportingPeriods = set(reportingPeriods)       

        sortedRoleTypes.reverse() # now in descending order
        for i, roleTypes in enumerate(sortedRoleTypes):
            roleDefinition, roleType = roleTypes
            # find defined non-default axes in pre hierarchy for table
            tableFacts = set()
            tableGroup, tableSeq, tableName = roleType._tableIndex
            roleURIdims, priItemQNames = EFMlinkRoleURIstructure(modelXbrl, roleType.roleURI)
            for priItemQName in priItemQNames:
                for fact in factsByQname[priItemQName]:
                    cntx = fact.context
                    # non-explicit dims must be default
                    if (cntx is not None and
                        all(dimQn in modelXbrl.qnameDimensionDefaults
                            for dimQn in (roleURIdims.keys() - cntx.qnameDims.keys())) and
                        all(mdlDim.memberQname in roleURIdims[dimQn]
                            for dimQn, mdlDim in cntx.qnameDims.items()
                            if dimQn in roleURIdims)):
                        # the flow-up part, drop
                        cntxStartDatetime = cntx.startDatetime
                        cntxEndDatetime = cntx.endDatetime
                        if (tableGroup != STMTS or
                            (cntxStartDatetime, cntxEndDatetime) in stmtReportingPeriods and
                             (fact not in reportedFacts or
                              all(dimQn not in cntx.qnameDims # unspecified dims are all defaulted if reported elsewhere
                                  for dimQn in (cntx.qnameDims.keys() - roleURIdims.keys())))):
                            tableFacts.add(fact)
                            reportedFacts.add(fact)
            roleType._tableFacts = tableFacts
            
            # find parent if any
            closestParentType = None
            closestParentMatchLength = 0
            for _parentRoleDefinition, parentRoleType in sortedRoleTypes[i+1:]:
                matchLen = parentNameMatchLen(tableName, parentRoleType)
                if matchLen > closestParentMatchLength:
                    closestParentMatchLength = matchLen
                    closestParentType = parentRoleType
            if closestParentType is not None:
                closestParentType._tableChildren.insert(0, roleType)
                
            # remove lesser-matched children if there was a parent match
            unmatchedChildRoles = set()
            longestChildMatchLen = 0
            numChildren = 0
            for childRoleType in roleType._tableChildren:
                matchLen = parentNameMatchLen(tableName, childRoleType)
                if matchLen < closestParentMatchLength:
                    unmatchedChildRoles.add(childRoleType)
                elif matchLen > longestChildMatchLen:
                    longestChildMatchLen = matchLen
                    numChildren += 1
            if numChildren > 1: 
                # remove children that don't have the full match pattern length to parent
                for childRoleType in roleType._tableChildren:
                    if (childRoleType not in unmatchedChildRoles and 
                        parentNameMatchLen(tableName, childRoleType) < longestChildMatchLen):
                        unmatchedChildRoles.add(childRoleType)

            for unmatchedChildRole in unmatchedChildRoles:
                roleType._tableChildren.remove(unmatchedChildRole)

            for childRoleType in roleType._tableChildren:
                childRoleType._tableParent = roleType
                
            unmatchedChildRoles = None # dereference
        
        global UGT_TOPICS
        if UGT_TOPICS is None:
            try:
                from arelle import FileSource
                fh = FileSource.openFileStream(modelXbrl.modelManager.cntlr, 
                                               os.path.join(modelXbrl.modelManager.cntlr.configDir, "ugt-topics.zip/ugt-topics.json"),
                                               'r', 'utf-8')
                UGT_TOPICS = json.load(fh)
                fh.close()
                for topic in UGT_TOPICS:
                    topic[6] = set(topic[6]) # change concept abstracts list into concept abstracts set
                    topic[7] = set(topic[7]) # change concept text blocks list into concept text blocks set
                    topic[8] = set(topic[8]) # change concept names list into concept names set
            except Exception as ex:
                    UGT_TOPICS = None

        if UGT_TOPICS is not None:
            def roleUgtConcepts(roleType):
                roleConcepts = set()
                for rel in modelXbrl.relationshipSet(XbrlConst.parentChild, roleType.roleURI).modelRelationships:
                    if rel.toModelObject is not None:
                        roleConcepts.add(rel.toModelObject.name)
                    if rel.fromModelObject is not None:
                        roleConcepts.add(rel.fromModelObject.name)
                if hasattr(roleType, "_tableChildren"):
                    for _tableChild in roleType._tableChildren:
                        roleConcepts |= roleUgtConcepts(_tableChild)
                return roleConcepts
            topicMatches = {} # topicNum: (best score, roleType)
    
            for roleDefinition, roleType in sortedRoleTypes:
                roleTopicType = 'S' if roleDefinition.startswith('S') else 'D'
                if getattr(roleType, "_tableParent", None) is None:                
                    # rooted tables in reverse order
                    concepts = roleUgtConcepts(roleType)
                    for i, ugtTopic in enumerate(UGT_TOPICS):
                        if ugtTopic[0] == roleTopicType:
                            countAbstracts = len(concepts & ugtTopic[6])
                            countTextBlocks = len(concepts & ugtTopic[7])
                            countLineItems = len(concepts & ugtTopic[8])
                            if countAbstracts or countTextBlocks or countLineItems:
                                _score = (10 * countAbstracts +
                                          1000 * countTextBlocks +
                                          countLineItems / len(concepts))
                                if i not in topicMatches or _score > topicMatches[i][0]:
                                    topicMatches[i] = (_score, roleType)
            for topicNum, scoredRoleType in topicMatches.items():
                _score, roleType = scoredRoleType
                if _score > getattr(roleType, "_tableTopicScore", 0):
                    ugtTopic = UGT_TOPICS[topicNum]
                    roleType._tableTopicScore = _score
                    roleType._tableTopicType = ugtTopic[0]
                    roleType._tableTopicName = ugtTopic[3]
                    roleType._tableTopicCode = ugtTopic[4]
                    # print ("Match score {:.2f} topic {} preGrp {}".format(_score, ugtTopic[3], roleType.definition))
        return firstTableLinkroleURI or firstDocumentLinkroleURI # did build _tableIndex attributes
    return None
Example #22
0
    def run(self, options, sourceZipStream=None):
        """Process command line arguments or web service request, such as to load and validate an XBRL document, or start web server.
        
        When a web server has been requested, this method may be called multiple times, once for each web service (REST) request that requires processing.
        Otherwise (when called for a command line request) this method is called only once for the command line arguments request.
           
        :param options: OptionParser options from parse_args of main argv arguments (when called from command line) or corresponding arguments from web service (REST) request.
        :type options: optparse.Values
        """
        if options.showOptions: # debug options
            for optName, optValue in sorted(options.__dict__.items(), key=lambda optItem: optItem[0]):
                self.addToLog("Option {0}={1}".format(optName, optValue), messageCode="info")
            self.addToLog("sys.argv {0}".format(sys.argv), messageCode="info")
        if options.uiLang: # set current UI Lang (but not config setting)
            self.setUiLanguage(options.uiLang)
        if options.proxy:
            if options.proxy != "show":
                proxySettings = proxyTuple(options.proxy)
                self.webCache.resetProxies(proxySettings)
                self.config["proxySettings"] = proxySettings
                self.saveConfig()
                self.addToLog(_("Proxy configuration has been set."), messageCode="info")
            useOsProxy, urlAddr, urlPort, user, password = self.config.get("proxySettings", proxyTuple("none"))
            if useOsProxy:
                self.addToLog(_("Proxy configured to use {0}.").format(
                    _('Microsoft Windows Internet Settings') if sys.platform.startswith("win")
                    else (_('Mac OS X System Configuration') if sys.platform in ("darwin", "macos")
                          else _('environment variables'))), messageCode="info")
            elif urlAddr:
                self.addToLog(_("Proxy setting: http://{0}{1}{2}{3}{4}").format(
                    user if user else "",
                    ":****" if password else "",
                    "@" if (user or password) else "",
                    urlAddr,
                    ":{0}".format(urlPort) if urlPort else ""), messageCode="info")
            else:
                self.addToLog(_("Proxy is disabled."), messageCode="info")
        if options.plugins:
            from arelle import PluginManager
            resetPlugins = False
            savePluginChanges = True
            showPluginModules = False
            for pluginCmd in options.plugins.split('|'):
                cmd = pluginCmd.strip()
                if cmd == "show":
                    showPluginModules = True
                elif cmd == "temp":
                    savePluginChanges = False
                elif cmd.startswith("+"):
                    moduleInfo = PluginManager.addPluginModule(cmd[1:])
                    if moduleInfo:
                        self.addToLog(_("Addition of plug-in {0} successful.").format(moduleInfo.get("name")), 
                                      messageCode="info", file=moduleInfo.get("moduleURL"))
                        resetPlugins = True
                    else:
                        self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:])
                elif cmd.startswith("~"):
                    if PluginManager.reloadPluginModule(cmd[1:]):
                        self.addToLog(_("Reload of plug-in successful."), messageCode="info", file=cmd[1:])
                        resetPlugins = True
                    else:
                        self.addToLog(_("Unable to reload plug-in."), messageCode="info", file=cmd[1:])
                elif cmd.startswith("-"):
                    if PluginManager.removePluginModule(cmd[1:]):
                        self.addToLog(_("Deletion of plug-in successful."), messageCode="info", file=cmd[1:])
                        resetPlugins = True
                    else:
                        self.addToLog(_("Unable to delete plug-in."), messageCode="info", file=cmd[1:])
                else: # assume it is a module or package
                    savePluginChanges = False
                    moduleInfo = PluginManager.addPluginModule(cmd)
                    if moduleInfo:
                        self.addToLog(_("Activation of plug-in {0} successful.").format(moduleInfo.get("name")), 
                                      messageCode="info", file=moduleInfo.get("moduleURL"))
                        resetPlugins = True
                    else:
                        self.addToLog(_("Unable to load {0} as a plug-in or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd)
                if resetPlugins:
                    PluginManager.reset()
                    if savePluginChanges:
                        PluginManager.save(self)
            if showPluginModules:
                self.addToLog(_("Plug-in modules:"), messageCode="info")
                for i, moduleItem in enumerate(sorted(PluginManager.pluginConfig.get("modules", {}).items())):
                    moduleInfo = moduleItem[1]
                    self.addToLog(_("Plug-in: {0}; author: {1}; version: {2}; status: {3}; date: {4}; description: {5}; license {6}.").format(
                                  moduleItem[0], moduleInfo.get("author"), moduleInfo.get("version"), moduleInfo.get("status"),
                                  moduleInfo.get("fileDate"), moduleInfo.get("description"), moduleInfo.get("license")),
                                  messageCode="info", file=moduleInfo.get("moduleURL"))
                
        # run utility command line options that don't depend on entrypoint Files
        hasUtilityPlugin = False
        for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Utility.Run"):
            hasUtilityPlugin = True
            pluginXbrlMethod(self, options)
            
        # if no entrypointFile is applicable, quit now
        if options.proxy or options.plugins or hasUtilityPlugin:
            if not options.entrypointFile:
                return True # success
        self.username = options.username
        self.password = options.password
        self.entrypointFile = options.entrypointFile
        if self.entrypointFile:
            filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream)
        else:
            filesource = None
        if options.validateEFM:
            if options.disclosureSystemName:
                self.addToLog(_("both --efm and --disclosureSystem validation are requested, proceeding with --efm only"),
                              messageCode="info", file=self.entrypointFile)
            self.modelManager.validateDisclosureSystem = True
            self.modelManager.disclosureSystem.select("efm")
        elif options.disclosureSystemName:
            self.modelManager.validateDisclosureSystem = True
            self.modelManager.disclosureSystem.select(options.disclosureSystemName)
        elif options.validateHMRC:
            self.modelManager.validateDisclosureSystem = True
            self.modelManager.disclosureSystem.select("hmrc")
        else:
            self.modelManager.disclosureSystem.select(None) # just load ordinary mappings
            self.modelManager.validateDisclosureSystem = False
        if options.utrUrl:  # override disclosureSystem utrUrl
            self.modelManager.disclosureSystem.utrUrl = options.utrUrl
            # can be set now because the utr is first loaded at validation time 
            
        # disclosure system sets logging filters, override disclosure filters, if specified by command line
        if options.logLevelFilter:
            self.setLogLevelFilter(options.logLevelFilter)
        if options.logCodeFilter:
            self.setLogCodeFilter(options.logCodeFilter)
        if options.calcDecimals:
            if options.calcPrecision:
                self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"),
                              messageCode="info", file=self.entrypointFile)
            self.modelManager.validateInferDecimals = True
            self.modelManager.validateCalcLB = True
        elif options.calcPrecision:
            self.modelManager.validateInferDecimals = False
            self.modelManager.validateCalcLB = True
        if options.utrValidate:
            self.modelManager.validateUtr = True
        if options.infosetValidate:
            self.modelManager.validateInfoset = True
        if options.abortOnMajorError:
            self.modelManager.abortOnMajorError = True
        if options.collectProfileStats:
            self.modelManager.collectProfileStats = True
        if options.internetConnectivity == "offline":
            self.webCache.workOffline = True
        elif options.internetConnectivity == "online":
            self.webCache.workOffline = False
        if options.internetTimeout is not None:
            self.webCache.timeout = (options.internetTimeout or None)  # use None if zero specified to disable timeout
        fo = FormulaOptions()
        if options.parameters:
            parameterSeparator = (options.parameterSeparator or ',')
            fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value)) 
                                       for param in options.parameters.split(parameterSeparator) 
                                       for key,sep,value in (param.partition('='),) ) )   
        if options.formulaParamExprResult:
            fo.traceParameterExpressionResult = True
        if options.formulaParamInputValue:
            fo.traceParameterInputValue = True
        if options.formulaCallExprSource:
            fo.traceCallExpressionSource = True
        if options.formulaCallExprCode:
            fo.traceCallExpressionCode = True
        if options.formulaCallExprEval:
            fo.traceCallExpressionEvaluation = True
        if options.formulaCallExprResult:
            fo.traceCallExpressionResult = True
        if options.formulaVarSetExprEval:
            fo.traceVariableSetExpressionEvaluation = True
        if options.formulaVarSetExprResult:
            fo.traceVariableSetExpressionResult = True
        if options.formulaAsserResultCounts:
            fo.traceAssertionResultCounts = True
        if options.formulaFormulaRules:
            fo.traceFormulaRules = True
        if options.formulaVarsOrder:
            fo.traceVariablesOrder = True
        if options.formulaVarExpressionSource:
            fo.traceVariableExpressionSource = True
        if options.formulaVarExpressionCode:
            fo.traceVariableExpressionCode = True
        if options.formulaVarExpressionEvaluation:
            fo.traceVariableExpressionEvaluation = True
        if options.formulaVarExpressionResult:
            fo.traceVariableExpressionResult = True
        if options.timeVariableSetEvaluation:
            fo.timeVariableSetEvaluation = True
        if options.formulaVarFilterWinnowing:
            fo.traceVariableFilterWinnowing = True
        if options.formulaVarFiltersResult:
            fo.traceVariableFiltersResult = True
        if options.formulaVarFiltersResult:
            fo.traceVariableFiltersResult = True
        self.modelManager.formulaOptions = fo
        timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
        firstStartedAt = startedAt = time.time()
        modelDiffReport = None
        success = True
        modelXbrl = None
        try:
            if filesource:
                modelXbrl = self.modelManager.load(filesource, _("views loading"))
        except ModelDocument.LoadingException:
            pass
        except Exception as err:
            self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
                        err,
                        traceback.format_tb(sys.exc_info()[2])))
            success = False    # loading errors, don't attempt to utilize loaded DTS
        if modelXbrl and modelXbrl.modelDocument:
            loadTime = time.time() - startedAt
            modelXbrl.profileStat(_("load"), loadTime)
            self.addToLog(format_string(self.modelManager.locale, 
                                        _("loaded in %.2f secs at %s"), 
                                        (loadTime, timeNow)), 
                                        messageCode="info", file=self.entrypointFile)
            if options.importFiles:
                for importFile in options.importFiles.split("|"):
                    fileName = importFile.strip()
                    if sourceZipStream is not None and not (fileName.startswith('http://') or os.path.isabs(fileName)):
                        fileName = os.path.dirname(modelXbrl.uri) + os.sep + fileName # make relative to sourceZipStream
                    ModelDocument.load(modelXbrl, fileName)
                    loadTime = time.time() - startedAt
                    self.addToLog(format_string(self.modelManager.locale, 
                                                _("import in %.2f secs at %s"), 
                                                (loadTime, timeNow)), 
                                                messageCode="info", file=importFile)
                    modelXbrl.profileStat(_("import"), loadTime)
                if modelXbrl.errors:
                    success = False    # loading errors, don't attempt to utilize loaded DTS
            if modelXbrl.modelDocument.type in ModelDocument.Type.TESTCASETYPES:
                for pluginXbrlMethod in pluginClassMethods("Testcases.Start"):
                    pluginXbrlMethod(self, options, modelXbrl)
            else: # not a test case, probably instance or DTS
                for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Loaded"):
                    pluginXbrlMethod(self, options, modelXbrl)
        else:
            success = False
        if success and options.diffFile and options.versReportFile:
            try:
                diffFilesource = FileSource.FileSource(options.diffFile,self)
                startedAt = time.time()
                modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading"))
                if modelXbrl2.errors:
                    if not options.keepOpen:
                        modelXbrl2.close()
                    success = False
                else:
                    loadTime = time.time() - startedAt
                    modelXbrl.profileStat(_("load"), loadTime)
                    self.addToLog(format_string(self.modelManager.locale, 
                                                _("diff comparison DTS loaded in %.2f secs"), 
                                                loadTime), 
                                                messageCode="info", file=self.entrypointFile)
                    startedAt = time.time()
                    modelDiffReport = self.modelManager.compareDTSes(options.versReportFile)
                    diffTime = time.time() - startedAt
                    modelXbrl.profileStat(_("diff"), diffTime)
                    self.addToLog(format_string(self.modelManager.locale, 
                                                _("compared in %.2f secs"), 
                                                diffTime), 
                                                messageCode="info", file=self.entrypointFile)
            except ModelDocument.LoadingException:
                success = False
            except Exception as err:
                success = False
                self.addToLog(_("[Exception] Failed to doad diff file: \n{0} \n{1}").format(
                            err,
                            traceback.format_tb(sys.exc_info()[2])))
        if success:
            try:
                modelXbrl = self.modelManager.modelXbrl
                hasFormulae = modelXbrl.hasFormulae
                if options.validate:
                    startedAt = time.time()
                    if options.formulaAction: # don't automatically run formulas
                        modelXbrl.hasFormulae = False
                    self.modelManager.validate()
                    if options.formulaAction: # restore setting
                        modelXbrl.hasFormulae = hasFormulae
                    self.addToLog(format_string(self.modelManager.locale, 
                                                _("validated in %.2f secs"), 
                                                time.time() - startedAt),
                                                messageCode="info", file=self.entrypointFile)
                if options.formulaAction in ("validate", "run"):  # do nothing here if "none"
                    from arelle import ValidateXbrlDimensions, ValidateFormula
                    startedAt = time.time()
                    if not options.validate:
                        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
                    # setup fresh parameters from formula optoins
                    modelXbrl.parameters = fo.typedParameters()
                    ValidateFormula.validate(modelXbrl, compileOnly=(options.formulaAction != "run"))
                    self.addToLog(format_string(self.modelManager.locale, 
                                                _("formula validation and execution in %.2f secs")
                                                if options.formulaAction == "run"
                                                else _("formula validation only in %.2f secs"), 
                                                time.time() - startedAt),
                                                messageCode="info", file=self.entrypointFile)
                    

                if options.testReport:
                    ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport, options.testReportCols)
                    
                if options.rssReport:
                    ViewFileRssFeed.viewRssFeed(self.modelManager.modelXbrl, options.rssReport, options.rssReportCols)
                    
                if options.DTSFile:
                    ViewFileDTS.viewDTS(modelXbrl, options.DTSFile)
                if options.factsFile:
                    ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols)
                if options.factTableFile:
                    ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang)
                if options.conceptsFile:
                    ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang)
                if options.preFile:
                    ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang)
                if options.calFile:
                    ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang)
                if options.dimFile:
                    ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang)
                if options.formulaeFile:
                    ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang)
                if options.viewArcrole and options.viewFile:
                    ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.viewFile, os.path.basename(options.viewArcrole), options.viewArcrole, labelrole=options.labelRole, lang=options.labelLang)
                for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"):
                    pluginXbrlMethod(self, options, modelXbrl)
                                        
            except (IOError, EnvironmentError) as err:
                self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err))
                success = False
            except Exception as err:
                self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
                            err,
                            traceback.format_tb(sys.exc_info()[2])))
                success = False
        if modelXbrl:
            modelXbrl.profileStat(_("total"), time.time() - firstStartedAt)
            if options.collectProfileStats and modelXbrl:
                modelXbrl.logProfileStats()
            if not options.keepOpen:
                if modelDiffReport:
                    self.modelManager.close(modelDiffReport)
                elif modelXbrl:
                    self.modelManager.close(modelXbrl)
        self.username = self.password = None #dereference password
        return success
Example #23
0
def saveTargetDocument(modelXbrl,
                       targetDocumentFilename,
                       targetDocumentSchemaRefs,
                       outputZip=None,
                       filingFiles=None,
                       *args,
                       **kwargs):
    targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
        targetDocumentFilename, modelXbrl.modelDocument.filepath)
    targetUrlParts = targetUrl.rpartition(".")
    targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
    modelXbrl.modelManager.showStatus(
        _("Extracting instance ") + os.path.basename(targetUrl))
    targetInstance = ModelXbrl.create(modelXbrl.modelManager,
                                      newDocumentType=Type.INSTANCE,
                                      url=targetUrl,
                                      schemaRefs=targetDocumentSchemaRefs,
                                      isEntry=True)
    ValidateXbrlDimensions.loadDimensionDefaults(
        targetInstance)  # need dimension defaults
    # roleRef and arcroleRef (of each inline document)
    for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
        for roleRefElt in sourceRefs.values():
            addChild(targetInstance.modelDocument.xmlRootElement,
                     roleRefElt.qname,
                     attributes=roleRefElt.items())

    # contexts
    for context in modelXbrl.contexts.values():
        newCntx = targetInstance.createContext(
            context.entityIdentifier[0],
            context.entityIdentifier[1],
            'instant' if context.isInstantPeriod else
            'duration' if context.isStartEndPeriod else 'forever',
            context.startDatetime,
            context.endDatetime,
            None,
            context.qnameDims, [], [],
            id=context.id)
    for unit in modelXbrl.units.values():
        measures = unit.measures
        newUnit = targetInstance.createUnit(measures[0],
                                            measures[1],
                                            id=unit.id)

    modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
    newFactForOldObjId = {}

    def createFacts(facts, parent):
        for fact in facts:
            if fact.isItem:
                attrs = {"contextRef": fact.contextID}
                if fact.id:
                    attrs["id"] = fact.id
                if fact.isNumeric:
                    attrs["unitRef"] = fact.unitID
                    if fact.get("decimals"):
                        attrs["decimals"] = fact.get("decimals")
                    if fact.get("precision"):
                        attrs["precision"] = fact.get("precision")
                if fact.isNil:
                    attrs[XbrlConst.qnXsiNil] = "true"
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.textValue
                newFact = targetInstance.createFact(fact.qname,
                                                    attributes=attrs,
                                                    text=text,
                                                    parent=parent)
                newFactForOldObjId[fact.objectIndex] = newFact
                if filingFiles and fact.concept is not None and fact.concept.isTextBlock:
                    # check for img and other filing references
                    for xmltext in [text] + CDATApattern.findall(text):
                        try:
                            for elt in XML(
                                    "<body>\n{0}\n</body>\n".format(xmltext)):
                                if elt.tag in ("a", "img") and not isHttpUrl(
                                        attrValue) and not os.path.isabs(
                                            attrvalue):
                                    for attrTag, attrValue in elt.items():
                                        if attrTag in ("href", "src"):
                                            filingFiles.add(attrValue)
                        except (XMLSyntaxError, UnicodeDecodeError):
                            pass
            elif fact.isTuple:
                newTuple = targetInstance.createFact(fact.qname, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newTuple
                createFacts(fact.modelTupleFacts, newTuple)

    createFacts(modelXbrl.facts, None)
    # footnote links
    footnoteIdCount = {}
    modelXbrl.modelManager.showStatus(
        _("Creating and validating footnotes & relationships"))
    HREF = "{http://www.w3.org/1999/xlink}href"
    footnoteLinks = defaultdict(list)
    for linkKey, linkPrototypes in modelXbrl.baseSets.items():
        arcrole, linkrole, linkqname, arcqname = linkKey
        if (linkrole and linkqname and arcqname and  # fully specified roles
                arcrole != "XBRL-footnotes" and any(
                    lP.modelDocument.type == Type.INLINEXBRL
                    for lP in linkPrototypes)):
            for linkPrototype in linkPrototypes:
                if linkPrototype not in footnoteLinks[linkrole]:
                    footnoteLinks[linkrole].append(linkPrototype)
    for linkrole in sorted(footnoteLinks.keys()):
        for linkPrototype in footnoteLinks[linkrole]:
            newLink = addChild(targetInstance.modelDocument.xmlRootElement,
                               linkPrototype.qname,
                               attributes=linkPrototype.attributes)
            for linkChild in linkPrototype:
                attributes = linkChild.attributes
                if isinstance(linkChild, LocPrototype):
                    if HREF not in linkChild.attributes:
                        linkChild.attributes[HREF] = \
                        "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
                    addChild(newLink, linkChild.qname, attributes=attributes)
                elif isinstance(linkChild, ArcPrototype):
                    addChild(newLink, linkChild.qname, attributes=attributes)
                elif isinstance(linkChild, ModelInlineFootnote):
                    idUseCount = footnoteIdCount.get(linkChild.footnoteID,
                                                     0) + 1
                    if idUseCount > 1:  # if footnote with id in other links bump the id number
                        attributes = linkChild.attributes.copy()
                        attributes["id"] = "{}_{}".format(
                            attributes["id"], idUseCount)
                    footnoteIdCount[linkChild.footnoteID] = idUseCount
                    newChild = addChild(newLink,
                                        linkChild.qname,
                                        attributes=attributes)
                    copyIxFootnoteHtml(
                        linkChild,
                        newChild,
                        targetModelDocument=targetInstance.modelDocument,
                        withText=True)
                    if filingFiles and linkChild.textValue:
                        footnoteHtml = XML("<body/>")
                        copyIxFootnoteHtml(linkChild, footnoteHtml)
                        for elt in footnoteHtml.iter():
                            if elt.tag in ("a", "img"):
                                for attrTag, attrValue in elt.items():
                                    if attrTag in (
                                            "href", "src") and not isHttpUrl(
                                                attrValue
                                            ) and not os.path.isabs(attrvalue):
                                        filingFiles.add(attrValue)

    targetInstance.saveInstance(overrideFilepath=targetUrl,
                                outputZip=outputZip)
    if getattr(modelXbrl, "isTestcaseVariation", False):
        modelXbrl.extractedInlineInstance = True  # for validation comparison
    modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
Example #24
0
def saveTargetDocument(modelXbrl,
                       targetDocumentFilename,
                       targetDocumentSchemaRefs,
                       outputZip=None,
                       filingFiles=None,
                       *args,
                       **kwargs):
    targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
        targetDocumentFilename, modelXbrl.modelDocument.filepath)

    def addLocallyReferencedFile(elt, filingFiles):
        if elt.tag in ("a", "img"):
            for attrTag, attrValue in elt.items():
                if attrTag in ("href", "src") and not isHttpUrl(
                        attrValue) and not os.path.isabs(attrValue):
                    attrValue = attrValue.partition('#')[0]  # remove anchor
                    if attrValue:  # ignore anchor references to base document
                        attrValue = os.path.normpath(
                            attrValue
                        )  # change url path separators to host separators
                        file = os.path.join(sourceDir, attrValue)
                        if modelXbrl.fileSource.isInArchive(
                                file,
                                checkExistence=True) or os.path.exists(file):
                            filingFiles.add(file)

    targetUrlParts = targetUrl.rpartition(".")
    targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
    modelXbrl.modelManager.showStatus(
        _("Extracting instance ") + os.path.basename(targetUrl))
    rootElt = modelXbrl.modelDocument.xmlRootElement
    # take baseXmlLang from <html> or <base>
    baseXmlLang = rootElt.get(
        "{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang")
    for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants(
            tag="{http://www.w3.org/1999/xhtml}body"):
        baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang"
                                ) or rootElt.get("lang") or baseXmlLang
    targetInstance = ModelXbrl.create(
        modelXbrl.modelManager,
        newDocumentType=Type.INSTANCE,
        url=targetUrl,
        schemaRefs=targetDocumentSchemaRefs,
        isEntry=True,
        discover=False)  # don't attempt to load DTS
    if baseXmlLang:
        targetInstance.modelDocument.xmlRootElement.set(
            "{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
    ValidateXbrlDimensions.loadDimensionDefaults(
        targetInstance)  # need dimension defaults
    # roleRef and arcroleRef (of each inline document)
    for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
        for roleRefElt in sourceRefs.values():
            addChild(targetInstance.modelDocument.xmlRootElement,
                     roleRefElt.qname,
                     attributes=roleRefElt.items())

    # contexts
    for context in sorted(modelXbrl.contexts.values(),
                          key=lambda c: elementChildSequence(c)):
        ignore = targetInstance.createContext(
            context.entityIdentifier[0],
            context.entityIdentifier[1],
            'instant' if context.isInstantPeriod else
            'duration' if context.isStartEndPeriod else 'forever',
            context.startDatetime,
            context.endDatetime,
            None,
            context.qnameDims, [], [],
            id=context.id)
    for unit in modelXbrl.units.values():
        measures = unit.measures
        ignore = targetInstance.createUnit(measures[0],
                                           measures[1],
                                           id=unit.id)

    modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
    newFactForOldObjId = {}

    def createFacts(facts, parent):
        for fact in facts:
            if fact.isItem:  # HF does not de-duplicate, which is currently-desired behavior
                attrs = {"contextRef": fact.contextID}
                if fact.id:
                    attrs["id"] = fact.id
                if fact.isNumeric:
                    attrs["unitRef"] = fact.unitID
                    if fact.get("decimals"):
                        attrs["decimals"] = fact.get("decimals")
                    if fact.get("precision"):
                        attrs["precision"] = fact.get("precision")
                if fact.isNil:
                    attrs[XbrlConst.qnXsiNil] = "true"
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.textValue
                    if fact.concept is not None and fact.concept.baseXsdType in (
                            "string", "normalizedString"):  # default
                        xmlLang = fact.xmlLang
                        if xmlLang is not None and xmlLang != baseXmlLang:
                            attrs[
                                "{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang
                newFact = targetInstance.createFact(fact.qname,
                                                    attributes=attrs,
                                                    text=text,
                                                    parent=parent)
                # if fact.isFraction, create numerator and denominator
                newFactForOldObjId[fact.objectIndex] = newFact
                if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
                    # check for img and other filing references so that referenced files are included in the zip.
                    for xmltext in [text] + CDATApattern.findall(text):
                        try:
                            for elt in XML("<body>\n{0}\n</body>\n".format(
                                    xmltext)).iter():
                                addLocallyReferencedFile(elt, filingFiles)
                        except (XMLSyntaxError, UnicodeDecodeError):
                            pass  # TODO: Why ignore UnicodeDecodeError?
            elif fact.isTuple:
                newTuple = targetInstance.createFact(fact.qname, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newTuple
                createFacts(fact.modelTupleFacts, newTuple)

    createFacts(modelXbrl.facts, None)
    modelXbrl.modelManager.showStatus(
        _("Creating and validating footnotes and relationships"))
    HREF = "{http://www.w3.org/1999/xlink}href"
    footnoteLinks = defaultdict(list)
    footnoteIdCount = {}
    for linkKey, linkPrototypes in modelXbrl.baseSets.items():
        arcrole, linkrole, linkqname, arcqname = linkKey
        if (linkrole and linkqname and arcqname and  # fully specified roles
                arcrole != "XBRL-footnotes" and any(
                    lP.modelDocument.type == Type.INLINEXBRL
                    for lP in linkPrototypes)):
            for linkPrototype in linkPrototypes:
                if linkPrototype not in footnoteLinks[linkrole]:
                    footnoteLinks[linkrole].append(linkPrototype)
    for linkrole in sorted(footnoteLinks.keys()):
        for linkPrototype in footnoteLinks[linkrole]:
            newLink = addChild(targetInstance.modelDocument.xmlRootElement,
                               linkPrototype.qname,
                               attributes=linkPrototype.attributes)
            for linkChild in linkPrototype:
                attributes = linkChild.attributes
                if isinstance(linkChild, LocPrototype):
                    if HREF not in linkChild.attributes:
                        linkChild.attributes[HREF] = \
                        "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
                    addChild(newLink, linkChild.qname, attributes=attributes)
                elif isinstance(linkChild, ArcPrototype):
                    addChild(newLink, linkChild.qname, attributes=attributes)
                elif isinstance(linkChild, ModelInlineFootnote):
                    idUseCount = footnoteIdCount.get(linkChild.footnoteID,
                                                     0) + 1
                    if idUseCount > 1:  # if footnote with id in other links bump the id number
                        attributes = linkChild.attributes.copy()
                        attributes["id"] = "{}_{}".format(
                            attributes["id"], idUseCount)
                    footnoteIdCount[linkChild.footnoteID] = idUseCount
                    newChild = addChild(newLink,
                                        linkChild.qname,
                                        attributes=attributes)
                    xmlLang = linkChild.xmlLang
                    if xmlLang is not None and xmlLang != baseXmlLang:  # default
                        newChild.set(
                            "{http://www.w3.org/XML/1998/namespace}lang",
                            xmlLang)
                    copyIxFootnoteHtml(
                        linkChild,
                        newChild,
                        targetModelDocument=targetInstance.modelDocument,
                        withText=True)

                    if filingFiles and linkChild.textValue:
                        footnoteHtml = XML("<body/>")
                        copyIxFootnoteHtml(linkChild, footnoteHtml)
                        for elt in footnoteHtml.iter():
                            addLocallyReferencedFile(elt, filingFiles)
    targetInstance.saveInstance(overrideFilepath=targetUrl,
                                outputZip=outputZip)
    if getattr(modelXbrl, "isTestcaseVariation", False):
        modelXbrl.extractedInlineInstance = True  # for validation comparison
    modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
Example #25
0
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
    from openpyxl import load_workbook
    from arelle import ModelDocument, ModelXbrl, XmlUtil
    from arelle.ModelDocument import ModelDocumentReference
    from arelle.ModelValue import qname
    
    try:
        currentAction = "initializing"
        startingErrorCount = len(modelXbrl.errors)
        startedAt = time.time()
        
        if os.path.isabs(oimFile):
            # allow relative filenames to loading directory
            priorCWD = os.getcwd()
            os.chdir(os.path.dirname(oimFile))
        else:
            priorCWD = None
            
        currentAction = "determining file type"
        isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json")
        isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
        isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
        isCSVorXL = isCSV or isXL
        instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"
        
        if isJSON:
            currentAction = "loading and parsing JSON OIM file"
            def loadDict(keyValuePairs):
                _dict = OrderedDict() # preserve fact order in resulting instance
                for key, value in keyValuePairs:
                    if isinstance(value, dict):
                        if DUPJSONKEY in value:
                            for _errKey, _errValue, _otherValue in value[DUPJSONKEY]:
                                if key == "prefixes":
                                    modelXbrl.error("oime:duplicatedPrefix",
                                                    _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"),
                                                    modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue)
                            del value[DUPJSONKEY]
                    if key in _dict:
                        if DUPJSONKEY not in _dict:
                            _dict[DUPJSONKEY] = []
                        _dict[DUPJSONKEY].append((key, value, _dict[key]))
                    else:
                        _dict[key] = value
                return _dict
            with io.open(oimFile, 'rt', encoding='utf-8') as f:
                oimObject = json.load(f, object_pairs_hook=loadDict)
            missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject]
            if missing:
                raise OIMException("oime:missingJSONElements", 
                                   _("Required element(s) are missing from JSON input: %(missing)s"),
                                   missing = ", ".join(missing))
            currentAction = "identifying JSON objects"
            dtsReferences = oimObject["dtsReferences"]
            prefixesList = oimObject["prefixes"].items()
            facts = oimObject["facts"]
            footnotes = oimObject["facts"] # shares this object
        elif isCSV:
            currentAction = "identifying CSV input tables"
            if sys.version[0] >= '3':
                csvOpenMode = 'w'
                csvOpenNewline = ''
            else:
                csvOpenMode = 'wb' # for 2.7
                csvOpenNewline = None
                
            oimFileBase = None
            if "-facts" in oimFile:
                oimFileBase = oimFile.partition("-facts")[0]
            else:
                for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"):
                    if oimFile.endswith(suffix):
                        oimFileBase = oimFile[:-len(suffix)]
                        break
            if oimFileBase is None:
                raise OIMException("oime:missingCSVTables", 
                                   _("Unable to identify CSV tables file name pattern"))
            if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or
                not os.path.exists(oimFileBase + "-prefixes.csv")):
                raise OIMException("oime:missingCSVTables", 
                                   _("Unable to identify CSV tables for dtsReferences or prefixes"))
            instanceFileName = oimFileBase + ".xbrl"
            currentAction = "loading CSV dtsReferences table"
            dtsReferences = []
            with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = row
                    else:
                        dtsReferences.append(dict((header[j], col) for j, col in enumerate(row)))
            currentAction = "loading CSV prefixes table"
            prefixesList = []
            with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = dict((col,i) for i,col in enumerate(row))
                    else:
                        prefixesList.append((row[header["prefix"]], row[header["URI"]]))
            defaults = {}
            if os.path.exists(oimFileBase + "-defaults.csv"):
                currentAction = "loading CSV defaults table"
                with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                            fileCol = row.index("file")
                        else:
                            defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol)
            currentAction = "loading CSV facts tables"
            facts = []
            _dir = os.path.dirname(oimFileBase)
            factsFileBasename = os.path.basename(oimFileBase) + "-facts"
            for filename in os.listdir(_dir):
                filepath = os.path.join(_dir, filename)
                if filename.startswith(factsFileBasename):
                    currentAction = "loading CSV facts table {}".format(filename)
                    tableDefaults = defaults.get(filename, {})
                    with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
                        csvReader = csv.reader(f)
                        for i, row in enumerate(csvReader):
                            if i == 0:
                                header = row
                            else:
                                fact = {}
                                fact.update(tableDefaults)
                                for j, col in enumerate(row):
                                    if col is not None:
                                        if header[j]: # skip cols with no header
                                            if header[j].endswith("Value"):
                                                if col: # ignore empty columns (= null CSV value)
                                                    fact["value"] = col
                                            else:
                                                fact[header[j]] = col
                                facts.append(fact)
            footnotes = []
            if os.path.exists(oimFileBase + "-footnotes.csv"):
                currentAction = "loading CSV footnotes table"
                with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                        else:
                            footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col))
        elif isXL:
            currentAction = "identifying workbook input worksheets"
            oimWb = load_workbook(oimFile, read_only=True, data_only=True)
            sheetNames = oimWb.get_sheet_names()
            if (not any(sheetName == "prefixes" for sheetName in sheetNames) or
                not any(sheetName == "dtsReferences" for sheetName in sheetNames) or
                not any("facts" in sheetName for sheetName in sheetNames)):
                raise OIMException("oime:missingWorkbookWorksheets", 
                                   _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts"))
            currentAction = "loading worksheet: dtsReferences"
            dtsReferences = []
            for i, row in enumerate(oimWb["dtsReferences"]):
                if i == 0:
                    header = [col.value for col in row]
                else:
                    dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row)))
            currentAction = "loading worksheet: prefixes"
            prefixesList = []
            for i, row in enumerate(oimWb["prefixes"]):
                if i == 0:
                    header = dict((col.value,i) for i,col in enumerate(row))
                else:
                    prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value))
            defaults = {}
            if "defaults" in sheetNames:
                currentAction = "loading worksheet: defaults"
                for i, row in enumerate(oimWb["defaults"]):
                    if i == 0:
                        header = dict((col.value,i) for i,col in enumerate(row))
                        fileCol = header["file"]
                    else:
                        defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol)
            facts = []
            for sheetName in sheetNames:
                if sheetName == "facts" or "-facts" in sheetName:
                    currentAction = "loading worksheet: {}".format(sheetName)
                    tableDefaults = defaults.get(sheetName, {})
                    for i, row in enumerate(oimWb[sheetName]):
                        if i == 0:
                            header = [col.value for col in row]
                        else:
                            fact = {}
                            fact.update(tableDefaults)
                            for j, col in enumerate(row):
                                if col.value is not None:
                                    if header[j]: # skip cols with no header
                                        if header[j].endswith("Value"):
                                            fact["value"] = str(col.value)
                                        else:
                                            fact[header[j]] = str(col.value)
                            facts.append(fact)
            footnotes = []
            if "footnotes" in sheetNames:
                currentAction = "loading worksheet: footnotes"
                for i, row in enumerate(oimWb["footnotes"]):
                    if i == 0:
                        header = dict((col.value,i) for i,col in enumerate(row) if col.value)
                    else:
                        footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value))
    
        currentAction = "identifying default dimensions"
        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults 
        
        currentAction = "validating OIM"
        prefixes = {}
        prefixedUris = {}
        for _prefix, _uri in prefixesList:
            if not _prefix:
                modelXbrl.error("oime:emptyPrefix",
                                _("The empty string must not be used as a prefix, uri %(uri)s"),
                                modelObject=modelXbrl, uri=_uri)
            elif not NCNamePattern.match(_prefix):
                modelXbrl.error("oime:prefixPattern",
                                _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"),
                                modelObject=modelXbrl, prefix=_prefix, uri=_uri)
            elif _prefix in prefixes:
                modelXbrl.error("oime:duplicatedPrefix",
                                _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"),
                                modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri)
            elif _uri in prefixedUris:
                modelXbrl.error("oime:duplicatedUri",
                                _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"),
                                modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix)
            else:
                prefixes[_prefix] = _uri
                prefixedUris[_uri] = _prefix
                
        oimPrefix = None
        for _nsOim in nsOim:
            if _nsOim in prefixedUris:
                oimPrefix = prefixedUris[_nsOim]
        if not oimPrefix:
            raise OIMException("oime:noOimPrefix",
                               _("The oim namespace must have a declared prefix"))
        oimConcept = "{}:concept".format(oimPrefix)
        oimEntity = "{}:entity".format(oimPrefix)
        oimPeriod = "{}:period".format(oimPrefix)
        oimPeriodStart = "{}:periodStart".format(oimPrefix)
        oimPeriodEnd = "{}:periodEnd".format(oimPrefix)
        oimUnit = "{}:unit".format(oimPrefix)
        oimPrefix = "{}:".format(oimPrefix)
            
        # create the instance document
        currentAction = "creating instance document"
        modelXbrl.blockDpmDBrecursion = True
        modelXbrl.modelDocument = createModelDocument(
              modelXbrl, 
              Type.INSTANCE,
              instanceFileName,
              schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"],
              isEntry=True,
              initialComment="extracted from OIM {}".format(mappedUri),
              documentEncoding="utf-8")
        cntxTbl = {}
        unitTbl = {}
        currentAction = "creating facts"
        for fact in facts:
            conceptQn = qname(fact[oimConcept], prefixes)
            concept = modelXbrl.qnameConcepts.get(conceptQn)
            entityAsQn = qname(fact[oimEntity], prefixes)
            if oimPeriod in fact:
                periodStart = fact[oimPeriod]["start"]
                periodEnd = fact[oimPeriod]["end"]
            else:
                periodStart = fact[oimPeriodStart]
                periodEnd = fact[oimPeriodEnd]
            cntxKey = ( # hashable context key
                ("periodType", concept.periodType),
                ("entity", entityAsQn),
                ("periodStart", periodStart),
                ("periodEnd", periodEnd)) + tuple(sorted(
                    (dimName, dimVal) 
                    for dimName, dimVal in fact.items()
                    if ":" in dimName and not dimName.startswith(oimPrefix)))
            if cntxKey in cntxTbl:
                _cntx = cntxTbl[cntxKey]
            else:
                cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                qnameDims = {}
                for dimName, dimVal in fact.items():
                    if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal:
                        dimQname = qname(dimName, prefixes)
                        dimConcept = modelXbrl.qnameConcepts.get(dimQname)
                        if ":" in dimVal and dimVal.partition(':')[0] in prefixes:
                            mem = qname(dimVal, prefixes) # explicit dim
                        elif dimConcept.isTypedDimension:
                            # a modelObject xml element is needed for all of the instance functions to manage the typed dim
                            mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
                        qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
                _cntx = modelXbrl.createContext(
                                        entityAsQn.namespaceURI,
                                        entityAsQn.localName,
                                        concept.periodType,
                                        None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME),
                                        dateTime(periodEnd, type=DATETIME),
                                        None, # no dimensional validity checking (like formula does)
                                        qnameDims, [], [],
                                        id=cntxId)
                cntxTbl[cntxKey] = _cntx
            if oimUnit in fact:
                unitKey = fact[oimUnit]
                if unitKey in unitTbl:
                    _unit = unitTbl[unitKey]
                else:
                    _unit = None
                    # validate unit
                    unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey)
                    if not UnitPattern.match(unitKeySub):
                        modelXbrl.error("oime:unitStringRepresentation",
                                        _("Unit string representation is lexically invalid, %(unit)s"),
                                        modelObject=modelXbrl, unit=unitKey)
                    else:
                        _mul, _sep, _div = unitKey.partition('/')
                        if _mul.startswith('('):
                            _mul = _mul[1:-1]
                        _muls = [u for u in _mul.split('*') if u]
                        if _div.startswith('('):
                            _div = _div[1:-1]
                        _divs = [u for u in _div.split('*') if u]
                        if _muls != sorted(_muls) or _divs != sorted(_divs):
                            modelXbrl.error("oime:unitStringRepresentation",
                                            _("Unit string representation measures are not in alphabetical order, %(unit)s"),
                                            modelObject=modelXbrl, unit=unitKey)
                        try:
                            mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
                                                                                      _("Unit prefix is not declared: %(unit)s"),
                                                                                      unit=u)) 
                                      for u in _muls]
                            divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
                                                                                      _("Unit prefix is not declared: %(unit)s"),
                                                                                      unit=u))
                                      for u in _divs]
                            unitId = 'u-{:02}'.format(len(unitTbl) + 1)
                            for _measures in mulQns, divQns:
                                for _measure in _measures:
                                    addQnameValue(modelXbrl.modelDocument, _measure)
                            _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId)
                        except OIMException as ex:
                            modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
                    unitTbl[unitKey] = _unit
            else:
                _unit = None
            
            attrs = {"contextRef": _cntx.id}
    
            if fact.get("value") is None:
                attrs[XbrlConst.qnXsiNil] = "true"
                text = None
            else:
                text = fact["value"]
                
            if fact.get("id"):
                attrs["id"] = fact["id"]
                
            if concept.isNumeric:
                if _unit is None:
                    continue # skip creating fact because unit was invalid
                attrs["unitRef"] = _unit.id
                if "accuracy" in fact:
                    attrs["decimals"] = fact["accuracy"]
                    
            # is value a QName?
            if concept.baseXbrliType == "QName":
                addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
    
            f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
            
        currentAction = "creating footnotes"
        footnoteLinks = {} # ELR elements
        factLocs = {} # index by (linkrole, factId)
        footnoteNbr = 0
        locNbr = 0
        for factOrFootnote in footnotes:
            if "factId" in factOrFootnote:
                factId = factOrFootnote["factId"]
                factFootnotes = (factOrFootnote,) # CSV or XL
            elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
                factId = factOrFootnote["id"]
                factFootnotes = factOrFootnote["footnotes"]
            else:
                factFootnotes = ()
            for footnote in factFootnotes:
                linkrole = footnote.get("group")
                arcrole = footnote.get("footnoteType")
                if not factId or not linkrole or not arcrole or not (
                    footnote.get("factRef") or footnote.get("footnote")):
                    # invalid footnote
                    continue
                if linkrole not in footnoteLinks:
                    footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, 
                                                       XbrlConst.qnLinkFootnoteLink, 
                                                       attributes={"{http://www.w3.org/1999/xlink}type": "extended",
                                                                   "{http://www.w3.org/1999/xlink}role": linkrole})
                footnoteLink = footnoteLinks[linkrole]
                if (linkrole, factId) not in factLocs:
                    locNbr += 1
                    locLabel = "l_{:02}".format(locNbr)
                    factLocs[(linkrole, factId)] = locLabel
                    addChild(footnoteLink, XbrlConst.qnLinkLoc, 
                             attributes={XLINKTYPE: "locator",
                                         XLINKHREF: "#" + factId,
                                         XLINKLABEL: locLabel})
                locLabel = factLocs[(linkrole, factId)]
                if footnote.get("footnote"):
                    footnoteNbr += 1
                    footnoteLabel = "f_{:02}".format(footnoteNbr)
                    attrs = {XLINKTYPE: "resource",
                             XLINKLABEL: footnoteLabel}
                    if footnote.get("language"):
                        attrs[XMLLANG] = footnote["language"]
                    # note, for HTML will need to build an element structure
                    addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"])
                elif footnote.get("factRef"):
                    factRef = footnote.get("factRef")
                    if (linkrole, factRef) not in factLocs:
                        locNbr += 1
                        locLabel = "f_{:02}".format(footnoteNbr)
                        factLoc[(linkrole, factRef)] = locLabel
                        addChild(footnoteLink, XbrlConst.qnLinkLoc, 
                                 attributes={XLINKTYPE: "locator",
                                             XLINKHREF: "#" + factRef,
                                             XLINKLABEL: locLabel})
                    footnoteLabel = factLoc[(linkrole, factId)]
                footnoteArc = addChild(footnoteLink, 
                                       XbrlConst.qnLinkFootnoteArc, 
                                       attributes={XLINKTYPE: "arc",
                                                   XLINKARCROLE: arcrole,
                                                   XLINKFROM: locLabel,
                                                   XLINKTO: footnoteLabel})
                    
        currentAction = "done loading facts and footnotes"
        
        #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
        #               messageCode="loadFromExcel:info")
    except Exception as ex:
        if isinstance(ex, OIMException):
            modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
        else:
            modelXbrl.error("arelleOIMloader:error",
                            "Error while %(action)s, error %(error)s\ntraceback %(traceback)s",
                            modelObject=modelXbrl, action=currentAction, error=ex,
                            traceback=traceback.format_tb(sys.exc_info()[2]))
    
    if priorCWD:
        os.chdir(priorCWD) # restore prior current working directory            startingErrorCount = len(modelXbrl.errors)
        
    if startingErrorCount < len(modelXbrl.errors):
        # had errors, don't allow ModelDocument.load to continue
        return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors")

    return getattr(modelXbrl, "modelDocument", None) # none if returning from exception
Example #26
0
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None, *args, **kwargs):
    targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
    def addLocallyReferencedFile(elt,filingFiles):
        if elt.tag in ("a", "img"):
            for attrTag, attrValue in elt.items():
                if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrValue):
                    attrValue = attrValue.partition('#')[0] # remove anchor
                    if attrValue: # ignore anchor references to base document
                        attrValue = os.path.normpath(attrValue) # change url path separators to host separators
                        file = os.path.join(sourceDir,attrValue)
                        if modelXbrl.fileSource.isInArchive(file, checkExistence=True) or os.path.exists(file):
                            filingFiles.add(file)
    targetUrlParts = targetUrl.rpartition(".")
    targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
    modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
    rootElt = modelXbrl.modelDocument.xmlRootElement
    # take baseXmlLang from <html> or <base>
    baseXmlLang = rootElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang")
    for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/1999/xhtml}body"):
        baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang") or baseXmlLang
    targetInstance = ModelXbrl.create(modelXbrl.modelManager, 
                                      newDocumentType=Type.INSTANCE,
                                      url=targetUrl,
                                      schemaRefs=targetDocumentSchemaRefs,
                                      isEntry=True,
                                      discover=False) # don't attempt to load DTS
    if baseXmlLang:
        targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
    ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults 
    # roleRef and arcroleRef (of each inline document)
    for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
        for roleRefElt in sourceRefs.values():
            addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, 
                     attributes=roleRefElt.items())
    
    # contexts
    for context in sorted(modelXbrl.contexts.values(), key=lambda c: elementChildSequence(c)):
        ignore = targetInstance.createContext(context.entityIdentifier[0],
                                               context.entityIdentifier[1],
                                               'instant' if context.isInstantPeriod else
                                               'duration' if context.isStartEndPeriod
                                               else 'forever',
                                               context.startDatetime,
                                               context.endDatetime,
                                               None, 
                                               context.qnameDims, [], [],
                                               id=context.id)
    for unit in modelXbrl.units.values():
        measures = unit.measures
        ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id)

    modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
    newFactForOldObjId = {}
    def createFacts(facts, parent):
        for fact in facts:
            if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior
                attrs = {"contextRef": fact.contextID}
                if fact.id:
                    attrs["id"] = fact.id
                if fact.isNumeric:
                    attrs["unitRef"] = fact.unitID
                    if fact.get("decimals"):
                        attrs["decimals"] = fact.get("decimals")
                    if fact.get("precision"):
                        attrs["precision"] = fact.get("precision")
                if fact.isNil:
                    attrs[XbrlConst.qnXsiNil] = "true"
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.textValue
                    if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default
                        xmlLang = fact.xmlLang
                        if xmlLang is not None and xmlLang != baseXmlLang:
                            attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang
                newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
                # if fact.isFraction, create numerator and denominator
                newFactForOldObjId[fact.objectIndex] = newFact
                if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
                    # check for img and other filing references so that referenced files are included in the zip.
                    for xmltext in [text] + CDATApattern.findall(text):
                        try:
                            for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter():
                                addLocallyReferencedFile(elt, filingFiles)
                        except (XMLSyntaxError, UnicodeDecodeError):
                            pass  # TODO: Why ignore UnicodeDecodeError?
            elif fact.isTuple:
                newTuple = targetInstance.createFact(fact.qname, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newTuple
                createFacts(fact.modelTupleFacts, newTuple)
                
    createFacts(modelXbrl.facts, None)
    modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships"))
    HREF = "{http://www.w3.org/1999/xlink}href"
    footnoteLinks = defaultdict(list)
    footnoteIdCount = {}
    for linkKey, linkPrototypes in modelXbrl.baseSets.items():
        arcrole, linkrole, linkqname, arcqname = linkKey
        if (linkrole and linkqname and arcqname and  # fully specified roles
            arcrole != "XBRL-footnotes" and
            any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
            for linkPrototype in linkPrototypes:
                if linkPrototype not in footnoteLinks[linkrole]:
                    footnoteLinks[linkrole].append(linkPrototype)
    for linkrole in sorted(footnoteLinks.keys()):
        for linkPrototype in footnoteLinks[linkrole]:
            newLink = addChild(targetInstance.modelDocument.xmlRootElement, 
                               linkPrototype.qname, 
                               attributes=linkPrototype.attributes)
            for linkChild in linkPrototype:
                attributes = linkChild.attributes
                if isinstance(linkChild, LocPrototype):
                    if HREF not in linkChild.attributes:
                        linkChild.attributes[HREF] = \
                        "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
                    addChild(newLink, linkChild.qname, 
                             attributes=attributes)
                elif isinstance(linkChild, ArcPrototype):
                    addChild(newLink, linkChild.qname, attributes=attributes)
                elif isinstance(linkChild, ModelInlineFootnote):
                    idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
                    if idUseCount > 1: # if footnote with id in other links bump the id number
                        attributes = linkChild.attributes.copy()
                        attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
                    footnoteIdCount[linkChild.footnoteID] = idUseCount
                    newChild = addChild(newLink, linkChild.qname, 
                                        attributes=attributes)
                    xmlLang = linkChild.xmlLang
                    if xmlLang is not None and xmlLang != baseXmlLang: # default
                        newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang)
                    copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True)

                    if filingFiles and linkChild.textValue:
                        footnoteHtml = XML("<body/>")
                        copyIxFootnoteHtml(linkChild, footnoteHtml)
                        for elt in footnoteHtml.iter():
                            addLocallyReferencedFile(elt,filingFiles)
    targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip)
    if getattr(modelXbrl, "isTestcaseVariation", False):
        modelXbrl.extractedInlineInstance = True # for validation comparison
    modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
Example #27
0
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
    from openpyxl import load_workbook
    from arelle import ModelDocument, ModelXbrl, XmlUtil
    from arelle.ModelDocument import ModelDocumentReference
    from arelle.ModelValue import qname
    
    _return = None # modelDocument or an exception
    
    try:
        currentAction = "initializing"
        startingErrorCount = len(modelXbrl.errors)
        startedAt = time.time()
        
        if os.path.isabs(oimFile):
            # allow relative filenames to loading directory
            priorCWD = os.getcwd()
            os.chdir(os.path.dirname(oimFile))
        else:
            priorCWD = None
            
        currentAction = "determining file type"
        isJSON = oimFile.endswith(".json")
        isCSV = oimFile.endswith(".csv") # this option is not currently supported
        isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
        isCSVorXL = isCSV or isXL
        instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"
        _csvwContext = None
        
        if isJSON:
            errPrefix = "xbrlje"
            currentAction = "loading and parsing JSON OIM file"
            def loadDict(keyValuePairs):
                _dict = OrderedDict() # preserve fact order in resulting instance
                for key, value in keyValuePairs:
                    if isinstance(value, dict):
                        if DUPJSONKEY in value:
                            for _errKey, _errValue, _otherValue in value[DUPJSONKEY]:
                                if key == "prefixes":
                                    modelXbrl.error("xbrlje:duplicatedPrefix",
                                                    _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"),
                                                    modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue)
                            del value[DUPJSONKEY]
                    if key in _dict:
                        if DUPJSONKEY not in _dict:
                            _dict[DUPJSONKEY] = []
                        _dict[DUPJSONKEY].append((key, value, _dict[key]))
                    else:
                        _dict[key] = value
                return _dict
            with io.open(oimFile, 'rt', encoding='utf-8') as f:
                oimObject = json.load(f, object_pairs_hook=loadDict)
            # check if it's a CSVW metadata
            _csvwContext = oimObject.get("@context")
            if _csvwContext == "http://www.w3.org/ns/csvw" or (
                isinstance(_csvwContext, list) and "http://www.w3.org/ns/csvw" in _csvwContext):
                isJSON = False
                isCSV = True
            else:
                if oimObject.get("documentType", None) != "http://www.xbrl.org/WGWD/YYYY-MM-DD/xbrl-json":
                    # is it likely to be an OIM document?
                    if any(t in oimObject for t in ("dtsReferences", "prefixes", "facts")):
                        raise OIMException("xbrlje:missingJSONdocumentType", 
                                           _("Required documentType is missing from JSON input"))
                    else:
                        raise NotOIMException() # return None, not an OIM document
                missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject]
                if missing:
                    raise OIMException("xbrlje:missingJSONElements", 
                                       _("Required element(s) are missing from JSON input: %(missing)s"),
                                       missing = ", ".join(missing))
                currentAction = "identifying JSON objects"
                dtsReferences = oimObject["dtsReferences"]
                prefixesList = oimObject["prefixes"].items()
                facts = oimObject["facts"]
                footnotes = oimObject["facts"] # shares this object
        if isCSV:
            errPrefix = "xbrlce"
            currentAction = "compiling metadata"
            if sys.version[0] >= '3':
                csvOpenMode = 'w'
                csvOpenNewline = ''
            else:
                csvOpenMode = 'wb' # for 2.7
                csvOpenNewline = None
                
            if not _csvwContext: # json metadata wasn't specified, try to find it
                raise OIMException("xbrlce:missingCSVMetadata", 
                                   _("Unable to identify CSV metadata file"))
            # process CSV metadata
            # mandatory sections of metadata file
            oimMetadata = oimObject.get(CSVmetadata)
            if not oimMetadata:
                raise OIMException("xbrlce:missingOIMMetadata", 
                                   _("Unable to identify OIM metadata infile"))
            if oimMetadata.get("documentType") != CSVdocumentType:
                raise OIMException("xbrlce:documentType", 
                                   _("Document type %(documentType)s not recognized, expecting %(expectedDocumentType)s"),
                                   documentType=oimMetadata.get("documentType"), expectedDocumentType=CSVdocumentType)
            
            dtsReferences = oimMetadata.get("dtsReferences", {})
            prefixesList = oimMetadata.get("prefixes", {}).items()
            topLevelAspects = oimObject.get(CSVaspects, {})
            
            currentAction = "loading CSV facts tables"
            facts = []
            footnotes = []
            _dir = os.path.dirname(oimFile)
            for oimTable in oimObject.get("tables", []):
                tableType = oimTable.get(CSVtableType)
                tableLevelAspects =  oimTable.get(CSVaspects, {})
                tableUrl = oimTable.get("url")
                tableColumns = oimTable.get("tableSchema",{}).get("columns", [])
                # compile column dependencies
                aspectCols = []
                factCols = []
                for iCol, col in enumerate(tableColumns):
                    if CSVcolumnAspect in col:
                        aspectCols.append(iCol)
                    if CSVtupleFactAspects in col or CSVsimpleFactAspects in col:
                        factCols.append(iCol)
                if tableType not in ("fact", "footnote"):
                    modelXbrl.error("xbrlce:unrecognizedTableType",
                                    _("Table type %(tableType)s was not recognized, table URI %(uri)s."),
                                    modelObject=modelXbrl, uri=oimFile, tableType=tableType)
                    continue
                if not tableColumns:
                    modelXbrl.error("xbrlce:noTableColumns",
                                    _("Table has no columns, table URI %(uri)s."),
                                    modelObject=modelXbrl, uri=_uri)
                    continue
                tableAspects = topLevelAspects.copy()
                tableAspects.update(tableLevelAspects)
                filepath = os.path.join(_dir, tableUrl)
                tupleIds = set()
                with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                        else:
                            if tableType == "fact":
                                colAspects = tableAspects.copy()
                                specificColAspects = defaultdict(dict)
                                for iCol in aspectCols:
                                    value = row[iCol]
                                    aspect = tableColumns[iCol][CSVcolumnAspect]
                                    if isinstance(aspect, str): # applies to all cols
                                        colAspects[aspect] = value
                                    elif isinstance(aspect, dict): # applies to specific cols
                                        for _aspect, _colNames in aspect.items():
                                            for _colName in _colNames:
                                                specificColAspects[_colName][_aspect] = value
                                for iCol in factCols:
                                    fact = {"aspects": {}}
                                    cellValue = row[iCol]
                                    if cellValue == "": # no fact produced for this cell
                                        continue
                                    tableCol = tableColumns[iCol]
                                    if CSVtupleFactAspects in tableCol:
                                        aspectsObjectName = CSVtupleFactAspects
                                    else:
                                        aspectsObjectName = CSVsimpleFactAspects
                                    cellAspects = (colAspects, 
                                                   specificColAspects.get(tableCol.get("name"), EMPTYDICT),
                                                   tableColumns[iCol].get(aspectsObjectName), EMPTYDICT)
                                    
                                    inapplicableAspects = set()
                                    if tableCol.get(CSVtupleReferenceId) == "true":
                                        if cellValue:
                                            if cellValue in tupleIds:
                                                continue # don't duplicate parent tuple
                                            fact["id"] = cellValue
                                            tupleIds.add(cellValue) # prevent tuple duplication
                                    elif CSVsimpleFactAspects in tableCol:
                                        fact["value"] = csvCellValue(cellValue)
                                            
                                    if CSVtupleFactAspects in tableCol:
                                        inapplicableAspects.update(oimSimpleFactAspects)
                                        for _aspects in cellAspects:
                                            for aspectName, aspectValue in _aspects.items():
                                                if not aspectName.startswith(oimPrefix):
                                                    inapplicableAspects.add(aspectName)
                                            
                                    # block any row aspect produced by this column from this column's fact
                                    _colAspect = tableCol.get(CSVcolumnAspect)
                                    if isinstance(_colAspect, str): # applies to all cols
                                        inapplicableAspects.add(_colAspect)
                                            
                                    for _aspects in cellAspects:
                                        if _aspects:
                                            for aspectName, aspectValue in _aspects.items():
                                                if aspectName not in inapplicableAspects and aspectValue  != "":
                                                    if ":" in aspectName:
                                                        fact["aspects"][aspectName] = csvCellValue(aspectValue)
                                                    else:
                                                        fact[aspectName] = aspectValue
                                    facts.append(fact)
                            elif tableType == "footnote":
                                footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col))
                del tupleIds
                
        elif isXL:
            errPrefix = "xbrlwe"
            currentAction = "identifying workbook input worksheets"
            oimWb = load_workbook(oimFile, read_only=True, data_only=True)
            sheetNames = oimWb.get_sheet_names()
            if (not any(sheetName == "prefixes" for sheetName in sheetNames) or
                not any(sheetName == "dtsReferences" for sheetName in sheetNames) or
                not any("facts" in sheetName for sheetName in sheetNames)):
                raise OIMException("xbrlwe:missingWorkbookWorksheets", 
                                   _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts"))
            currentAction = "loading worksheet: dtsReferences"
            dtsReferences = []
            for i, row in enumerate(oimWb["dtsReferences"]):
                if i == 0:
                    header = [col.value for col in row]
                else:
                    dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row)))
            currentAction = "loading worksheet: prefixes"
            prefixesList = []
            for i, row in enumerate(oimWb["prefixes"]):
                if i == 0:
                    header = dict((col.value,i) for i,col in enumerate(row))
                else:
                    prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value))
            defaults = {}
            if "defaults" in sheetNames:
                currentAction = "loading worksheet: defaults"
                for i, row in enumerate(oimWb["defaults"]):
                    if i == 0:
                        header = dict((col.value,i) for i,col in enumerate(row))
                        fileCol = header["file"]
                    else:
                        defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol)
            facts = []
            for sheetName in sheetNames:
                if sheetName == "facts" or "-facts" in sheetName:
                    currentAction = "loading worksheet: {}".format(sheetName)
                    tableDefaults = defaults.get(sheetName, {})
                    for i, row in enumerate(oimWb[sheetName]):
                        if i == 0:
                            header = [col.value for col in row]
                        else:
                            fact = {}
                            fact.update(tableDefaults)
                            for j, col in enumerate(row):
                                if col.value is not None:
                                    if header[j]: # skip cols with no header
                                        if header[j].endswith("Value"):
                                            fact["value"] = str(col.value)
                                        else:
                                            fact[header[j]] = str(col.value)
                            facts.append(fact)
            footnotes = []
            if "footnotes" in sheetNames:
                currentAction = "loading worksheet: footnotes"
                for i, row in enumerate(oimWb["footnotes"]):
                    if i == 0:
                        header = [col.value for j,col in enumerate(row) if col.value]
                    else:
                        footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value))
    
        currentAction = "identifying default dimensions"
        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults 
        
        currentAction = "validating OIM"
        prefixes = {}
        prefixedUris = {}
        for _prefix, _uri in prefixesList:
            if not _prefix:
                modelXbrl.error("{}:emptyPrefix".format(errPrefix),
                                _("The empty string must not be used as a prefix, uri %(uri)s"),
                                modelObject=modelXbrl, uri=_uri)
            elif not NCNamePattern.match(_prefix):
                modelXbrl.error("oime:prefixPattern",
                                _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"),
                                modelObject=modelXbrl, prefix=_prefix, uri=_uri)
            elif _prefix in prefixes:
                modelXbrl.error("{}:duplicatedPrefix".format(errPrefix),
                                _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"),
                                modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri)
            elif _uri in prefixedUris:
                modelXbrl.error("{}:duplicatedUri".format(errPrefix),
                                _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"),
                                modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix)
            else:
                prefixes[_prefix] = _uri
                prefixedUris[_uri] = _prefix
                
        for _nsOim in nsOim:
            if _nsOim in prefixedUris:
                _oimPrefix = prefixedUris[_nsOim]
        if not _oimPrefix:
            raise OIMException("oime:noOimPrefix",
                               _("The oim namespace must have a declared prefix"))
            
        # create the instance document
        currentAction = "creating instance document"
        modelXbrl.blockDpmDBrecursion = True
        modelXbrl.modelDocument = _return = createModelDocument(
              modelXbrl, 
              Type.INSTANCE,
              instanceFileName,
              schemaRefs=[dtsRef["href"] 
                          for dtsRef in dtsReferences 
                          if dtsRef.get("type") == "schema" and dtsRef.get("href")],
              isEntry=True,
              initialComment="extracted from OIM {}".format(mappedUri),
              documentEncoding="utf-8")
        modelXbrl.modelDocument.inDTS = True
        
        # add linkbase, role and arcrole refs
        for refType in ("linkbase", "role", "arcrole"):
            for dtsRef in dtsReferences:
                if dtsRef.get("type") == refType and dtsRef.get("href"):
                    elt = addChild(modelXbrl.modelDocument.xmlRootElement, 
                                   qname(link, refType+"Ref"), 
                                   attributes=(("{http://www.w3.org/1999/xlink}href", dtsRef["href"]),
                                               ("{http://www.w3.org/1999/xlink}type", "simple")))
                    href = modelXbrl.modelDocument.discoverHref(elt)
                    if href:
                        _elt, hrefDoc, hrefId = href
                        _defElt = hrefDoc.idObjects.get(hrefId)
                        _uriAttrName = refType + "URI"
                        _uriAttrValue = _defElt.get(_uriAttrName)
                        if _defElt is not None and _uriAttrValue:
                            elt.set(_uriAttrName, _uriAttrValue)
        cntxTbl = {}
        unitTbl = {}
        # determine tuple dependency order
        idFacts = {}
        parentedFacts =  defaultdict(list)
        for i, fact in enumerate(facts):
            aspects = fact["aspects"]
            id = fact.get("id")
            tupleParent = fact.get("aspects", EMPTYDICT).get(oimTupleParent)
            if id is not None:
                idFacts[id] = i
            parentedFacts[tupleParent].append(i) # root facts have tupleParent None
            
        currentAction = "creating facts"
        
        def createModelFact(fact, parentModelFact, topTupleFact):
            aspects = fact.get("aspects", EMPTYDICT)
            if oimConcept not in aspects:
                modelXbrl.error("{}:conceptQName".format(errPrefix),
                                _("The concept QName could not be determined"),
                                modelObject=modelXbrl)
                return
            conceptQn = qname(aspects[oimConcept], prefixes)
            concept = modelXbrl.qnameConcepts.get(conceptQn)
            attrs = {}
            if concept.isItem:
                missingAspects = []
                if oimEntity not in aspects: 
                    missingAspects.append(oimEntity)
                if oimPeriod not in aspects and (oimPeriodStart not in aspects or oimPeriodEnd not in aspects):
                    missingAspects.append(oimPeriod)
                if missingAspects:
                    modelXbrl.error("{}:missingAspects".format(errPrefix),
                                    _("The concept %(element)s is missing aspects %(missingAspects)s"),
                                    modelObject=modelXbrl, element=conceptQn, missingAspects=", ".join(missingAspects))
                    return
                entityAsQn = qname(aspects[oimEntity], prefixes) or qname("error",fact[oimEntity])
                if oimPeriod in aspects:
                    periodStart = periodEnd = aspects[oimPeriod]
                if oimPeriodStart in aspects and oimPeriodEnd in aspects:
                    periodStart = aspects[oimPeriodStart]
                    periodEnd = aspects[oimPeriodEnd]
                cntxKey = ( # hashable context key
                    ("periodType", concept.periodType),
                    ("entity", entityAsQn),
                    ("periodStart", periodStart),
                    ("periodEnd", periodEnd)) + tuple(sorted(
                        (dimName, dimVal["value"] if isinstance(dimVal,dict) else dimVal) 
                        for dimName, dimVal in aspects.items()
                        if ":" in dimName and not dimName.startswith(oimPrefix)))
                if cntxKey in cntxTbl:
                    _cntx = cntxTbl[cntxKey]
                else:
                    cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                    qnameDims = {}
                    for dimName, dimVal in aspects.items():
                        if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal:
                            dimQname = qname(dimName, prefixes)
                            dimConcept = modelXbrl.qnameConcepts.get(dimQname)
                            if dimConcept is None:
                                modelXbrl.error("{}:taxonomyDefinedAspectQName".format(errPrefix),
                                                _("The taxonomy defined aspect concept QName %(qname)s could not be determined"),
                                                modelObject=modelXbrl, qname=dimQname)
                                continue
                            if isinstance(dimVal, dict):
                                dimVal = dimVal["value"]
                            else:
                                dimVal = str(dimVal) # may be int or boolean
                            if isinstance(dimVal,str) and ":" in dimVal and dimVal.partition(':')[0] in prefixes:
                                mem = qname(dimVal, prefixes) # explicit dim
                            elif dimConcept.isTypedDimension:
                                # a modelObject xml element is needed for all of the instance functions to manage the typed dim
                                mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
                            qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
                    _cntx = modelXbrl.createContext(
                                            entityAsQn.namespaceURI,
                                            entityAsQn.localName,
                                            concept.periodType,
                                            None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME),
                                            dateTime(periodEnd, type=DATETIME),
                                            None, # no dimensional validity checking (like formula does)
                                            qnameDims, [], [],
                                            id=cntxId,
                                            beforeSibling=topTupleFact)
                    cntxTbl[cntxKey] = _cntx
                if oimUnit in aspects and concept.isNumeric:
                    unitKey = aspects[oimUnit]
                    if unitKey in unitTbl:
                        _unit = unitTbl[unitKey]
                    else:
                        _unit = None
                        # validate unit
                        unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey)
                        if not UnitPattern.match(unitKeySub):
                            modelXbrl.error("oime:unitStringRepresentation",
                                            _("Unit string representation is lexically invalid, %(unit)s"),
                                            modelObject=modelXbrl, unit=unitKey)
                        else:
                            _mul, _sep, _div = unitKey.partition('/')
                            if _mul.startswith('('):
                                _mul = _mul[1:-1]
                            _muls = [u for u in _mul.split('*') if u]
                            if _div.startswith('('):
                                _div = _div[1:-1]
                            _divs = [u for u in _div.split('*') if u]
                            if _muls != sorted(_muls) or _divs != sorted(_divs):
                                modelXbrl.error("oime:unitStringRepresentation",
                                                _("Unit string representation measures are not in alphabetical order, %(unit)s"),
                                                modelObject=modelXbrl, unit=unitKey)
                            try:
                                mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
                                                                                          _("Unit prefix is not declared: %(unit)s"),
                                                                                          unit=u)) 
                                          for u in _muls]
                                divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
                                                                                          _("Unit prefix is not declared: %(unit)s"),
                                                                                          unit=u))
                                          for u in _divs]
                                unitId = 'u-{:02}'.format(len(unitTbl) + 1)
                                for _measures in mulQns, divQns:
                                    for _measure in _measures:
                                        addQnameValue(modelXbrl.modelDocument, _measure)
                                _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId, beforeSibling=topTupleFact)
                            except OIMException as ex:
                                modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
                        unitTbl[unitKey] = _unit
                else:
                    _unit = None
            
                attrs["contextRef"] = _cntx.id
        
                if fact.get("value") is None:
                    attrs[XbrlConst.qnXsiNil] = "true"
                    text = None
                else:
                    text = fact["value"]
                    
                if concept.isNumeric:
                    if _unit is None:
                        return # skip creating fact because unit was invalid
                    attrs["unitRef"] = _unit.id
                    if "accuracy" in attrs or attrs.get(XbrlConst.qnXsiNil, "false") != "true":
                        attrs["decimals"] = fact.get("accuracy", "INF")
            else:
                text = None #tuple
                    
            id = fact.get("id")
            if id is not None:
                attrs["id"] = fact["id"]
                    
            # is value a QName?
            if concept.baseXbrliType == "QName":
                addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
    
            f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text, parent=parentModelFact, validate=False)
            
            if id is not None and id in parentedFacts:
                # create child facts
                for i in sorted(parentedFacts[id],
                                key=lambda j: facts[j].get("aspects", EMPTYDICT).get(oimTupleOrder,0)):
                    createModelFact(facts[i], f, f if topTupleFact is None else topTupleFact)
                    
            # validate after creating tuple contents
            xmlValidate(modelXbrl, f)
                    
        for i in parentedFacts[None]:
            createModelFact(facts[i], None, None)
            
        currentAction = "creating footnotes"
        footnoteLinks = OrderedDict() # ELR elements
        factLocs = {} # index by (linkrole, factId)
        footnoteNbr = 0
        locNbr = 0
        for factOrFootnote in footnotes:
            if "factId" in factOrFootnote:
                factId = factOrFootnote["factId"]
                factFootnotes = (factOrFootnote,) # CSV or XL
            elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
                factId = factOrFootnote["id"]
                factFootnotes = factOrFootnote["footnotes"]
            else:
                factFootnotes = ()
            for footnote in factFootnotes:
                linkrole = footnote.get("group")
                arcrole = footnote.get("footnoteType")
                if not factId or not linkrole or not arcrole or not (
                    footnote.get("factRef") or footnote.get("footnote")):
                    # invalid footnote
                    continue
                if linkrole not in footnoteLinks:
                    footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, 
                                                       XbrlConst.qnLinkFootnoteLink, 
                                                       attributes={"{http://www.w3.org/1999/xlink}type": "extended",
                                                                   "{http://www.w3.org/1999/xlink}role": linkrole})
                footnoteLink = footnoteLinks[linkrole]
                if (linkrole, factId) not in factLocs:
                    locNbr += 1
                    locLabel = "l_{:02}".format(locNbr)
                    factLocs[(linkrole, factId)] = locLabel
                    addChild(footnoteLink, XbrlConst.qnLinkLoc, 
                             attributes={XLINKTYPE: "locator",
                                         XLINKHREF: "#" + factId,
                                         XLINKLABEL: locLabel})
                locFromLabel = factLocs[(linkrole, factId)]
                if footnote.get("footnote"):
                    footnoteNbr += 1
                    footnoteToLabel = "f_{:02}".format(footnoteNbr)
                    attrs = {XLINKTYPE: "resource",
                             XLINKLABEL: footnoteToLabel}
                    if footnote.get("language"):
                        attrs[XMLLANG] = footnote["language"]
                    # note, for HTML will need to build an element structure
                    addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"])
                elif footnote.get("factRef"):
                    factRef = footnote.get("factRef")
                    if (linkrole, factRef) not in factLocs:
                        locNbr += 1
                        locLabel = "l_{:02}".format(locNbr)
                        factLocs[(linkrole, factRef)] = locLabel
                        addChild(footnoteLink, XbrlConst.qnLinkLoc, 
                                 attributes={XLINKTYPE: "locator",
                                             XLINKHREF: "#" + factRef,
                                             XLINKLABEL: locLabel})
                    footnoteToLabel = factLocs[(linkrole, factRef)]
                footnoteArc = addChild(footnoteLink, 
                                       XbrlConst.qnLinkFootnoteArc, 
                                       attributes={XLINKTYPE: "arc",
                                                   XLINKARCROLE: arcrole,
                                                   XLINKFROM: locFromLabel,
                                                   XLINKTO: footnoteToLabel})
        if footnoteLinks:
            modelXbrl.modelDocument.linkbaseDiscover(footnoteLinks.values(), inInstance=True)
                    
        currentAction = "done loading facts and footnotes"
        
        #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
        #               messageCode="loadFromExcel:info")
    except NotOIMException as ex:
        _return = ex # not an OIM document
    except Exception as ex:
        _return = ex
        if isinstance(ex, OIMException):
            modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
        else:
            modelXbrl.error("arelleOIMloader:error",
                            "Error while %(action)s, error %(error)s\ntraceback %(traceback)s",
                            modelObject=modelXbrl, action=currentAction, error=ex,
                            traceback=traceback.format_tb(sys.exc_info()[2]))
    
    if priorCWD:
        os.chdir(priorCWD) # restore prior current working directory            startingErrorCount = len(modelXbrl.errors)
        
    #if startingErrorCount < len(modelXbrl.errors):
    #    # had errors, don't allow ModelDocument.load to continue
    #    return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors")

    return _return
Example #28
0
    def loadXbrlFromDB(self, loadDBsaveToFile):
        # load from database
        modelXbrl = self.modelXbrl
        
        # find instance in DB
        instanceURI = os.path.basename(loadDBsaveToFile)
        results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant"
                               " FROM dInstance WHERE FileName = '{}'"
                               .format(instanceURI))
        instanceId = moduleId = None
        for instanceId, moduleId, entScheme, entId, datePerEnd in results:
            break

        # find module in DB        
        results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId))
        xbrlSchemaRef = None
        for result in results:
            xbrlSchemaRef = result[0]
            break
        
        if not instanceId or not xbrlSchemaRef:
            raise XPDBException("sqlDB:MissingDTS",
                    _("The instance and module were not found for %(instanceURI)"),
                    instanceURI = instanceURI) 
            
        if modelXbrl.skipDTS:
            # find prefixes and namespaces in DB
            results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]")            
            dpmPrefixedNamespaces = dict((prefix, namespace)
                                         for owner, prefix, namespace in results)
            
        # create the instance document and resulting filing
        modelXbrl.blockDpmDBrecursion = True
        modelXbrl.modelDocument = createModelDocument(modelXbrl, 
                                                      Type.INSTANCE,
                                                      loadDBsaveToFile,
                                                      schemaRefs=[xbrlSchemaRef],
                                                      isEntry=True)
        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults 
        
        addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement, 
                                 'xbrl-streamable-instance', 
                                 'version="1.0" contextBuffer="1"')

        # add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue)
        
        # filing indicator code IDs
        # get filing indicators
        results = self.execute("SELECT mToT.TemplateOrTableCode "
                               "  FROM dFilingIndicator dFI, mTemplateOrTable mToT "
                               "  WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID"
                               .format(instanceId))
        filingIndicatorCodes = [code[0] for code in results]
        
        if filingIndicatorCodes:
            modelXbrl.createContext(entScheme,
                        entId,
                        'instant',
                        None,
                        datePerEnd,
                        None, # no dimensional validity checking (like formula does)
                        {}, [], [],
                        id='c')
            filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators)
            for filingIndicatorCode in filingIndicatorCodes:
                modelXbrl.createFact(qnFindFilingIndicator, 
                                     parent=filingIndicatorsTuple,
                                     attributes={"contextRef": "c"}, 
                                     text=filingIndicatorCode)

        
        # facts in this instance
        factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards,"
                                " Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue "
                                "FROM dFact WHERE InstanceID = {} "
                                "ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL "
                                "                          THEN DataPointSignature"
                                "                          ELSE DataPointSignatureWithValuesForWildcards"
                                "                END, instr(DataPointSignature,'|') + 1)"
                                .format(instanceId))
        
        # results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal

        # get typed dimension values
        prefixedNamespaces = modelXbrl.prefixedNamespaces
        prefixedNamespaces["iso4217"] = XbrlConst.iso4217
        if modelXbrl.skipDTS:
            prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed
        
        cntxTbl = {} # index by d
        unitTbl = {}
        
        def typedDimElt(s):
            # add xmlns into s for known qnames
            tag, angleBrkt, rest = s[1:].partition('>')
            text, angleBrkt, rest = rest.partition("<")
            qn = qname(tag, prefixedNamespaces)
            # a modelObject xml element is needed for all of the instance functions to manage the typed dim
            return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
        
        # contexts and facts
        for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl:
            metric, sep, dims = (dpSigTypedDims or dpSig).partition('|')
            conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces)
            concept = modelXbrl.qnameConcepts.get(conceptQn)
            isNumeric = isBool = isDateTime = isQName = isText = False
            if concept is not None:
                if concept.isNumeric:
                    isNumeric = True
                else:
                    baseXbrliType = concept.baseXbrliType
                    if baseXbrliType == "booleanItemType":
                        isBool = True
                    elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
                        isDateTime = True
                    elif baseXbrliType == "QNameItemType":
                        isQName = True
            else:
                c = conceptQn.localName[0]
                if c == 'm':
                    isNumeric = True
                elif c == 'd':
                    isDateTime = True
                elif c == 'b':
                    isBool = True
                elif c == 'e':
                    isQName = True
            isText = not (isNumeric or isBool or isDateTime or isQName)
            if isinstance(datePerEnd, _STR_BASE):
                datePerEnd = datetimeValue(datePerEnd, addOneDay=True)
            cntxKey = (dims, entId, datePerEnd)
            if cntxKey in cntxTbl:
                cntxId = cntxTbl[cntxKey]
            else:
                cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                cntxTbl[cntxKey] = cntxId
                qnameDims = {}
                if dims:
                    for dim in dims.split('|'):
                        dQn, sep, dVal = dim[:-1].partition('(')
                        dimQname = qname(dQn, prefixedNamespaces)
                        if dVal.startswith('<'): # typed dim
                            mem = typedDimElt(dVal)
                        else:
                            mem = qname(dVal, prefixedNamespaces)
                        qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario")
                    
                modelXbrl.createContext(entScheme,
                                        entId,
                                        'instant',
                                        None,
                                        datePerEnd,
                                        None, # no dimensional validity checking (like formula does)
                                        qnameDims, [], [],
                                        id=cntxId)
            if unit:
                if unit in unitTbl:
                    unitId = unitTbl[unit]
                else:
                    unitQn = qname(unit, prefixedNamespaces)
                    unitId = 'u{}'.format(unitQn.localName)
                    unitTbl[unit] = unitId
                    modelXbrl.createUnit([unitQn], [], id=unitId)
            else:
                unitId = None
            attrs = {"contextRef": cntxId}
            if unitId:
                attrs["unitRef"] = unitId
            if dec is not None:
                if isinstance(dec, float): # must be an integer
                    dec = int(dec)
                elif isinstance(dec, _STR_BASE) and '.' in dec:
                    dec = dec.partition('.')[0] # drop .0 from any SQLite string
                attrs["decimals"] = str(dec)  # somehow it is float from the database
            if False: # fact.isNil:
                attrs[XbrlConst.qnXsiNil] = "true"
                text = None
            elif numVal is not None:
                num = roundValue(numVal, None, dec) # round using reported decimals
                if dec is None or dec == "INF":  # show using decimals or reported format
                    dec = len(numVal.partition(".")[2])
                else: # max decimals at 28
                    dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
                text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num))
            elif dateVal is not None:
                text = dateVal
            elif boolVal is not None:
                text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false'
            else:
                if isQName: # declare namespace
                    addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces))
                text = textVal
            modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
            
        # add footnotes if any
        
        # save to file
        modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile)
        modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
        return modelXbrl.modelDocument
Example #29
0
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath):
    # check if big instance and has header with an initial incomplete tree walk (just 2 elements
    def logSyntaxErrors(parsercontext):
        for error in parsercontext.error_log:
            modelXbrl.error("xmlSchema:syntax",
                    _("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"),
                    modelObject=modelDocument, fileName=os.path.basename(filepath), 
                    error=error.message, line=error.line, column=error.column, sourceAction="streaming")
    #### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
    #### note: iterparse wants a binary file, but file is text mode
    _file, = modelXbrl.fileSource.file(filepath, binary=True)
    startedAt = time.time()
    modelXbrl.profileActivity()
    parsercontext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
    foundInstance = False
    foundErrors = False
    streamingAspects = None
    numRootFacts1 = 0
    numElts = 0
    elt = None
    for event, elt in parsercontext:
        if event == "start":
            if elt.getparent() is not None:
                if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    if not foundInstance:
                        foundInstance = True
                        pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance")
                        if pi is None:
                            break
                        else:
                            streamingAspects = dict(pi.attrib.copy())
                    if not elt.tag.startswith("{http://www.xbrl.org/"):
                        numRootFacts1 += 1
                        if numRootFacts1 % 1000 == 0:
                            modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
                elif not foundInstance:       
                    break
            elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl" and precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None:
                modelXbrl.error("streamingExtensions:headerMisplaced",
                        _("Header is misplaced: %(error)s, must follow xbrli:xbrl element"),
                        modelObject=elt)
        elif event == "end":
            elt.clear()
            numElts += 1
            if numElts % 1000 == 0 and elt.getparent() is not None:
                while elt.getprevious() is not None and elt.getparent() is not None:
                    del elt.getparent()[0]
    if elt is not None:
        elt.clear()
    _file.seek(0,io.SEEK_SET) # allow reparsing
    if not foundInstance or streamingAspects is None:
        del elt, parsercontext
        _file.close()
        return None
    modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
    startedAt = time.time()
    try:
        version = Decimal(streamingAspects.get("version"))
        if int(version) != 1:
            modelXbrl.error("streamingExtensions:unsupportedVersion",
                    _("Streaming version %(version)s, major version number must be 1"),
                    modelObject=elt, version=version)
            foundErrors = True
    except (InvalidOperation, OverflowError):
        modelXbrl.error("streamingExtensions:versionError",
                _("Version %(version)s, number must be 1.n"),
                modelObject=elt, version=streamingAspects.get("version", "(none)"))
        foundErrors = True
    for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
        try:
            bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
            if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
                raise InvalidOperation
            elif bufAspect == "contextBuffer":
                contextBufferLimit = bufLimit
            elif bufAspect == "unitBuffer":
                unitBufferLimit = bufLimit
            elif bufAspect == "footnoteBuffer":
                footnoteBufferLimit = bufLimit
        except InvalidOperation:
            modelXbrl.error("streamingExtensions:valueError",
                    _("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"),
                    modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect))
            foundErrors = True
    if parsercontext.error_log:
        foundErrors = True
    logSyntaxErrors(parsercontext)
    
    if foundErrors:
        _file.close()
        return None
    parsercontext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
    _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl,filepath)
    eltMdlObjs = {}
    beforeInstanceStream = True
    validator = None
    contextBuffer = []
    unitBuffer = []
    footnoteBuffer = []
    factBuffer = []
    numFacts = numRootFacts2 = 1
    for event, elt in parsercontext:
        if event == "start":
            mdlObj = _parser.makeelement(elt.tag, attrib=elt.attrib, nsmap=elt.nsmap)
            mdlObj.sourceline = elt.sourceline
            eltMdlObjs[elt] = mdlObj
            if elt.getparent() is None:
                modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, etree.ElementTree(mdlObj))
                modelDocument.xmlRootElement = mdlObj
                modelXbrl.modelDocument = modelDocument # needed for incremental validation
                mdlObj.init(modelDocument)
                modelXbrl.info("streamingExtensions:streaming",
                               _("Stream processing this instance."),
                               modelObject = modelDocument)    
            else:
                eltMdlObjs[elt.getparent()].append(mdlObj)
                mdlObj._init()
                ns = mdlObj.namespaceURI
                ln = mdlObj.localName
                if (beforeInstanceStream and (
                    (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
                    (ns == XbrlConst.xbrli and ln in ("context", "unit")) or
                    (ns not in (XbrlConst.link, XbrlConst.xbrli)))):
                    beforeInstanceStream = False
                    if _streamingExtensionsValidate:
                        validator = Validate(modelXbrl)
                        validator.instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters())
                    else: # need default dimensions
                        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
            mdlObj = None # deref
                        
        elif event == "end":
            mdlObj = eltMdlObjs.pop(elt)
            if elt.text: # text available after child nodes processed
                mdlObj.text = elt.text
            ns = mdlObj.namespaceURI
            ln = mdlObj.localName
            parentMdlObj = mdlObj.getparent()
            if ns == XbrlConst.xbrli:
                if ln == "context":
                    if mdlObj.get("sticky"):
                        del mdlObj.attrib["sticky"]
                        modelDocument.contextDiscover(mdlObj)
                    else:
                        if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
                            # drop before adding as dropped may have same id as added
                            cntx = contextBuffer.pop(0)
                            dropContext(modelXbrl, cntx)
                            del parentMdlObj[parentMdlObj.index(cntx)]
                            cntx = None
                        modelDocument.contextDiscover(mdlObj)
                        if contextBufferLimit.is_finite():
                            contextBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        contextsToCheck = (mdlObj,)
                        validator.instValidator.checkContexts(contextsToCheck)
                        if modelXbrl.hasXDT:
                            validator.instValidator.checkContextsDimensions(contextsToCheck)
                        del contextsToCheck # dereference
                elif ln == "unit":
                    if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
                        # drop before additing as dropped may have same id as added
                        unit = unitBuffer.pop(0)
                        dropUnit(modelXbrl, unit)
                        del parentMdlObj[parentMdlObj.index(unit)]
                        unit = None 
                    modelDocument.unitDiscover(mdlObj)
                    if unitBufferLimit.is_finite():
                        unitBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        validator.instValidator.checkUnits( (mdlObj,) )
                elif ln == "xbrl": # end of document
                    # check remaining footnote refs
                    for footnoteLink in footnoteBuffer:
                        checkFootnoteHrefs(modelXbrl, footnoteLink)
                elt.clear()
            elif ns == XbrlConst.link:
                if ln in ("schemaRef", "linkbaseRef"):
                    modelDocument.discoverHref(mdlObj)
                elif ln in ("roleRef", "arcroleRef"):
                    modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
                elif ln == "footnoteLink":
                    footnoteLinks = (mdlObj,)
                    modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
                    if footnoteBufferLimit.is_finite():
                        footnoteBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        validator.instValidator.checkLinks(footnoteLinks)
                        if len(footnoteBuffer) > footnoteBufferLimit:
                            # check that hrefObjects for locators were all satisfied
                                # drop before addition as dropped may have same id as added
                            footnoteLink = footnoteBuffer.pop(0)
                            checkFootnoteHrefs(modelXbrl, footnoteLink)
                            dropFootnoteLink(modelXbrl, footnoteLink)
                            del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            footnoteLink = None
                    footnoteLinks = None
                elt.clear()
            elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
                numRootFacts2 += 1
                modelDocument.factDiscover(mdlObj, modelXbrl.facts)
                XmlValidate.validate(modelXbrl, mdlObj)
                if _streamingExtensionsValidate:
                    factsToCheck = (mdlObj,)
                    validator.instValidator.checkFacts(factsToCheck)
                    if modelXbrl.hasXDT:
                        validator.instValidator.checkFactsDimensions(factsToCheck)
                    del factsToCheck
                    dropFact(modelXbrl, mdlObj, modelXbrl.facts)
                    del parentMdlObj[parentMdlObj.index(mdlObj)]
                if numRootFacts2 % 1000 == 0:
                    modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(numRootFacts2, numRootFacts1, 100.0 * numRootFacts2 / numRootFacts1), 
                                              minTimeToShow=20.0)
                # get rid of root element from iterparse's tree
                elt.clear()
                while elt.getprevious() is not None:  # cleans up any prior siblings
                    del elt.getparent()[0]
            mdlObj = None # deref
    logSyntaxErrors(parsercontext)
    del parsercontext
    if validator is not None:
        validator.close()
    _file.close()
    modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
    return modelDocument
Example #30
0
def evaluateTableIndex(modelXbrl):
    disclosureSystem = modelXbrl.modelManager.disclosureSystem
    if disclosureSystem.EFM:
        COVER = "1Cover"
        STMTS = "2Financial Statements"
        NOTES = "3Notes to Financial Statements"
        POLICIES = "4Accounting Policies"
        TABLES = "5Notes Tables"
        DETAILS = "6Notes Details"
        UNCATEG = "7Uncategorized"
        roleDefinitionPattern = re.compile(
            r"([0-9]+) - (Statement|Disclosure|Schedule|Document) - (.+)")
        # build EFM rendering-compatible index
        definitionElrs = dict(
            (roleType.definition, roleType)
            for roleURI in modelXbrl.relationshipSet(
                XbrlConst.parentChild).linkRoleUris
            for roleType in modelXbrl.roleTypes.get(roleURI, ()))
        isRR = any(
            ns.startswith("http://xbrl.sec.gov/rr/")
            for ns in modelXbrl.namespaceDocs.keys())
        tableGroup = None
        firstTableLinkroleURI = None
        firstDocumentLinkroleURI = None
        sortedRoleTypes = sorted(definitionElrs.items(),
                                 key=lambda item: item[0])
        for roleDefinition, roleType in sortedRoleTypes:
            roleType._tableChildren = []
            match = roleDefinitionPattern.match(
                roleDefinition) if roleDefinition else None
            if not match:
                roleType._tableIndex = (UNCATEG, "", roleType.roleURI)
                continue
            seq, tblType, tblName = match.groups()
            if isRR:
                tableGroup = COVER
            elif not tableGroup:
                tableGroup = ("Paren" in tblName and COVER
                              or tblType == "Statement" and STMTS
                              or "(Polic" in tblName and NOTES
                              or "(Table" in tblName and TABLES
                              or "(Detail" in tblName and DETAILS or COVER)
            elif tableGroup == COVER:
                tableGroup = (tblType == "Statement" and STMTS
                              or "Paren" in tblName and COVER
                              or "(Polic" in tblName and NOTES
                              or "(Table" in tblName and TABLES
                              or "(Detail" in tblName and DETAILS or NOTES)
            elif tableGroup == STMTS:
                tableGroup = ((tblType == "Statement" or "Paren" in tblName)
                              and STMTS or "(Polic" in tblName and NOTES
                              or "(Table" in tblName and TABLES
                              or "(Detail" in tblName and DETAILS or NOTES)
            elif tableGroup == NOTES:
                tableGroup = ("(Polic" in tblName and POLICIES
                              or "(Table" in tblName and TABLES
                              or "(Detail" in tblName and DETAILS
                              or tblType == "Disclosure" and NOTES or UNCATEG)
            elif tableGroup == POLICIES:
                tableGroup = ("(Table" in tblName and TABLES
                              or "(Detail" in tblName and DETAILS
                              or ("Paren" in tblName or "(Polic" in tblName)
                              and POLICIES or UNCATEG)
            elif tableGroup == TABLES:
                tableGroup = ("(Detail" in tblName and DETAILS
                              or ("Paren" in tblName or "(Table" in tblName)
                              and TABLES or UNCATEG)
            elif tableGroup == DETAILS:
                tableGroup = (("Paren" in tblName or "(Detail" in tblName)
                              and DETAILS or UNCATEG)
            else:
                tableGroup = UNCATEG
            if firstTableLinkroleURI is None and tableGroup == COVER:
                firstTableLinkroleURI = roleType.roleURI
            if tblType == "Document" and not firstDocumentLinkroleURI:
                firstDocumentLinkroleURI = roleType.roleURI
            roleType._tableIndex = (tableGroup, seq, tblName)

        # flow allocate facts to roles (SEC presentation groups)
        if not modelXbrl.qnameDimensionDefaults:  # may not have run validatino yet
            from arelle import ValidateXbrlDimensions
            ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
        reportedFacts = set(
        )  # facts which were shown in a higher-numbered ELR table
        reportingPeriods = set()
        nextEnd = None
        deiFact = {}
        for conceptName in ("DocumentPeriodEndDate", "DocumentType",
                            "CurrentFiscalPeriodEndDate"):
            for concept in modelXbrl.nameConcepts[conceptName]:
                for fact in modelXbrl.factsByQname(concept.qname):
                    deiFact[conceptName] = fact
                    if fact.context is not None:
                        reportingPeriods.add(
                            (None, fact.context.endDatetime))  # for instant
                        reportingPeriods.add(
                            (fact.context.startDatetime,
                             fact.context.endDatetime))  # for startEnd
                        nextEnd = fact.context.startDatetime
                        duration = (fact.context.endDatetime -
                                    fact.context.startDatetime).days + 1
                        break
        if "DocumentType" in deiFact:
            fact = deiFact["DocumentType"]
            if "-Q" in fact.xValue:
                # need quarterly and yr to date durations
                endDatetime = fact.context.endDatetime
                # if within 2 days of end of month use last day of month
                endDatetimeMonth = endDatetime.month
                if (endDatetime + timedelta(2)).month != endDatetimeMonth:
                    # near end of month
                    endOfMonth = True
                    while endDatetime.month == endDatetimeMonth:
                        endDatetime += timedelta(1)  # go forward to next month
                else:
                    endOfMonth = False
                startYr = endDatetime.year
                startMo = endDatetime.month - 3
                if startMo <= 0:
                    startMo += 12
                    startYr -= 1
                startDatetime = datetime(startYr, startMo, endDatetime.day,
                                         endDatetime.hour, endDatetime.minute,
                                         endDatetime.second)
                if endOfMonth:
                    startDatetime -= timedelta(1)
                    endDatetime -= timedelta(1)
                reportingPeriods.add((startDatetime, endDatetime))
                duration = 91
        # find preceding compatible default context periods
        while (nextEnd is not None):
            thisEnd = nextEnd
            prevMaxStart = thisEnd - timedelta(duration * .9)
            prevMinStart = thisEnd - timedelta(duration * 1.1)
            nextEnd = None
            for cntx in modelXbrl.contexts.values():
                if cntx is not None:
                    if (cntx.isStartEndPeriod and not cntx.qnameDims
                            and thisEnd == cntx.endDatetime and prevMinStart <=
                            cntx.startDatetime <= prevMaxStart):
                        reportingPeriods.add((None, cntx.endDatetime))
                        reportingPeriods.add(
                            (cntx.startDatetime, cntx.endDatetime))
                        nextEnd = cntx.startDatetime
                        break
                    elif (cntx.isInstantPeriod and not cntx.qnameDims
                          and thisEnd == cntx.endDatetime):
                        reportingPeriods.add((None, cntx.endDatetime))
        stmtReportingPeriods = set(reportingPeriods)

        sortedRoleTypes.reverse()  # now in descending order
        for i, roleTypes in enumerate(sortedRoleTypes):
            roleDefinition, roleType = roleTypes
            # find defined non-default axes in pre hierarchy for table
            tableFacts = set()
            tableGroup, tableSeq, tableName = roleType._tableIndex
            roleURIdims, priItemQNames = EFMlinkRoleURIstructure(
                modelXbrl, roleType.roleURI)
            for priItemQName in priItemQNames:
                for fact in modelXbrl.factsByQname(priItemQName):
                    cntx = fact.context
                    # non-explicit dims must be default
                    if (cntx is not None
                            and all(dimQn in modelXbrl.qnameDimensionDefaults
                                    for dimQn in (roleURIdims.keys() -
                                                  cntx.qnameDims.keys())) and
                            all(mdlDim.memberQname in roleURIdims[dimQn]
                                for dimQn, mdlDim in cntx.qnameDims.items()
                                if dimQn in roleURIdims)):
                        # the flow-up part, drop
                        cntxStartDatetime = cntx.startDatetime
                        cntxEndDatetime = cntx.endDatetime
                        if (tableGroup != STMTS
                                or (cntxStartDatetime, cntxEndDatetime)
                                in stmtReportingPeriods and
                            (fact not in reportedFacts or all(
                                dimQn not in cntx.
                                qnameDims  # unspecified dims are all defaulted if reported elsewhere
                                for dimQn in (cntx.qnameDims.keys() -
                                              roleURIdims.keys())))):
                            tableFacts.add(fact)
                            reportedFacts.add(fact)
            roleType._tableFacts = tableFacts

            # find parent if any
            closestParentType = None
            closestParentMatchLength = 0
            for _parentRoleDefinition, parentRoleType in sortedRoleTypes[i +
                                                                         1:]:
                matchLen = parentNameMatchLen(tableName, parentRoleType)
                if matchLen > closestParentMatchLength:
                    closestParentMatchLength = matchLen
                    closestParentType = parentRoleType
            if closestParentType is not None:
                closestParentType._tableChildren.insert(0, roleType)

            # remove lesser-matched children if there was a parent match
            unmatchedChildRoles = set()
            longestChildMatchLen = 0
            numChildren = 0
            for childRoleType in roleType._tableChildren:
                matchLen = parentNameMatchLen(tableName, childRoleType)
                if matchLen < closestParentMatchLength:
                    unmatchedChildRoles.add(childRoleType)
                elif matchLen > longestChildMatchLen:
                    longestChildMatchLen = matchLen
                    numChildren += 1
            if numChildren > 1:
                # remove children that don't have the full match pattern length to parent
                for childRoleType in roleType._tableChildren:
                    if (childRoleType not in unmatchedChildRoles
                            and parentNameMatchLen(tableName, childRoleType) <
                            longestChildMatchLen):
                        unmatchedChildRoles.add(childRoleType)

            for unmatchedChildRole in unmatchedChildRoles:
                roleType._tableChildren.remove(unmatchedChildRole)

            for childRoleType in roleType._tableChildren:
                childRoleType._tableParent = roleType

            unmatchedChildRoles = None  # dereference

        global UGT_TOPICS
        if UGT_TOPICS is None:
            try:
                from arelle import FileSource
                fh = FileSource.openFileStream(
                    modelXbrl.modelManager.cntlr,
                    os.path.join(modelXbrl.modelManager.cntlr.configDir,
                                 "ugt-topics.zip/ugt-topics.json"), 'r',
                    'utf-8')
                UGT_TOPICS = json.load(fh)
                fh.close()
                for topic in UGT_TOPICS:
                    topic[6] = set(
                        topic[6]
                    )  # change concept abstracts list into concept abstracts set
                    topic[7] = set(
                        topic[7]
                    )  # change concept text blocks list into concept text blocks set
                    topic[8] = set(
                        topic[8]
                    )  # change concept names list into concept names set
            except Exception as ex:
                UGT_TOPICS = None

        if UGT_TOPICS is not None:

            def roleUgtConcepts(roleType):
                roleConcepts = set()
                for rel in modelXbrl.relationshipSet(
                        XbrlConst.parentChild,
                        roleType.roleURI).modelRelationships:
                    if rel.toModelObject is not None:
                        roleConcepts.add(rel.toModelObject.name)
                    if rel.fromModelObject is not None:
                        roleConcepts.add(rel.fromModelObject.name)
                if hasattr(roleType, "_tableChildren"):
                    for _tableChild in roleType._tableChildren:
                        roleConcepts |= roleUgtConcepts(_tableChild)
                return roleConcepts

            topicMatches = {}  # topicNum: (best score, roleType)

            for roleDefinition, roleType in sortedRoleTypes:
                roleTopicType = 'S' if roleDefinition.startswith('S') else 'D'
                if getattr(roleType, "_tableParent", None) is None:
                    # rooted tables in reverse order
                    concepts = roleUgtConcepts(roleType)
                    for i, ugtTopic in enumerate(UGT_TOPICS):
                        if ugtTopic[0] == roleTopicType:
                            countAbstracts = len(concepts & ugtTopic[6])
                            countTextBlocks = len(concepts & ugtTopic[7])
                            countLineItems = len(concepts & ugtTopic[8])
                            if countAbstracts or countTextBlocks or countLineItems:
                                _score = (10 * countAbstracts +
                                          1000 * countTextBlocks +
                                          countLineItems / len(concepts))
                                if i not in topicMatches or _score > topicMatches[
                                        i][0]:
                                    topicMatches[i] = (_score, roleType)
            for topicNum, scoredRoleType in topicMatches.items():
                _score, roleType = scoredRoleType
                if _score > getattr(roleType, "_tableTopicScore", 0):
                    ugtTopic = UGT_TOPICS[topicNum]
                    roleType._tableTopicScore = _score
                    roleType._tableTopicType = ugtTopic[0]
                    roleType._tableTopicName = ugtTopic[3]
                    roleType._tableTopicCode = ugtTopic[4]
                    # print ("Match score {:.2f} topic {} preGrp {}".format(_score, ugtTopic[3], roleType.definition))
        return firstTableLinkroleURI or firstDocumentLinkroleURI  # did build _tableIndex attributes
    return None
Example #31
0
def evaluateTableIndex(modelXbrl):
    disclosureSystem = modelXbrl.modelManager.disclosureSystem
    if disclosureSystem.EFM:
        COVER    = "1Cover"
        STMTS    = "2Financial Statements"
        NOTES    = "3Notes to Financial Statements"
        POLICIES = "4Accounting Policies"
        TABLES   = "5Notes Tables"
        DETAILS  = "6Notes Details"
        UNCATEG  = "7Uncategorized"
        roleDefinitionPattern = re.compile(r"([0-9]+) - (Statement|Disclosure|Schedule|Document) - (.+)")
        # build EFM rendering-compatible index
        definitionElrs = dict((roleType.definition, roleType)
                              for roleURI in modelXbrl.relationshipSet(XbrlConst.parentChild).linkRoleUris
                              for roleType in modelXbrl.roleTypes.get(roleURI,()))
        isRR = any(ns.startswith("http://xbrl.sec.gov/rr/") for ns in modelXbrl.namespaceDocs.keys())
        tableGroup = None
        firstTableLinkroleURI = None
        firstDocumentLinkroleURI = None
        sortedRoleTypes = sorted(definitionElrs.items(), key=lambda item: item[0])
        for roleDefinition, roleType in sortedRoleTypes:
            match = roleDefinitionPattern.match(roleDefinition) if roleDefinition else None
            if not match: 
                roleType._tableIndex = (UNCATEG, roleType.roleURI)
                continue
            seq, tblType, tblName = match.groups()
            if isRR:
                tableGroup = COVER
            elif not tableGroup:
                tableGroup = ("Paren" in tblName and COVER or tblType == "Statement" and STMTS or
                              "(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or
                              "(Detail" in tblName and DETAILS or COVER)
            elif tableGroup == COVER:
                tableGroup = (tblType == "Statement" and STMTS or "Paren" in tblName and COVER or
                              "(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or
                              "(Detail" in tblName and DETAILS or NOTES)
            elif tableGroup == STMTS:
                tableGroup = ((tblType == "Statement" or "Paren" in tblName) and STMTS or
                              "(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or
                              "(Detail" in tblName and DETAILS or NOTES)
            elif tableGroup == NOTES:
                tableGroup = ("(Polic" in tblName and POLICIES or "(Table" in tblName and TABLES or 
                              "(Detail" in tblName and DETAILS or tblType == "Disclosure" and NOTES or UNCATEG)
            elif tableGroup == POLICIES:
                tableGroup = ("(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or 
                              ("Paren" in tblName or "(Polic" in tblName) and POLICIES or UNCATEG)
            elif tableGroup == TABLES:
                tableGroup = ("(Detail" in tblName and DETAILS or 
                              ("Paren" in tblName or "(Table" in tblName) and TABLES or UNCATEG)
            elif tableGroup == DETAILS:
                tableGroup = (("Paren" in tblName or "(Detail" in tblName) and DETAILS or UNCATEG)
            else:
                tableGroup = UNCATEG
            if firstTableLinkroleURI is None and tableGroup == COVER:
                firstTableLinkroleURI = roleType.roleURI
            if tblType == "Document" and not firstDocumentLinkroleURI:
                firstDocumentLinkroleURI = roleType.roleURI
            roleType._tableIndex = (tableGroup, seq, tblName)

        # flow allocate facts to roles (SEC presentation groups)
        if not modelXbrl.qnameDimensionDefaults: # may not have run validatino yet
            from arelle import ValidateXbrlDimensions
            ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
        reportedFacts = set() # facts which were shown in a higher-numbered ELR table
        factsByQname = modelXbrl.factsByQname
        reportingPeriods = set()
        nextEnd = None
        deiFact = {}
        for conceptName in ("DocumentPeriodEndDate", "DocumentType", "CurrentFiscalPeriodEndDate"):
            for concept in modelXbrl.nameConcepts[conceptName]:
                for fact in factsByQname[concept.qname]:
                    deiFact[conceptName] = fact
                    if fact.context is not None:
                        reportingPeriods.add((None, fact.context.endDatetime)) # for instant
                        reportingPeriods.add((fact.context.startDatetime, fact.context.endDatetime)) # for startEnd
                        nextEnd = fact.context.startDatetime
                        duration = (fact.context.endDatetime - fact.context.startDatetime).days + 1
                        break
        if "DocumentType" in deiFact:
            fact = deiFact["DocumentType"]
            if "-Q" in fact.xValue:
                # need quarterly and yr to date durations
                endDatetime = fact.context.endDatetime
                # if within 2 days of end of month use last day of month
                endDatetimeMonth = endDatetime.month
                if (endDatetime + timedelta(2)).month != endDatetimeMonth:
                    # near end of month
                    endOfMonth = True
                    while endDatetime.month == endDatetimeMonth:
                        endDatetime += timedelta(1) # go forward to next month
                else:
                    endOfMonth = False
                startYr = endDatetime.year
                startMo = endDatetime.month - 3
                if startMo < 0:
                    startMo += 12
                    startYr -= 1
                startDatetime = datetime(startYr, startMo, endDatetime.day, endDatetime.hour, endDatetime.minute, endDatetime.second)
                if endOfMonth:
                    startDatetime -= timedelta(1)
                    endDatetime -= timedelta(1)
                reportingPeriods.add((startDatetime, endDatetime))
                duration = 91
        # find preceding compatible default context periods
        while (nextEnd is not None):
            thisEnd = nextEnd
            prevMaxStart = thisEnd - timedelta(duration * .9)
            prevMinStart = thisEnd - timedelta(duration * 1.1)
            nextEnd = None
            for cntx in modelXbrl.contexts.values():
                if (cntx.isStartEndPeriod and not cntx.qnameDims and thisEnd == cntx.endDatetime and
                    prevMinStart <= cntx.startDatetime <= prevMaxStart):
                    reportingPeriods.add((None, cntx.endDatetime))
                    reportingPeriods.add((cntx.startDatetime, cntx.endDatetime))
                    nextEnd = cntx.startDatetime
                    break
                elif (cntx.isInstantPeriod and not cntx.qnameDims and thisEnd == cntx.endDatetime):
                    reportingPeriods.add((None, cntx.endDatetime))
        stmtReportingPeriods = set(reportingPeriods)       

        for roleDefinition, roleType in reversed(sortedRoleTypes):
            # find defined non-default axes in pre hierarchy for table
            tableFacts = set()
            tableGroup = roleType._tableIndex[0]
            roleURIdims, priItemQNames = EFMlinkRoleURIstructure(modelXbrl, roleType.roleURI)
            for priItemQName in priItemQNames:
                for fact in factsByQname[priItemQName]:
                    cntx = fact.context
                    # non-explicit dims must be default
                    if (cntx is not None and
                        all(dimQn in modelXbrl.qnameDimensionDefaults
                            for dimQn in (roleURIdims.keys() - cntx.qnameDims.keys())) and
                        all(mdlDim.memberQname in roleURIdims[dimQn]
                            for dimQn, mdlDim in cntx.qnameDims.items()
                            if dimQn in roleURIdims)):
                        # the flow-up part, drop
                        cntxStartDatetime = cntx.startDatetime
                        cntxEndDatetime = cntx.endDatetime
                        if (tableGroup != STMTS or
                            (cntxStartDatetime, cntxEndDatetime) in stmtReportingPeriods and
                             (fact not in reportedFacts or
                              all(dimQn not in cntx.qnameDims # unspecified dims are all defaulted if reported elsewhere
                                  for dimQn in (cntx.qnameDims.keys() - roleURIdims.keys())))):
                            tableFacts.add(fact)
                            reportedFacts.add(fact)
            roleType._tableFacts = tableFacts
            
        return firstTableLinkroleURI or firstDocumentLinkroleURI # did build _tableIndex attributes
    return None
Example #32
0
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
    from openpyxl import load_workbook
    from arelle import ModelDocument, ModelXbrl, XmlUtil
    from arelle.ModelDocument import ModelDocumentReference
    from arelle.ModelValue import qname
    
    try:
        currentAction = "initializing"
        startedAt = time.time()
        
        if os.path.isabs(oimFile):
            # allow relative filenames to loading directory
            priorCWD = os.getcwd()
            os.chdir(os.path.dirname(oimFile))
        else:
            priorCWD = None
            
        currentAction = "determining file type"
        isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json")
        isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
        isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
        isCSVorXL = isCSV or isXL
        instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"
        
        if isJSON:
            currentAction = "loading and parsing JSON OIM file"
            with io.open(oimFile, 'rt', encoding='utf-8') as f:
                oimObject = json.load(f)
            missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject]
            if missing:
                raise OIMException("oime:missingJSONelements", 
                                   _("The required elements %(missing)s {} in JSON input"),
                                   missing = ", ".join(missing))
            currentAction = "identifying JSON objects"
            dtsReferences = oimObject["dtsReferences"]
            prefixes = oimObject["prefixes"]
            facts = oimObject["facts"]
            footnotes = oimOject["facts"] # shares this object
        elif isCSV:
            currentAction = "identifying CSV input tables"
            if sys.version[0] >= '3':
                csvOpenMode = 'w'
                csvOpenNewline = ''
            else:
                csvOpenMode = 'wb' # for 2.7
                csvOpenNewline = None
                
            oimFileBase = None
            if "-facts" in oimFile:
                oimFileBase = oimFile.partition("-facts")[0]
            else:
                for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"):
                    if oimFile.endswith(suffix):
                        oimFileBase = oimFile[:-length(suffix)]
                        break
            if oimFileBase is None:
                raise OIMException("oime:missingCSVtables", 
                                   _("Unable to identify CSV tables file name pattern"))
            if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or
                not os.path.exists(oimFileBase + "-prefixes.csv")):
                raise OIMException("oime:missingCSVtables", 
                                   _("Unable to identify CSV tables for dtsReferences or prefixes"))
            instanceFileName = oimFileBase + ".xbrl"
            currentAction = "loading CSV dtsReferences table"
            dtsReferences = []
            with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = row
                    else:
                        dtsReferences.append(dict((header[j], col) for j, col in enumerate(row)))
            currentAction = "loading CSV prefixes table"
            prefixes = {}
            with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = dict((col,i) for i,col in enumerate(row))
                    else:
                        prefixes[row[header["prefix"]]] = row[header["URI"]]
            currentAction = "loading CSV defaults table"
            defaults = {}
            if os.path.exists(oimFileBase + "-defaults.csv"):
                with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                            fileCol = row.index("file")
                        else:
                            defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol)
            currentAction = "loading CSV facts tables"
            facts = []
            _dir = os.path.dirname(oimFileBase)
            for filename in os.listdir(_dir):
                filepath = os.path.join(_dir, filename)
                if "-facts" in filename and filepath.startswith(oimFileBase):
                    tableDefaults = defaults.get(filename, {})
                    with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
                        csvReader = csv.reader(f)
                        for i, row in enumerate(csvReader):
                            if i == 0:
                                header = row
                            else:
                                fact = {}
                                fact.update(tableDefaults)
                                for j, col in enumerate(row):
                                    if col is not None:
                                        if header[j].endswith("Value"):
                                            if col: # ignore empty columns (= null CSV value)
                                                fact["value"] = col
                                        else:
                                            fact[header[j]] = col
                                facts.append(fact)
            footnotes = []
            if os.path.exists(oimFileBase + "-footnotes.csv"):
                with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                        else:
                            footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col))
        elif isXL:
            oimWb = load_workbook(oimFile, read_only=True, data_only=True)
            sheetNames = oimWb.get_sheet_names()
            if (not any(sheetName == "prefixes" for sheetName in sheetNames) or
                not any(sheetName == "dtsReferences" for sheetName in sheetNames) or
                not any("facts" in sheetName for sheetName in sheetNames)):
                if priorCWD: os.chdir(priorCWD)
                return None
            try:
                dtsReferences = []
                for i, row in oimWb["dtsReferences"]:
                    if i == 0:
                        header = [col.value for col in row]
                    else:
                        dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row)))
                prefixes = {}
                for i, row in oimWb["dtsReferences"]:
                    if i == 0:
                        header = dict((col.value,i) for i,col in enumerate(row))
                    else:
                        prefixes[row[header["prefix"]].value] = row[header["URI"].value]
                defaults = {}
                for i, row in oimW.get("defaults", ()):
                    if i == 0:
                        header = dict((col.value,i) for i,col in enumerate(row))
                        fileCol = header["file"]
                    else:
                        defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol)
                facts = []
                _dir = os.path.dirpart(oimFileBase)
                for sheetName in sheetNames:
                    if sheetName == "facts" or "-facts" in sheetName:
                        for i, row in oimWb[sheetName]:
                            if i == 0:
                                header = [col.value for col in row]
                            else:
                                fact = {}
                                fact.update(tableDefaults)
                                for j, col in enumerate(row):
                                    if col.value is not None:
                                        if header[j].endswith("Value"):
                                            fact["value"] = str(col.value)
                                        else:
                                            fact[header[j]] = str(col.value)
                                facts.append(fact)
                footnotes = []
                for i, row in oimWb.get("footnotes", ()):
                    if i == 0:
                        header = dict((col.value,i) for i,col in enumerate(row) if col.value)
                    else:
                        footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value))
            except Exception as ex:
                if priorCWD: os.chdir(priorCWD)
                return None
    
        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults 
            
        # create the instance document
        modelXbrl.blockDpmDBrecursion = True
        modelXbrl.modelDocument = createModelDocument(
              modelXbrl, 
              Type.INSTANCE,
              instanceFileName,
              schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"],
              isEntry=True,
              initialComment="extracted from OIM {}".format(mappedUri),
              documentEncoding="utf-8")
        cntxTbl = {}
        unitTbl = {}
        for fact in facts:
            conceptQn = qname(fact["oim:concept"], prefixes)
            concept = modelXbrl.qnameConcepts.get(conceptQn)
            entityAsQn = qname(fact["oim:entity"], prefixes)
            if "oim:period" in fact:
                periodStart = fact["oim:period"]["start"]
                periodEnd = fact["oim:period"]["end"]
            else:
                periodStart = fact["oim:periodStart"]
                periodEnd = fact["oim:periodEnd"]
            cntxKey = ( # hashable context key
                ("periodType", concept.periodType),
                ("entity", entityAsQn),
                ("periodStart", periodStart),
                ("periodEnd", periodEnd)) + tuple(sorted(
                    (dimName, dimVal) 
                    for dimName, dimVal in fact.items()
                    if ":" in dimName and not dimName.startswith("oim:")))
            if cntxKey in cntxTbl:
                _cntx = cntxTbl[cntxKey]
            else:
                cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                qnameDims = {}
                for dimName, dimVal in fact.items():
                    if ":" in dimName and not dimName.startswith("oim:"):
                        dimQname = qname(dimName, prefixes)
                        dimConcept = modelXbrl.qnameConcepts.get(dimQname)
                        if ":" in dimVal and dimVal.partition[':'][0] in prefixes:
                            mem = qname(dimVal, prefixes) # explicit dim
                        elif dimConcept.isTypedDimension:
                            # a modelObject xml element is needed for all of the instance functions to manage the typed dim
                            mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
                        qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
                _cntx = modelXbrl.createContext(
                                        entityAsQn.namespaceURI,
                                        entityAsQn.localName,
                                        concept.periodType,
                                        None if concept.periodType == "instnat" else dateTime(periodStart, type=DATETIME),
                                        dateTime(periodEnd, type=DATETIME),
                                        None, # no dimensional validity checking (like formula does)
                                        qnameDims, [], [],
                                        id=cntxId)
                cntxTbl[cntxKey] = _cntx
            if "oim:unit" in fact:
                unitKey = fact["oim:unit"]
                if unitKey in unitTbl:
                    _unit = unitTbl[unitKey]
                else:
                    _mul, _sep, _div = unitKey.partition('/')
                    if _mul.startswith('('):
                        _mul = _mul[1:-1]
                    mulQns = [qname(u, prefixes) for u in _mul.split('*') if u]
                    if _div.startswith('('):
                        _div = _div[1:-1]
                    divQns = [qname(u, prefixes) for u in _div.split('*') if u]
                    unitId = 'u-{:02}'.format(len(unitTbl) + 1)
                    for _measures in mulQns, divQns:
                        for _measure in _measures:
                            addQnameValue(modelXbrl.modelDocument, _measure)
                    _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId)
                    unitTbl[unitKey] = _unit
            else:
                _unit = None
            
            attrs = {"contextRef": _cntx.id}
    
            if fact.get("value") is None:
                attrs[XbrlConst.qnXsiNil] = "true"
                text = None
            else:
                text = fact["value"]
                
            if fact.get("id"):
                attrs["id"] = fact["id"]
                
            if concept.isNumeric:
                if _unit is not None:
                    attrs["unitRef"] = _unit.id
                if "accuracy" in fact:
                    attrs["decimals"] = fact["accuracy"]
                    
            # is value a QName?
            if concept.baseXbrliType == "QName":
                addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
    
            f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
            
        footnoteLinks = {} # ELR elements
        factLocs = {} # index by (linkrole, factId)
        footnoteNbr = 0
        locNbr = 0
        for factOrFootnote in footnotes:
            if "factId" in factOrFootnote:
                factId = factOrFootnote["factId"]
                factFootnotes = (factOrFootnote,) # CSV or XL
            elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
                factId = factOrFootnote["id"]
                factFootnotes = factOrFootnote["footnotes"]
            else:
                factFootnotes = ()
            for footnote in factFootnotes:
                linkrole = footnote.get("group")
                arcrole = footnote.get("footnoteType")
                if not factId or not linkrole or not arcrole or not (
                    footnote.get("factRef") or footnote.get("footnote")):
                    # invalid footnote
                    continue
                if linkrole not in footnoteLinks:
                    footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, 
                                                       XbrlConst.qnLinkFootnoteLink, 
                                                       attributes={"{http://www.w3.org/1999/xlink}type": "extended",
                                                                   "{http://www.w3.org/1999/xlink}role": linkrole})
                footnoteLink = footnoteLinks[linkrole]
                if (linkrole, factId) not in factLocs:
                    locNbr += 1
                    locLabel = "l_{:02}".format(locNbr)
                    factLocs[(linkrole, factId)] = locLabel
                    addChild(footnoteLink, XbrlConst.qnLinkLoc, 
                             attributes={XLINKTYPE: "locator",
                                         XLINKHREF: "#" + factId,
                                         XLINKLABEL: locLabel})
                locLabel = factLocs[(linkrole, factId)]
                if footnote.get("footnote"):
                    footnoteNbr += 1
                    footnoteLabel = "f_{:02}".format(footnoteNbr)
                    attrs = {XLINKTYPE: "resource",
                             XLINKLABEL: footnoteLabel}
                    if footnote.get("language"):
                        attrs[XMLLANG] = footnote["language"]
                    # note, for HTML will need to build an element structure
                    addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"])
                elif footnote.get("factRef"):
                    factRef = footnote.get("factRef")
                    if (linkrole, factRef) not in factLocs:
                        locNbr += 1
                        locLabel = "f_{:02}".format(footnoteNbr)
                        factLoc[(linkrole, factRef)] = locLabel
                        addChild(footnoteLink, XbrlConst.qnLinkLoc, 
                                 attributes={XLINKTYPE: "locator",
                                             XLINKHREF: "#" + factRef,
                                             XLINKLABEL: locLabel})
                    footnoteLabel = factLoc[(linkrole, factId)]
                footnoteArc = addChild(footnoteLink, 
                                       XbrlConst.qnLinkFootnoteArc, 
                                       attributes={XLINKTYPE: "arc",
                                                   XLINKARCROLE: arcrole,
                                                   XLINKFROM: locLabel,
                                                   XLINKTO: footnoteLabel})
                    
        
        #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
        #               messageCode="loadFromExcel:info")
    except Exception as ex:
        if ex is OIMException:
            modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
        else:
            modelXbrl.error("Error while %(action)s, error %(error)s",
                            modelObject=modelXbrl, action=currentAction, error=ex)
    
    if priorCWD:
        os.chdir(priorCWD) # restore prior current working directory
    return modelXbrl.modelDocument
def init(modelXbrl):
    # setup modelXbrl for rendering evaluation

    # dimension defaults required in advance of validation
    from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument
    ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
    
    hasXbrlTables = False
    
    # validate table linkbase dimensions
    for baseSetKey in modelXbrl.baseSets.keys():
        arcrole, ELR, linkqname, arcqname = baseSetKey
        if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(arcrole):
            ValidateFormula.checkBaseSet(modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname))
            if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableAxis2011):
                hasXbrlTables = True

    # provide context for view
    if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE:
        instance = None # use instance of the entry pont
    else: # need dummy instance
        instance = ModelDocument.create(modelXbrl, ModelDocument.Type.INSTANCE, 
                                        "dummy.xml",  # fake URI and fake schemaRef 
                                        ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd",))
        
    if hasXbrlTables:
        # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables
        modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance)
        
        modelXbrl.profileStat(None)
        
        # setup fresh parameters from formula optoins
        modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters()
        
        # validate parameters and custom function signatures
        ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables"))
        
        # check and extract message expressions into compilable programs
        for msgArcrole in (XbrlConst.tableDefinitionNodeMessage, XbrlConst.tableDefinitionNodeSelectionMessage,
                           XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011):
            for msgRel in modelXbrl.relationshipSet(msgArcrole).modelRelationships:
                ValidateFormula.checkMessageExpressions(modelXbrl, msgRel.toModelObject)
                
        # compile and validate tables
        for modelTable in modelXbrl.modelRenderingTables:
            modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining
            modelTable.compile()

            # check aspectModel
            if modelTable.aspectModel not in ("non-dimensional", "dimensional"):
                modelXbrl.error("xbrlte:unknownAspectModel",
                    _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"),
                    modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel)
            else:
                modelTable.priorAspectAxisDisposition = {}
                # check ordinate aspects against aspectModel
                oppositeAspectModel = (_DICT_SET({'dimensional','non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop()
                uncoverableAspects = aspectModels[oppositeAspectModel] - aspectModels[modelTable.aspectModel]
                for tblAxisRel in modelXbrl.relationshipSet((XbrlConst.tableBreakdown,XbrlConst.tableAxis2011)).fromModelObject(modelTable):
                    checkDefinitionNodeAspectModel(modelXbrl, modelTable, tblAxisRel, uncoverableAspects)
                del modelTable.priorAspectAxisDisposition
    
        modelXbrl.profileStat(_("compileTables"))
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs):
    targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
    targetUrlParts = targetUrl.rpartition(".")
    targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
    modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
    targetInstance = ModelXbrl.create(modelXbrl.modelManager, 
                                      newDocumentType=Type.INSTANCE,
                                      url=targetUrl,
                                      schemaRefs=targetDocumentSchemaRefs,
                                      isEntry=True)
    ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults 
    # roleRef and arcroleRef (of each inline document)
    for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
        for roleRefElt in sourceRefs.values():
            XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, 
                             attributes=roleRefElt.items())
    
    # contexts
    for context in modelXbrl.contexts.values():
        newCntx = targetInstance.createContext(context.entityIdentifier[0],
                                               context.entityIdentifier[1],
                                               'instant' if context.isInstantPeriod else
                                               'duration' if context.isStartEndPeriod
                                               else 'forever',
                                               context.startDatetime,
                                               context.endDatetime,
                                               None, 
                                               context.qnameDims, [], [],
                                               id=context.id)
    for unit in modelXbrl.units.values():
        measures = unit.measures
        newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id)

    modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
    newFactForOldObjId = {}
    def createFacts(facts, parent):
        for fact in modelXbrl.facts:
            if fact.isItem:
                attrs = {"contextRef": fact.contextID}
                if fact.isNumeric:
                    attrs["unitRef"] = fact.unitID
                    if fact.get("decimals"):
                        attrs["decimals"] = fact.get("decimals")
                    if fact.get("precision"):
                        attrs["precision"] = fact.get("precision")
                if fact.isNil:
                    attrs[XbrlConst.qnXsiNil] = "true"
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.textValue
                newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newFact
            elif fact.isTuple:
                newTuple = targetInstance.createFact(fact.qname, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newTuple
                createFacts(fact.modelTupleFacts, newTuple)
                
    createFacts(modelXbrl.facts, None)
    # footnote links
    modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships"))
    for linkKey, linkPrototypes in modelXbrl.baseSets.items():
        arcrole, linkrole, linkqname, arcqname = linkKey
        if (linkrole and linkqname and arcqname and # fully specified roles
            any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
            for linkPrototype in linkPrototypes:
                newLink = XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, linkqname, 
                                           attributes=linkPrototype.attributes)
                for linkChild in linkPrototype:
                    if isinstance(linkChild, LocPrototype) and "{http://www.w3.org/1999/xlink}href" not in linkChild.attributes:
                        linkChild.attributes["{http://www.w3.org/1999/xlink}href"] = \
                        "#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
                    XmlUtil.addChild(newLink, linkChild.qname, 
                                     attributes=linkChild.attributes,
                                     text=linkChild.textValue)
            
    targetInstance.saveInstance(overrideFilepath=targetUrl)
    modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
Example #35
0
    def loadXbrlFromDB(self, loadDBsaveToFile):
        # load from database
        modelXbrl = self.modelXbrl
        
        # find instance in DB
        instanceURI = os.path.basename(loadDBsaveToFile)
        results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant"
                               " FROM dInstance WHERE FileName = '{}'"
                               .format(instanceURI))
        instanceId = moduleId = None
        for instanceId, moduleId, entScheme, entId, datePerEnd in results:
            break

        # find module in DB        
        results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId))
        xbrlSchemaRef = None
        for result in results:
            xbrlSchemaRef = result[0]
            break
        
        if not instanceId or not xbrlSchemaRef:
            raise XPDBException("sqlDB:MissingDTS",
                    _("The instance and module were not found for %(instanceURI)"),
                    instanceURI = instanceURI) 
            
        if modelXbrl.skipDTS:
            # find prefixes and namespaces in DB
            results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]")            
            dpmPrefixedNamespaces = dict((prefix, namespace)
                                         for owner, prefix, namespace in results)
            
        # create the instance document and resulting filing
        modelXbrl.blockDpmDBrecursion = True
        modelXbrl.modelDocument = createModelDocument(modelXbrl, 
                                                      Type.INSTANCE,
                                                      loadDBsaveToFile,
                                                      schemaRefs=[xbrlSchemaRef],
                                                      isEntry=True)
        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults 
        
        addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement, 
                                 'xbrl-streamable-instance', 
                                 'version="1.0" contextBuffer="1"')

        # add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue)
        
        # filing indicator code IDs
        # get filing indicators
        results = self.execute("SELECT mToT.TemplateOrTableCode "
                               "  FROM dFilingIndicator dFI, mTemplateOrTable mToT "
                               "  WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID"
                               .format(instanceId))
        filingIndicatorCodes = [code[0] for code in results]
        
        if filingIndicatorCodes:
            modelXbrl.createContext(entScheme,
                        entId,
                        'instant',
                        None,
                        datePerEnd,
                        None, # no dimensional validity checking (like formula does)
                        {}, [], [],
                        id='c')
            filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators)
            for filingIndicatorCode in filingIndicatorCodes:
                modelXbrl.createFact(qnFindFilingIndicator, 
                                     parent=filingIndicatorsTuple,
                                     attributes={"contextRef": "c"}, 
                                     text=filingIndicatorCode)

        
        # facts in this instance
        factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards,"
                                " Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue "
                                "FROM dFact WHERE InstanceID = {} "
                                "ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL "
                                "                          THEN DataPointSignature"
                                "                          ELSE DataPointSignatureWithValuesForWildcards"
                                "                END, instr(DataPointSignature,'|') + 1)"
                                .format(instanceId))
        
        # results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal

        # get typed dimension values
        prefixedNamespaces = modelXbrl.prefixedNamespaces
        prefixedNamespaces["iso4217"] = XbrlConst.iso4217
        if modelXbrl.skipDTS:
            prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed
        
        cntxTbl = {} # index by d
        unitTbl = {}
        
        def typedDimElt(s):
            # add xmlns into s for known qnames
            tag, angleBrkt, rest = s[1:].partition('>')
            text, angleBrkt, rest = rest.partition("<")
            qn = qname(tag, prefixedNamespaces)
            # a modelObject xml element is needed for all of the instance functions to manage the typed dim
            return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
        
        # contexts and facts
        for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl:
            metric, sep, dims = (dpSigTypedDims or dpSig).partition('|')
            conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces)
            concept = modelXbrl.qnameConcepts.get(conceptQn)
            isNumeric = isBool = isDateTime = isQName = isText = False
            if concept is not None:
                if concept.isNumeric:
                    isNumeric = True
                else:
                    baseXbrliType = concept.baseXbrliType
                    if baseXbrliType == "booleanItemType":
                        isBool = True
                    elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
                        isDateTime = True
                    elif baseXbrliType == "QNameItemType":
                        isQName = True
            else:
                c = conceptQn.localName[0]
                if c == 'm':
                    isNumeric = True
                elif c == 'd':
                    isDateTime = True
                elif c == 'b':
                    isBool = True
                elif c == 'e':
                    isQName = True
            isText = not (isNumeric or isBool or isDateTime or isQName)
            if isinstance(datePerEnd, _STR_BASE):
                datePerEnd = datetimeValue(datePerEnd, addOneDay=True)
            cntxKey = (dims, entId, datePerEnd)
            if cntxKey in cntxTbl:
                cntxId = cntxTbl[cntxKey]
            else:
                cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                cntxTbl[cntxKey] = cntxId
                qnameDims = {}
                if dims:
                    for dim in dims.split('|'):
                        dQn, sep, dVal = dim[:-1].partition('(')
                        dimQname = qname(dQn, prefixedNamespaces)
                        if dVal.startswith('<'):
                            mem = typedDimElt(dVal)  # typed dim
                        else:
                            mem = qname(dVal, prefixedNamespaces) # explicit dim (even if treat-as-typed)
                        qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario")
                    
                modelXbrl.createContext(entScheme,
                                        entId,
                                        'instant',
                                        None,
                                        datePerEnd,
                                        None, # no dimensional validity checking (like formula does)
                                        qnameDims, [], [],
                                        id=cntxId)
            if unit:
                if unit in unitTbl:
                    unitId = unitTbl[unit]
                else:
                    unitQn = qname(unit, prefixedNamespaces)
                    unitId = 'u{}'.format(unitQn.localName)
                    unitTbl[unit] = unitId
                    modelXbrl.createUnit([unitQn], [], id=unitId)
            else:
                unitId = None
            attrs = {"contextRef": cntxId}
            if unitId:
                attrs["unitRef"] = unitId
            if dec is not None:
                if isinstance(dec, float): # must be an integer
                    dec = int(dec)
                elif isinstance(dec, _STR_BASE) and '.' in dec:
                    dec = dec.partition('.')[0] # drop .0 from any SQLite string
                attrs["decimals"] = str(dec)  # somehow it is float from the database
            if False: # fact.isNil:
                attrs[XbrlConst.qnXsiNil] = "true"
                text = None
            elif numVal is not None:
                num = roundValue(numVal, None, dec) # round using reported decimals
                if dec is None or dec == "INF":  # show using decimals or reported format
                    dec = len(numVal.partition(".")[2])
                else: # max decimals at 28
                    dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
                text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num))
            elif dateVal is not None:
                text = dateVal
            elif boolVal is not None:
                text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false'
            else:
                if isQName: # declare namespace
                    addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces))
                text = textVal
            modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
            
        # add footnotes if any
        
        # save to file
        modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile)
        modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
        return modelXbrl.modelDocument
Example #36
0
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, *args, **kwargs):
    # check if big instance and has header with an initial incomplete tree walk (just 2 elements
    if not _streamingExtensionsCheck:
        return None
    
    # track whether modelXbrl has been validated by this streaming extension
    modelXbrl._streamingExtensionValidated = False
        
    def logSyntaxErrors(parsercontext):
        for error in parsercontext.error_log:
            modelXbrl.error("xmlSchema:syntax",
                    _("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"),
                    modelObject=modelXbrl, fileName=os.path.basename(filepath), 
                    error=error.message, line=error.line, column=error.column, sourceAction="streaming")
    #### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
    #### note: iterparse wants a binary file, but file is text mode
    _file, = modelXbrl.fileSource.file(filepath, binary=True)
    startedAt = time.time()
    modelXbrl.profileActivity()
    ''' this seems twice as slow as iterparse
    class instInfoTarget():
        def __init__(self, element_factory=None, parser=None):
            self.newTree = True
            self.streamingAspects = None
            self.foundInstance = False
            self.creationSoftwareComment = ''
            self.currentEltTag = "(before xbrli:xbrl)"
            self.numRootFacts = 0
        def start(self, tag, attrib, nsmap=None):
            if self.newTree:
                if tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    self.foundInstance = True
                    self.newTree = False
                else: # break 
                    raise NotInstanceDocumentException()
            elif not tag.startswith("{http://www.xbrl.org/"):
                self.numRootFacts += 1
                if self.numRootFacts % 1000 == 0:
                    modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
            self.currentEltTag = tag
        def end(self, tag):
            pass
        def data(self, data):
            pass
        def comment(self, text):
            if not self.foundInstance: # accumulate comments before xbrli:xbrl
                self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text
            elif not self.creationSoftwareComment:
                self.creationSoftwareComment = text # or first comment after xbrli:xbrl
        def pi(self, target, data):
            if target == "xbrl-streamable-instance":
                if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl":
                    self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results
                else:
                    modelXbrl.error("streamingExtensions:headerMisplaced",
                            _("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"),
                            modelObject=modelXbrl, target=target, element=self.currentEltTag)
        def close(self):
            if not self.creationSoftwareComment:
                self.creationSoftwareComment = None
            return True
    instInfo = instInfoTarget()
    infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo)
    try:
        etree.parse(_file, parser=infoParser, base_url=filepath)
    except NotInstanceDocumentException:
        pass
    '''
    foundErrors = False
    foundInstance = False
    streamingAspects = None
    creationSoftwareComment = None
    instInfoNumRootFacts = 0
    numElts = 0
    elt = None
    instInfoContext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
    try:
        for event, elt in instInfoContext:
            if event == "start":
                if elt.getparent() is not None:
                    if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
                        if not foundInstance:
                            foundInstance = True
                            pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance")
                            if pi is None:
                                break
                            else:
                                streamingAspects = dict(pi.attrib.copy())
                                if creationSoftwareComment is None:
                                    creationSoftwareComment = precedingComment(elt)
                        if not elt.tag.startswith("{http://www.xbrl.org/"):
                            instInfoNumRootFacts += 1
                            if instInfoNumRootFacts % 1000 == 0:
                                modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
                    elif not foundInstance:       
                        break
                elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    creationSoftwareComment = precedingComment(elt)
                    if precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None:
                        modelXbrl.error("streamingExtensions:headerMisplaced",
                                _("Header is misplaced: %(error)s, must follow xbrli:xbrl element"),
                                modelObject=elt)
            elif event == "end":
                elt.clear()
                numElts += 1
                if numElts % 1000 == 0 and elt.getparent() is not None:
                    while elt.getprevious() is not None and elt.getparent() is not None:
                        del elt.getparent()[0]
    except etree.XMLSyntaxError as err:
        modelXbrl.error("xmlSchema:syntax",
                _("Unrecoverable error: %(error)s"),
                error=err)
        _file.close()
        return err
        
    _file.seek(0,io.SEEK_SET) # allow reparsing
    if not foundInstance or streamingAspects is None:
        del elt
        _file.close()
        return None
    modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
    startedAt = time.time()
    try:
        version = Decimal(streamingAspects.get("version"))
        if int(version) != 1:
            modelXbrl.error("streamingExtensions:unsupportedVersion",
                    _("Streaming version %(version)s, major version number must be 1"),
                    modelObject=elt, version=version)
            foundErrors = True
    except (InvalidOperation, OverflowError):
        modelXbrl.error("streamingExtensions:versionError",
                _("Version %(version)s, number must be 1.n"),
                modelObject=elt, version=streamingAspects.get("version", "(none)"))
        foundErrors = True
    for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
        try:
            bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
            if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
                raise InvalidOperation
            elif bufAspect == "contextBuffer":
                contextBufferLimit = bufLimit
            elif bufAspect == "unitBuffer":
                unitBufferLimit = bufLimit
            elif bufAspect == "footnoteBuffer":
                footnoteBufferLimit = bufLimit
        except InvalidOperation:
            modelXbrl.error("streamingExtensions:valueError",
                    _("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"),
                    modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect))
            foundErrors = True
    if _streamingExtensionsValidate:
        incompatibleValidations = []
        _validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
        _disclosureSystem = modelXbrl.modelManager.disclosureSystem
        if _validateDisclosureSystem and _disclosureSystem.validationType == "EFM":
            incompatibleValidations.append("EFM")
        if _validateDisclosureSystem and _disclosureSystem.validationType == "GFM":
            incompatibleValidations.append("GFM")
        if _validateDisclosureSystem and _disclosureSystem.validationType == "HMRC":
            incompatibleValidations.append("HMRC")
        if modelXbrl.modelManager.validateCalcLB:
            incompatibleValidations.append("calculation LB")
        if incompatibleValidations:
            modelXbrl.error("streamingExtensions:incompatibleValidation",
                    _("Streaming instance validation does not support %(incompatibleValidations)s validation"),
                    modelObject=modelXbrl, incompatibleValidations=', '.join(incompatibleValidations))
            foundErrors = True
    if instInfoContext.error_log:
        foundErrors = True
    logSyntaxErrors(instInfoContext)
    del instInfoContext # dereference

    for pluginMethod in pluginClassMethods("Streaming.BlockStreaming"):
        _blockingPluginName = pluginMethod(modelXbrl)
        if _blockingPluginName: # name of blocking plugin is returned
            modelXbrl.error("streamingExtensions:incompatiblePlugIn",
                    _("Streaming instance not supported by plugin %(blockingPlugin)s"),
                    modelObject=modelXbrl, blockingPlugin=_blockingPluginName)
            foundErrors = True
    
    if foundErrors:
        _file.close()
        return None

    _encoding = XmlUtil.encoding(_file.read(512))
    _file.seek(0,io.SEEK_SET) # allow reparsing

    if _streamingExtensionsValidate:
        validator = Validate(modelXbrl)
        instValidator = validator.instValidator

    contextBuffer = []
    contextsToDrop = []
    unitBuffer = []
    unitsToDrop = []
    footnoteBuffer = []
    footnoteLinksToDrop = []
    
    _streamingFactsPlugin = any(True for pluginMethod in pluginClassMethods("Streaming.Facts"))
    _streamingValidateFactsPlugin = (_streamingExtensionsValidate and 
                                     any(True for pluginMethod in pluginClassMethods("Streaming.ValidateFacts")))

    ''' this is very much slower than iterparse
    class modelLoaderTarget():
        def __init__(self, element_factory=None, parser=None):
            self.newTree = True
            self.currentMdlObj = None
            self.beforeInstanceStream = True
            self.beforeStartStreamingPlugin = True
            self.numRootFacts = 1
            modelXbrl.makeelementParentModelObject = None
            modelXbrl.isStreamingMode = True
            self.factsCheckVersion = None
            self.factsCheckMd5s = Md5Sum()
        def start(self, tag, attrib, nsmap=None):
            modelXbrl.makeelementParentModelObject = self.currentMdlObj # pass parent to makeelement for ModelObjectFactory
            mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
            mdlObj.sourceline = 1
            if self.newTree:
                self.newTree = False
                self.currentMdlObj = mdlObj
                modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
                modelXbrl.modelDocument = modelDocument # needed for incremental validation
                mdlObj.init(modelDocument)
                modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement 
                modelDocument.parserLookupName = _parserLookupName
                modelDocument.parserLookupClass = _parserLookupClass
                modelDocument.xmlRootElement = mdlObj
                modelDocument.schemaLocationElements.add(mdlObj)
                modelDocument.documentEncoding = _encoding
                modelDocument._creationSoftwareComment = creationSoftwareComment
                modelXbrl.info("streamingExtensions:streaming",
                               _("Stream processing this instance."),
                               modelObject = modelDocument)
            else:
                self.currentMdlObj.append(mdlObj)
                self.currentMdlObj = mdlObj
                mdlObj._init()
                ns = mdlObj.namespaceURI
                ln = mdlObj.localName
                if (self.beforeInstanceStream and (
                    (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
                    (ns == XbrlConst.xbrli and ln in ("context", "unit")) or
                    (ns not in (XbrlConst.link, XbrlConst.xbrli)))):
                    self.beforeInstanceStream = False
                    if _streamingExtensionsValidate:
                        instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters(modelXbrl.prefixedNamespaces))
                    else: # need default dimensions
                        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
                elif not self.beforeInstanceStream and self.beforeStartStreamingPlugin:
                    for pluginMethod in pluginClassMethods("Streaming.Start"):
                        pluginMethod(modelXbrl)
                    self.beforeStartStreamingPlugin = False
            return mdlObj
        def end(self, tag):
            modelDocument = modelXbrl.modelDocument
            mdlObj = self.currentMdlObj
            parentMdlObj = mdlObj.getparent()
            self.currentMdlObj = parentMdlObj
            ns = mdlObj.namespaceURI
            ln = mdlObj.localName
            if ns == XbrlConst.xbrli:
                if ln == "context":
                    if mdlObj.get("sticky"):
                        del mdlObj.attrib["sticky"]
                        XmlValidate.validate(modelXbrl, mdlObj)
                        modelDocument.contextDiscover(mdlObj)
                    else:
                        if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
                            # drop before adding as dropped may have same id as added
                            cntx = contextBuffer.pop(0)
                            if _streamingValidateFactsPlugin:
                                contextsToDrop.append(cntx)
                            else:
                                dropContext(modelXbrl, cntx)
                                del parentMdlObj[parentMdlObj.index(cntx)]
                            cntx = None
                        #>>XmlValidate.validate(modelXbrl, mdlObj)
                        #>>modelDocument.contextDiscover(mdlObj)
                        if contextBufferLimit.is_finite():
                            contextBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        contextsToCheck = (mdlObj,)
                        instValidator.checkContexts(contextsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkContextsDimensions(contextsToCheck)
                        del contextsToCheck # dereference
                elif ln == "unit":
                    if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
                        # drop before adding as dropped may have same id as added
                        unit = unitBuffer.pop(0)
                        if _streamingValidateFactsPlugin:
                            unitsToDrop.append(unit)
                        else:
                            dropUnit(modelXbrl, unit)
                            del parentMdlObj[parentMdlObj.index(unit)]
                        unit = None 
                    #>>XmlValidate.validate(modelXbrl, mdlObj)
                    #>>modelDocument.unitDiscover(mdlObj)
                    if unitBufferLimit.is_finite():
                        unitBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkUnits( (mdlObj,) )
                elif ln == "xbrl": # end of document
                    # check remaining batched facts if any
                    if _streamingValidateFactsPlugin:
                        # plugin attempts to process batch of all root facts not yet processed (not just current one)
                        # finish any final batch of facts
                        if len(modelXbrl.facts) > 0:
                            factsToCheck = modelXbrl.facts.copy()
                            factsHaveBeenProcessed = True
                            # can block facts deletion if required data not yet available, such as numeric unit for DpmDB
                            for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
                                if not pluginMethod(modelXbrl, factsToCheck):
                                    factsHaveBeenProcessed = False
                            if factsHaveBeenProcessed:
                                for fact in factsToCheck:
                                    dropFact(modelXbrl, fact, modelXbrl.facts)
                                    del parentMdlObj[parentMdlObj.index(fact)]
                                for cntx in contextsToDrop:
                                    dropContext(modelXbrl, cntx)
                                    del parentMdlObj[parentMdlObj.index(cntx)]
                                for unit in unitsToDrop:
                                    dropUnit(modelXbrl, unit)
                                    del parentMdlObj[parentMdlObj.index(unit)]
                                for footnoteLink in footnoteLinksToDrop:
                                    dropFootnoteLink(modelXbrl, footnoteLink)
                                    del parentMdlObj[parentMdlObj.index(footnoteLink)]
                                fact = cntx = unit = footnoteLink = None
                                del contextsToDrop[:]
                                del unitsToDrop[:]
                                del footnoteLinksToDrop[:]
                            del factsToCheck
                    # check remaining footnote refs
                    for footnoteLink in footnoteBuffer:
                        checkFootnoteHrefs(modelXbrl, footnoteLink)
                    for pluginMethod in pluginClassMethods("Streaming.Finish"):
                        pluginMethod(modelXbrl)
            elif ns == XbrlConst.link:
                if ln == "footnoteLink":
                    XmlValidate.validate(modelXbrl, mdlObj)
                    footnoteLinks = (mdlObj,)
                    modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
                    if footnoteBufferLimit.is_finite():
                        footnoteBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkLinks(footnoteLinks)
                        if len(footnoteBuffer) > footnoteBufferLimit:
                            # check that hrefObjects for locators were all satisfied
                                # drop before addition as dropped may have same id as added
                            footnoteLink = footnoteBuffer.pop(0)
                            checkFootnoteHrefs(modelXbrl, footnoteLink)
                            if _streamingValidateFactsPlugin:
                                footnoteLinksToDrop.append(footnoteLink)
                            else:
                                dropFootnoteLink(modelXbrl, footnoteLink)
                                del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            footnoteLink = None
                    footnoteLinks = None
                elif ln in ("schemaRef", "linkbaseRef"):
                    modelDocument.discoverHref(mdlObj)
                elif not modelXbrl.skipDTS:
                    if ln in ("roleRef", "arcroleRef"):
                        modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
            elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
                self.numRootFacts += 1
                #>>XmlValidate.validate(modelXbrl, mdlObj)
                #>>modelDocument.factDiscover(mdlObj, modelXbrl.facts)
                if self.factsCheckVersion:
                    self.factCheckFact(mdlObj)
                if _streamingExtensionsValidate or _streamingValidateFactsPlugin:
                    factsToCheck = (mdlObj,)  # validate current fact by itself
                    if _streamingExtensionsValidate:
                        instValidator.checkFacts(factsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkFactsDimensions(factsToCheck)
                    if _streamingValidateFactsPlugin:
                        # plugin attempts to process batch of all root facts not yet processed (not just current one)
                        # use batches of 1000 facts
                        if len(modelXbrl.facts) > 1000:
                            factsToCheck = modelXbrl.facts.copy()
                            factsHaveBeenProcessed = True
                            # can block facts deletion if required data not yet available, such as numeric unit for DpmDB
                            for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
                                if not pluginMethod(modelXbrl, factsToCheck):
                                    factsHaveBeenProcessed = False
                            if factsHaveBeenProcessed:
                                for fact in factsToCheck:
                                    dropFact(modelXbrl, fact, modelXbrl.facts)
                                    del parentMdlObj[parentMdlObj.index(fact)]
                                for cntx in contextsToDrop:
                                    dropContext(modelXbrl, cntx)
                                    del parentMdlObj[parentMdlObj.index(cntx)]
                                for unit in unitsToDrop:
                                    dropUnit(modelXbrl, unit)
                                    del parentMdlObj[parentMdlObj.index(unit)]
                                for footnoteLink in footnoteLinksToDrop:
                                    dropFootnoteLink(modelXbrl, footnoteLink)
                                    del parentMdlObj[parentMdlObj.index(footnoteLink)]
                                fact = cntx = unit = footnoteLink = None
                                del contextsToDrop[:]
                                del unitsToDrop[:]
                                del footnoteLinksToDrop[:]
                            del factsToCheck # dereference fact or batch of facts
                    else:
                        dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed
                        del parentMdlObj[parentMdlObj.index(mdlObj)]
                if self.numRootFacts % 1000 == 0:
                    pass
                    #modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, 
                    #                                                                          100.0 * self.numRootFacts / instInfoNumRootFacts), 
                    #                          minTimeToShow=20.0)
                    gc.collect()
                    sys.stdout.write ("\rAt fact {} of {} mem {}".format(self.numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed))
            return mdlObj
        def data(self, data):
            self.currentMdlObj.text = data
        def comment(self, text):
            pass
        def pi(self, target, data):
            if target == "xbrl-facts-check":
                _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", data)
                if _match:
                    _matchGroups = _match.groups()
                    if len(_matchGroups) == 2:
                        if _matchGroups[0] == "version":
                            self.factsCheckVersion = _matchGroups[1]
                        elif _matchGroups[0] == "sum-of-fact-md5s":
                            try:
                                expectedMd5 = Md5Sum(_matchGroups[1])
                                if self.factsCheckMd5s != expectedMd5:
                                    modelXbrl.warning("streamingExtensions:xbrlFactsCheckWarning",
                                            _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"),
                                            modelObject=modelXbrl, expectedMd5=expectedMd5, actualMd5Sum=self.factsCheckMd5s)
                                else:
                                    modelXbrl.info("info",
                                            _("Successful XBRL facts sum of md5s."),
                                            modelObject=modelXbrl)
                            except ValueError:
                                modelXbrl.error("streamingExtensions:xbrlFactsCheckError",
                                        _("Invalid sum-of-md5s %(sumOfMd5)s"),
                                        modelObject=modelXbrl, sumOfMd5=_matchGroups[1])
        def close(self):
            del modelXbrl.makeelementParentModelObject
            return None
        
        def factCheckFact(self, fact):
            self.factsCheckMd5s += fact.md5sum
            for _tupleFact in fact.modelTupleFacts:
                self.factCheckFact(_tupleFact)
        
    _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget())
    etree.parse(_file, parser=_parser, base_url=filepath)
    logSyntaxErrors(_parser)
    '''
    # replace modelLoaderTarget with iterparse (as it now supports CustomElementClassLookup)
    streamingParserContext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
    from arelle.ModelObjectFactory import setParserElementClassLookup
    modelXbrl.isStreamingMode = True # must be set before setting element class lookup
    (_parser, _parserLookupName, _parserLookupClass) = setParserElementClassLookup(streamingParserContext, modelXbrl)
    foundInstance = False
    beforeInstanceStream = beforeStartStreamingPlugin = True
    numRootFacts = 0
    factsCheckVersion = None
    def factCheckFact(fact):
        modelDocument._factsCheckMd5s += fact.md5sum
        for _tupleFact in fact.modelTupleFacts:
            factCheckFact(_tupleFact)
    for event, mdlObj in streamingParserContext:
        if event == "start":
            if mdlObj.tag == "{http://www.xbrl.org/2003/instance}xbrl":
                modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
                modelXbrl.modelDocument = modelDocument # needed for incremental validation
                mdlObj.init(modelDocument)
                modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement 
                modelDocument.parserLookupName = _parserLookupName
                modelDocument.parserLookupClass = _parserLookupClass
                modelDocument.xmlRootElement = mdlObj
                modelDocument.schemaLocationElements.add(mdlObj)
                modelDocument.documentEncoding = _encoding
                modelDocument._creationSoftwareComment = precedingComment(mdlObj)
                modelDocument._factsCheckMd5s = Md5Sum()
                modelXbrl.info("streamingExtensions:streaming",
                               _("Stream processing this instance."),
                               modelObject = modelDocument)
            elif mdlObj.getparent() is not None:
                mdlObj._init() # requires discovery as part of start elements
                if mdlObj.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    if not foundInstance:
                        foundInstance = True
                        pi = precedingProcessingInstruction(mdlObj, "xbrl-facts-check")
                        if pi is not None:
                            factsCheckVersion = pi.attrib.get("version", None)
                elif not foundInstance:       
                    break
                ns = mdlObj.qname.namespaceURI
                ln = mdlObj.qname.localName
                if beforeInstanceStream:
                    if ((ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
                        (ns == XbrlConst.xbrli and ln in ("context", "unit")) or
                        (ns not in (XbrlConst.link, XbrlConst.xbrli))):
                        beforeInstanceStream = False
                        if _streamingExtensionsValidate:
                            instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters(modelXbrl.prefixedNamespaces))
                        else: # need default dimensions
                            ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
                elif not beforeInstanceStream and beforeStartStreamingPlugin:
                    for pluginMethod in pluginClassMethods("Streaming.Start"):
                        pluginMethod(modelXbrl)
                    beforeStartStreamingPlugin = False
        elif event == "end":
            parentMdlObj = mdlObj.getparent()
            ns = mdlObj.namespaceURI
            ln = mdlObj.localName
            if ns == XbrlConst.xbrli:
                if ln == "context":
                    if mdlObj.get("sticky"):
                        del mdlObj.attrib["sticky"]
                        XmlValidate.validate(modelXbrl, mdlObj)
                        modelDocument.contextDiscover(mdlObj)
                    else:
                        if len(contextBuffer) >= contextBufferLimit:
                            # drop before adding as dropped may have same id as added
                            cntx = contextBuffer.pop(0)
                            if _streamingFactsPlugin or _streamingValidateFactsPlugin:
                                contextsToDrop.append(cntx)
                            else:
                                dropContext(modelXbrl, cntx)
                                #>>del parentMdlObj[parentMdlObj.index(cntx)]
                            cntx = None
                        XmlValidate.validate(modelXbrl, mdlObj)
                        modelDocument.contextDiscover(mdlObj)
                        if contextBufferLimit.is_finite():
                            contextBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        contextsToCheck = (mdlObj,)
                        instValidator.checkContexts(contextsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkContextsDimensions(contextsToCheck)
                        del contextsToCheck # dereference
                elif ln == "unit":
                    if len(unitBuffer) >= unitBufferLimit:
                        # drop before additing as dropped may have same id as added
                        unit = unitBuffer.pop(0)
                        if _streamingFactsPlugin or _streamingValidateFactsPlugin:
                            unitsToDrop.append(unit)
                        else:
                            dropUnit(modelXbrl, unit)
                            #>>del parentMdlObj[parentMdlObj.index(unit)]
                        unit = None 
                    XmlValidate.validate(modelXbrl, mdlObj)
                    modelDocument.unitDiscover(mdlObj)
                    if unitBufferLimit.is_finite():
                        unitBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkUnits( (mdlObj,) )
                elif ln == "xbrl": # end of document
                    # check remaining batched facts if any
                    if _streamingFactsPlugin or _streamingValidateFactsPlugin:
                        # plugin attempts to process batch of all root facts not yet processed (not just current one)
                        # finish any final batch of facts
                        if len(modelXbrl.facts) > 0:
                            factsToCheck = modelXbrl.facts.copy()
                            # can block facts deletion if required data not yet available, such as numeric unit for DpmDB
                            if _streamingValidateFactsPlugin:
                                for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
                                    pluginMethod(instValidator, factsToCheck)
                            if _streamingFactsPlugin:
                                for pluginMethod in pluginClassMethods("Streaming.Facts"):
                                    pluginMethod(modelXbrl, factsToCheck)
                            for fact in factsToCheck:
                                dropFact(modelXbrl, fact, modelXbrl.facts)
                                #>>del parentMdlObj[parentMdlObj.index(fact)]
                            for cntx in contextsToDrop:
                                dropContext(modelXbrl, cntx)
                                #>>del parentMdlObj[parentMdlObj.index(cntx)]
                            for unit in unitsToDrop:
                                dropUnit(modelXbrl, unit)
                                #>>del parentMdlObj[parentMdlObj.index(unit)]
                            for footnoteLink in footnoteLinksToDrop:
                                dropFootnoteLink(modelXbrl, footnoteLink)
                                #>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            fact = cntx = unit = footnoteLink = None
                            del contextsToDrop[:]
                            del unitsToDrop[:]
                            del footnoteLinksToDrop[:]
                            del factsToCheck
                    # check remaining footnote refs
                    for footnoteLink in footnoteBuffer:
                        checkFootnoteHrefs(modelXbrl, footnoteLink)
                    pi = childProcessingInstruction(mdlObj, "xbrl-facts-check", reversed=True)
                    if pi is not None: # attrib is in .text, not attrib, no idea why!!!
                        _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text)
                        if _match:
                            _matchGroups = _match.groups()
                            if len(_matchGroups) == 2:
                                if _matchGroups[0] == "sum-of-fact-md5s":
                                    try:
                                        expectedMd5 = Md5Sum(_matchGroups[1])
                                        if modelDocument._factsCheckMd5s != expectedMd5:
                                            modelXbrl.warning("streamingExtensions:xbrlFactsCheckWarning",
                                                    _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"),
                                                    modelObject=modelXbrl, expectedMd5=expectedMd5, actualMd5Sum=modelDocument._factsCheckMd5s)
                                        else:
                                            modelXbrl.info("info",
                                                    _("Successful XBRL facts sum of md5s."),
                                                    modelObject=modelXbrl)
                                    except ValueError:
                                        modelXbrl.error("streamingExtensions:xbrlFactsCheckError",
                                                _("Invalid sum-of-md5s %(sumOfMd5)s"),
                                                modelObject=modelXbrl, sumOfMd5=_matchGroups[1])
                    if _streamingValidateFactsPlugin:
                        for pluginMethod in pluginClassMethods("Streaming.ValidateFinish"):
                            pluginMethod(instValidator)
                    if _streamingFactsPlugin:
                        for pluginMethod in pluginClassMethods("Streaming.Finish"):
                            pluginMethod(modelXbrl)
            elif ns == XbrlConst.link:
                if ln in ("schemaRef", "linkbaseRef"):
                    modelDocument.discoverHref(mdlObj, urlRewritePluginClass="ModelDocument.InstanceSchemaRefRewriter")
                elif ln in ("roleRef", "arcroleRef"):
                    modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
                elif ln == "footnoteLink":
                    XmlValidate.validate(modelXbrl, mdlObj)
                    footnoteLinks = (mdlObj,)
                    modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
                    if footnoteBufferLimit.is_finite():
                        footnoteBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkLinks(footnoteLinks)
                        if len(footnoteBuffer) > footnoteBufferLimit:
                            # check that hrefObjects for locators were all satisfied
                                # drop before addition as dropped may have same id as added
                            footnoteLink = footnoteBuffer.pop(0)
                            checkFootnoteHrefs(modelXbrl, footnoteLink)
                            if _streamingValidateFactsPlugin:
                                footnoteLinksToDrop.append(footnoteLink)
                            else:
                                dropFootnoteLink(modelXbrl, footnoteLink)
                                #>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            footnoteLink = None
                    footnoteLinks = None
            elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl and isinstance(mdlObj, ModelFact):
                numRootFacts += 1
                XmlValidate.validate(modelXbrl, mdlObj)
                modelDocument.factDiscover(mdlObj, modelXbrl.facts)
                if factsCheckVersion:
                    factCheckFact(mdlObj)
                if _streamingExtensionsValidate or _streamingFactsPlugin or _streamingValidateFactsPlugin:
                    factsToCheck = (mdlObj,)  # validate current fact by itself
                    if _streamingExtensionsValidate:
                        instValidator.checkFacts(factsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkFactsDimensions(factsToCheck)
                    if _streamingFactsPlugin or _streamingValidateFactsPlugin:
                        # plugin attempts to process batch of all root facts not yet processed (not just current one)
                        # use batches of 1000 facts
                        if len(modelXbrl.facts) > 1000:
                            factsToCheck = modelXbrl.facts.copy()
                            # can block facts deletion if required data not yet available, such as numeric unit for DpmDB
                            if _streamingValidateFactsPlugin:
                                for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
                                    pluginMethod(instValidator, factsToCheck)
                            if _streamingFactsPlugin:
                                for pluginMethod in pluginClassMethods("Streaming.Facts"):
                                    pluginMethod(modelXbrl, factsToCheck)
                            for fact in factsToCheck:
                                dropFact(modelXbrl, fact, modelXbrl.facts)
                                #>>del parentMdlObj[parentMdlObj.index(fact)]
                            for cntx in contextsToDrop:
                                dropContext(modelXbrl, cntx)
                                #>>del parentMdlObj[parentMdlObj.index(cntx)]
                            for unit in unitsToDrop:
                                dropUnit(modelXbrl, unit)
                                #>>del parentMdlObj[parentMdlObj.index(unit)]
                            for footnoteLink in footnoteLinksToDrop:
                                dropFootnoteLink(modelXbrl, footnoteLink)
                                #>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            fact = cntx = unit = footnoteLink = None
                            del contextsToDrop[:]
                            del unitsToDrop[:]
                            del footnoteLinksToDrop[:]
                            del factsToCheck # dereference fact or batch of facts
                    else:
                        dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed
                        #>>del parentMdlObj[parentMdlObj.index(mdlObj)]
                if numRootFacts % 1000 == 0:
                    pass
                    #modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, 
                    #                                                                          100.0 * self.numRootFacts / instInfoNumRootFacts), 
                    #                          minTimeToShow=20.0)
                    #gc.collect()
                    #sys.stdout.write ("\rAt fact {} of {} mem {}".format(numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed))
    if mdlObj is not None:
        mdlObj.clear()
    del _parser, _parserLookupName, _parserLookupClass                
                
    if _streamingExtensionsValidate and validator is not None:
        _file.close()
        del instValidator
        validator.close()
        # track that modelXbrl has been validated by this streaming extension
        modelXbrl._streamingExtensionValidated = True
        
    modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
    return modelXbrl.modelDocument
Example #37
0
def createTargetInstance(modelXbrl, targetUrl, targetDocumentSchemaRefs, filingFiles, baseXmlLang=None, defaultXmlLang=None):
    targetInstance = ModelXbrl.create(modelXbrl.modelManager,
                                      newDocumentType=Type.INSTANCE,
                                      url=targetUrl,
                                      schemaRefs=targetDocumentSchemaRefs,
                                      isEntry=True,
                                      discover=False) # don't attempt to load DTS
    if baseXmlLang:
        targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
        if defaultXmlLang is None:
            defaultXmlLang = baseXmlLang # allows facts/footnotes to override baseXmlLang
    ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults 
    # roleRef and arcroleRef (of each inline document)
    for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
        for roleRefElt in sourceRefs.values():
            addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
                     attributes=roleRefElt.items())
    
    # contexts
    for context in sorted(modelXbrl.contexts.values(), key=lambda c: c.objectIndex): # contexts may come from multiple IXDS files
        ignore = targetInstance.createContext(context.entityIdentifier[0],
                                               context.entityIdentifier[1],
                                               'instant' if context.isInstantPeriod else
                                               'duration' if context.isStartEndPeriod
                                               else 'forever',
                                               context.startDatetime,
                                               context.endDatetime,
                                               None,
                                               context.qnameDims, [], [],
                                               id=context.id)
    for unit in sorted(modelXbrl.units.values(), key=lambda u: u.objectIndex): # units may come from multiple IXDS files
        measures = unit.measures
        ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id)

    modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
    newFactForOldObjId = {}
    def createFacts(facts, parent):
        for fact in facts:
            if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior
                attrs = {"contextRef": fact.contextID}
                if fact.id:
                    attrs["id"] = fact.id
                if fact.isNumeric:
                    attrs["unitRef"] = fact.unitID
                    if fact.get("decimals"):
                        attrs["decimals"] = fact.get("decimals")
                    if fact.get("precision"):
                        attrs["precision"] = fact.get("precision")
                if fact.isNil:
                    attrs[XbrlConst.qnXsiNil] = "true"
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.textValue
                    if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default
                        xmlLang = fact.xmlLang
                        if xmlLang is not None and xmlLang != defaultXmlLang:
                            attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang
                newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
                # if fact.isFraction, create numerator and denominator
                newFactForOldObjId[fact.objectIndex] = newFact
                if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
                    # check for img and other filing references so that referenced files are included in the zip.
                    for xmltext in [text] + CDATApattern.findall(text):
                        try:
                            for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter():
                                addLocallyReferencedFile(elt, filingFiles)
                        except (XMLSyntaxError, UnicodeDecodeError):
                            pass  # TODO: Why ignore UnicodeDecodeError?
            elif fact.isTuple:
                newTuple = targetInstance.createFact(fact.qname, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newTuple
                createFacts(fact.modelTupleFacts, newTuple)
                
    createFacts(modelXbrl.facts, None)
    modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships"))
    HREF = "{http://www.w3.org/1999/xlink}href"
    footnoteLinks = defaultdict(list)
    footnoteIdCount = {}
    for linkKey, linkPrototypes in modelXbrl.baseSets.items():
        arcrole, linkrole, linkqname, arcqname = linkKey
        if (linkrole and linkqname and arcqname and  # fully specified roles
            arcrole != "XBRL-footnotes" and
            any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
            for linkPrototype in linkPrototypes:
                if linkPrototype not in footnoteLinks[linkrole]:
                    footnoteLinks[linkrole].append(linkPrototype)
    for linkrole in sorted(footnoteLinks.keys()):
        for linkPrototype in footnoteLinks[linkrole]:
            newLink = addChild(targetInstance.modelDocument.xmlRootElement, 
                               linkPrototype.qname, 
                               attributes=linkPrototype.attributes)
            for linkChild in linkPrototype:
                attributes = linkChild.attributes
                if isinstance(linkChild, LocPrototype):
                    if HREF not in linkChild.attributes:
                        linkChild.attributes[HREF] = \
                        "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
                    addChild(newLink, linkChild.qname, 
                             attributes=attributes)
                elif isinstance(linkChild, ArcPrototype):
                    addChild(newLink, linkChild.qname, attributes=attributes)
                elif isinstance(linkChild, ModelInlineFootnote):
                    idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
                    if idUseCount > 1: # if footnote with id in other links bump the id number
                        attributes = linkChild.attributes.copy()
                        attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
                    footnoteIdCount[linkChild.footnoteID] = idUseCount
                    newChild = addChild(newLink, linkChild.qname, 
                                        attributes=attributes)
                    xmlLang = linkChild.xmlLang
                    if xmlLang is not None and xmlLang != defaultXmlLang: # default
                        newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang)
                    copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True)

                    if filingFiles and linkChild.textValue:
                        footnoteHtml = XML("<body/>")
                        copyIxFootnoteHtml(linkChild, footnoteHtml)
                        for elt in footnoteHtml.iter():
                            addLocallyReferencedFile(elt,filingFiles)
    return targetInstance
Example #38
0
    def validate(self, modelXbrl, parameters=None):
        self.parameters = parameters
        self.precisionPattern = re.compile("^([0-9]+|INF)$")
        self.decimalsPattern = re.compile("^(-?[0-9]+|INF)$")
        self.isoCurrencyPattern = re.compile(r"^[A-Z]{3}$")
        self.modelXbrl = modelXbrl
        self.validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
        self.disclosureSystem = modelXbrl.modelManager.disclosureSystem
        self.validateEFM = self.validateDisclosureSystem and self.disclosureSystem.EFM
        self.validateGFM = self.validateDisclosureSystem and self.disclosureSystem.GFM
        self.validateEFMorGFM = self.validateDisclosureSystem and self.disclosureSystem.EFMorGFM
        self.validateHMRC = self.validateDisclosureSystem and self.disclosureSystem.HMRC
        self.validateSBRNL = self.validateDisclosureSystem and self.disclosureSystem.SBRNL
        self.validateXmlLang = self.validateDisclosureSystem and self.disclosureSystem.xmlLangPattern
        self.validateCalcLB = modelXbrl.modelManager.validateCalcLB
        self.validateInferDecimals = modelXbrl.modelManager.validateInferDecimals
        
        # xlink validation
        modelXbrl.profileStat(None)
        modelXbrl.modelManager.showStatus(_("validating links"))
        modelLinks = set()
        self.remoteResourceLocElements = set()
        self.genericArcArcroles = set()
        for baseSetExtLinks in modelXbrl.baseSets.values():
            for baseSetExtLink in baseSetExtLinks:
                modelLinks.add(baseSetExtLink)    # ext links are unique (no dups)
        for modelLink in modelLinks:
            fromToArcs = {}
            locLabels = {}
            resourceLabels = {}
            resourceArcTos = []
            for arcElt in modelLink.iterchildren():
                if isinstance(arcElt,ModelObject):
                    xlinkType = arcElt.get("{http://www.w3.org/1999/xlink}type")
                    # locator must have an href
                    if xlinkType == "locator":
                        if arcElt.get("{http://www.w3.org/1999/xlink}href") is None:
                            modelXbrl.error("xlink:locatorHref",
                                _("Xlink locator %(xlinkLabel)s missing href in extended link %(linkrole)s"),
                                modelObject=arcElt,
                                linkrole=modelLink.role, 
                                xlinkLabel=arcElt.get("{http://www.w3.org/1999/xlink}label")) 
                        locLabels[arcElt.get("{http://www.w3.org/1999/xlink}label")] = arcElt
                    elif xlinkType == "resource":
                        resourceLabels[arcElt.get("{http://www.w3.org/1999/xlink}label")] = arcElt
                    # can be no duplicated arcs between same from and to
                    elif xlinkType == "arc":
                        fromLabel = arcElt.get("{http://www.w3.org/1999/xlink}from")
                        toLabel = arcElt.get("{http://www.w3.org/1999/xlink}to")
                        fromTo = (fromLabel,toLabel)
                        if fromTo in fromToArcs:
                            modelXbrl.error("xlink:dupArcs",
                                _("Duplicate xlink arcs  in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s"),
                                modelObject=arcElt,
                                linkrole=modelLink.role, 
                                xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel)
                        else:
                            fromToArcs[fromTo] = arcElt
                        if arcElt.namespaceURI == XbrlConst.link:
                            if arcElt.localName in arcNamesTo21Resource: #("labelArc","referenceArc"):
                                resourceArcTos.append((toLabel, arcElt.get("use"), arcElt))
                        elif self.isGenericArc(arcElt):
                            arcrole = arcElt.get("{http://www.w3.org/1999/xlink}arcrole")
                            self.genericArcArcroles.add(arcrole)
                            if arcrole in (XbrlConst.elementLabel, XbrlConst.elementReference):
                                resourceArcTos.append((toLabel, arcrole, arcElt))
                    # values of type (not needed for validating parsers)
                    if xlinkType not in xlinkTypeValues: # ("", "simple", "extended", "locator", "arc", "resource", "title", "none"):
                        modelXbrl.error("xlink:type",
                            _("Xlink type %(xlinkType)s invalid in extended link %(linkrole)s"),
                            modelObject=arcElt, linkrole=modelLink.role, xlinkType=xlinkType)
                    # values of actuate (not needed for validating parsers)
                    xlinkActuate = arcElt.get("{http://www.w3.org/1999/xlink}actuate")
                    if xlinkActuate not in xlinkActuateValues: # ("", "onLoad", "onRequest", "other", "none"):
                        modelXbrl.error("xlink:actuate",
                            _("Actuate %(xlinkActuate)s invalid in extended link %(linkrole)s"),
                            modelObject=arcElt, linkrole=modelLink.role, xlinkActuate=xlinkActuate)
                    # values of show (not needed for validating parsers)
                    xlinkShow = arcElt.get("{http://www.w3.org/1999/xlink}show")
                    if xlinkShow not in xlinkShowValues: # ("", "new", "replace", "embed", "other", "none"):
                        modelXbrl.error("xlink:show",
                            _("Show %(xlinkShow)s invalid in extended link %(linkrole)s"),
                            modelObject=arcElt, linkrole=modelLink.role, xlinkShow=xlinkShow)
            # check from, to of arcs have a resource or loc
            for fromTo, arcElt in fromToArcs.items():
                fromLabel, toLabel = fromTo
                for name, value, sect in (("from", fromLabel, "3.5.3.9.2"),("to",toLabel, "3.5.3.9.3")):
                    if value not in locLabels and value not in resourceLabels:
                        modelXbrl.error("xbrl.{0}:arcResource".format(sect),
                            _("Arc in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s attribute '%(attribute)s' has no matching loc or resource label"),
                            modelObject=arcElt, 
                            linkrole=modelLink.role, xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel, 
                            attribute=name)
                if arcElt.localName == "footnoteArc" and arcElt.namespaceURI == XbrlConst.link and \
                   arcElt.get("{http://www.w3.org/1999/xlink}arcrole") == XbrlConst.factFootnote:
                    if fromLabel not in locLabels:
                        modelXbrl.error("xbrl.4.11.1.3.1:factFootnoteArcFrom",
                            _("Footnote arc in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s \"from\" is not a loc"),
                            modelObject=arcElt, 
                            linkrole=modelLink.role, xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel)
                    if toLabel not in resourceLabels or qname(resourceLabels[toLabel]) != XbrlConst.qnLinkFootnote:
                        modelXbrl.error("xbrl.4.11.1.3.1:factFootnoteArcTo",
                            _("Footnote arc in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s \"to\" is not a footnote resource"),
                            modelObject=arcElt, 
                            linkrole=modelLink.role, xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel)
            # check unprohibited label arcs to remote locs
            for resourceArcTo in resourceArcTos:
                resourceArcToLabel, resourceArcUse, arcElt = resourceArcTo
                if resourceArcToLabel in locLabels:
                    toLabel = locLabels[resourceArcToLabel]
                    if resourceArcUse == "prohibited":
                        self.remoteResourceLocElements.add(toLabel)
                    else:
                        modelXbrl.error("xbrl.5.2.2.3:labelArcRemoteResource",
                            _("Unprohibited labelArc in extended link %(linkrole)s has illegal remote resource loc labeled %(xlinkLabel)s href %(xlinkHref)s"),
                            modelObject=arcElt, 
                            linkrole=modelLink.role, 
                            xlinkLabel=resourceArcToLabel,
                            xlinkHref=toLabel.get("{http://www.w3.org/1999/xlink}href"))
                elif resourceArcToLabel in resourceLabels:
                    toResource = resourceLabels[resourceArcToLabel]
                    if resourceArcUse == XbrlConst.elementLabel:
                        if not self.isGenericLabel(toResource):
                            modelXbrl.error("xbrlle.2.1.1:genericLabelTarget",
                                _("Generic label arc in extended link %(linkrole)s to %(xlinkLabel)s must target a generic label"),
                                modelObject=arcElt, 
                                linkrole=modelLink.role, 
                                xlinkLabel=resourceArcToLabel)
                    elif resourceArcUse == XbrlConst.elementReference:
                        if not self.isGenericReference(toResource):
                            modelXbrl.error("xbrlre.2.1.1:genericReferenceTarget",
                                _("Generic reference arc in extended link %(linkrole)s to %(xlinkLabel)s must target a generic reference"),
                                modelObject=arcElt, 
                                linkrole=modelLink.role, 
                                xlinkLabel=resourceArcToLabel)
            resourceArcTos = None # dereference arcs
        modelXbrl.profileStat(_("validateLinks"))

        modelXbrl.dimensionDefaultConcepts = {}
        modelXbrl.qnameDimensionDefaults = {}
        modelXbrl.qnameDimensionContextElement = {}
        # check base set cycles, dimensions
        modelXbrl.modelManager.showStatus(_("validating relationship sets"))
        for baseSetKey in modelXbrl.baseSets.keys():
            arcrole, ELR, linkqname, arcqname = baseSetKey
            if arcrole.startswith("XBRL-") or ELR is None or \
                linkqname is None or arcqname is None:
                continue
            elif arcrole in XbrlConst.standardArcroleCyclesAllowed:
                # TODO: table should be in this module, where it is used
                cyclesAllowed, specSect = XbrlConst.standardArcroleCyclesAllowed[arcrole]
            elif arcrole in self.modelXbrl.arcroleTypes and len(self.modelXbrl.arcroleTypes[arcrole]) > 0:
                cyclesAllowed = self.modelXbrl.arcroleTypes[arcrole][0].cyclesAllowed
                if arcrole in self.genericArcArcroles:
                    specSect = "xbrlgene:violatedCyclesConstraint"
                else:
                    specSect = "xbrl.5.1.4.3:cycles"
            else:
                cyclesAllowed = "any"
                specSect = None
            if cyclesAllowed != "any" or arcrole in (XbrlConst.summationItem,) \
                                      or arcrole in self.genericArcArcroles  \
                                      or arcrole.startswith(XbrlConst.formulaStartsWith):
                relsSet = modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname)
            if cyclesAllowed != "any" and \
                   (XbrlConst.isStandardExtLinkQname(linkqname) and XbrlConst.isStandardArcQname(arcqname)) \
                   or arcrole in self.genericArcArcroles:
                noUndirected = cyclesAllowed == "none"
                fromRelationships = relsSet.fromModelObjects()
                for relFrom, rels in fromRelationships.items():
                    cycleFound = self.fwdCycle(relsSet, rels, noUndirected, {relFrom})
                    if cycleFound is not None:
                        path = str(relFrom.qname) + " " + " - ".join(
                            "{0}:{1} {2}".format(rel.modelDocument.basename, rel.sourceline, rel.toModelObject.qname)
                            for rel in reversed(cycleFound[1:]))
                        modelXbrl.error(specSect,
                            _("Relationships have a %(cycle)s cycle in arcrole %(arcrole)s \nlink role %(linkrole)s \nlink %(linkname)s, \narc %(arcname)s, \npath %(path)s"),
                            modelObject=cycleFound[1:], cycle=cycleFound[0], path=path,
                            arcrole=arcrole, linkrole=ELR, linkname=linkqname, arcname=arcqname), 
                        break
                
            # check calculation arcs for weight issues (note calc arc is an "any" cycles)
            if arcrole == XbrlConst.summationItem:
                for modelRel in relsSet.modelRelationships:
                    weight = modelRel.weight
                    fromConcept = modelRel.fromModelObject
                    toConcept = modelRel.toModelObject
                    if fromConcept is not None and toConcept is not None:
                        if weight == 0:
                            modelXbrl.error("xbrl.5.2.5.2.1:zeroWeight",
                                _("Calculation relationship has zero weight from %(source)s to %(target)s in link role %(linkrole)s"),
                                modelObject=modelRel,
                                source=fromConcept.qname, target=toConcept.qname, linkrole=ELR), 
                        fromBalance = fromConcept.balance
                        toBalance = toConcept.balance
                        if fromBalance and toBalance:
                            if (fromBalance == toBalance and weight < 0) or \
                               (fromBalance != toBalance and weight > 0):
                                modelXbrl.error("xbrl.5.1.1.2:balanceCalcWeightIllegal" +
                                                ("Negative" if weight < 0 else "Positive"),
                                    _("Calculation relationship has illegal weight %(weight)s from %(source)s, %(sourceBalance)s, to %(target)s, %(targetBalance)s, in link role %(linkrole)s (per 5.1.1.2 Table 6)"),
                                    modelObject=modelRel, weight=weight,
                                    source=fromConcept.qname, target=toConcept.qname, linkrole=ELR, 
                                    sourceBalance=fromBalance, targetBalance=toBalance)
                        if not fromConcept.isNumeric or not toConcept.isNumeric:
                            modelXbrl.error("xbrl.5.2.5.2:nonNumericCalc",
                                _("Calculation relationship has illegal concept from %(source)s%(sourceNumericDecorator)s to %(target)s%(targetNumericDecorator)s in link role %(linkrole)s"),
                                modelObject=modelRel,
                                source=fromConcept.qname, target=toConcept.qname, linkrole=ELR, 
                                sourceNumericDecorator="" if fromConcept.isNumeric else _(" (non-numeric)"), 
                                targetNumericDecorator="" if toConcept.isNumeric else _(" (non-numeric)"))
            # check presentation relationships for preferredLabel issues
            elif arcrole == XbrlConst.parentChild:
                for modelRel in relsSet.modelRelationships:
                    preferredLabel = modelRel.preferredLabel
                    toConcept = modelRel.toModelObject
                    if preferredLabel is not None and toConcept is not None and \
                       toConcept.label(preferredLabel=preferredLabel,fallbackToQname=False) is None:
                        modelXbrl.error("xbrl.5.2.4.2.1:preferredLabelMissing",
                            _("Presentation relationship from %(source)s to %(target)s in link role %(linkrole)s missing preferredLabel %(preferredLabel)s"),
                            modelObject=modelRel,
                            source=modelRel.fromModelObject.qname, target=toConcept.qname, linkrole=ELR, 
                            preferredLabel=preferredLabel)
            # check essence-alias relationships
            elif arcrole == XbrlConst.essenceAlias:
                for modelRel in relsSet.modelRelationships:
                    fromConcept = modelRel.fromModelObject
                    toConcept = modelRel.toModelObject
                    if fromConcept is not None and toConcept is not None:
                        if fromConcept.type != toConcept.type or fromConcept.periodType != toConcept.periodType:
                            modelXbrl.error("xbrl.5.2.6.2.2:essenceAliasTypes",
                                _("Essence-alias relationship from %(source)s to %(target)s in link role %(linkrole)s has different types or periodTypes"),
                                modelObject=modelRel,
                                source=fromConcept.qname, target=toConcept.qname, linkrole=ELR)
                        fromBalance = fromConcept.balance
                        toBalance = toConcept.balance
                        if fromBalance and toBalance:
                            if fromBalance and toBalance and fromBalance != toBalance:
                                modelXbrl.error("xbrl.5.2.6.2.2:essenceAliasBalance",
                                    _("Essence-alias relationship from %(source)s to %(target)s in link role %(linkrole)s has different balances")).format(
                                    modelObject=modelRel,
                                    source=fromConcept.qname, target=toConcept.qname, linkrole=ELR)
            elif modelXbrl.hasXDT and arcrole.startswith(XbrlConst.dimStartsWith):
                ValidateXbrlDimensions.checkBaseSet(self, arcrole, ELR, relsSet)             
            elif (modelXbrl.hasFormulae or modelXbrl.hasTableRendering) and arcrole.startswith(XbrlConst.formulaStartsWith):
                ValidateFormula.checkBaseSet(self, arcrole, ELR, relsSet)
        modelXbrl.isDimensionsValidated = True
        modelXbrl.profileStat(_("validateRelationships"))
                            
        # instance checks
        modelXbrl.modelManager.showStatus(_("validating instance"))
        self.footnoteRefs = set()
        if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE or \
           modelXbrl.modelDocument.type == ModelDocument.Type.INLINEXBRL:
            self.factsWithDeprecatedIxNamespace = []
            self.checkFacts(modelXbrl.facts)
            
            if self.factsWithDeprecatedIxNamespace:
                self.modelXbrl.info("arelle:info",
                    _("%(count)s facts have deprecated transformation namespace %(namespace)s"),
                        modelObject=self.factsWithDeprecatedIxNamespace,
                        count=len(self.factsWithDeprecatedIxNamespace), 
                        namespace=FunctionIxt.deprecatedNamespaceURI)
            del self.factsWithDeprecatedIxNamespace

            
            #instance checks
            for cntx in modelXbrl.contexts.values():
                if cntx.isStartEndPeriod:
                    try:
                        if cntx.endDatetime <= cntx.startDatetime:
                            self.modelXbrl.error("xbrl.4.7.2:periodStartBeforeEnd",
                                _("Context %(contextID)s must have startDate less than endDate"),
                                modelObject=cntx, contextID=cntx.id)
                    except (TypeError, ValueError) as err:
                        self.modelXbrl.error("xbrl.4.7.2:contextDateError",
                            _("Context %(contextID) startDate or endDate: %(error)s"),
                            modelObject=cntx, contextID=cntx.id, error=err)
                elif cntx.isInstantPeriod:
                    try:
                        cntx.instantDatetime #parse field
                    except ValueError as err:
                        self.modelXbrl.error("xbrl.4.7.2:contextDateError",
                            _("Context %(contextID)s instant date: %(error)s"),
                            modelObject=cntx, contextID=cntx.id, error=err)
                self.segmentScenario(cntx.segment, cntx.id, "segment", "4.7.3.2")
                self.segmentScenario(cntx.scenario, cntx.id, "scenario", "4.7.4")
                
            for unit in modelXbrl.units.values():
                mulDivMeasures = unit.measures
                if mulDivMeasures:
                    for measures in mulDivMeasures:
                        for measure in measures:
                            if measure.namespaceURI == XbrlConst.xbrli and not \
                                measure in (XbrlConst.qnXbrliPure, XbrlConst.qnXbrliShares):
                                    self.modelXbrl.error("xbrl.4.8.2:measureElement",
                                        _("Unit %(unitID)s illegal measure: %(measure)s"),
                                        modelObject=unit, unitID=unit.id, measure=measure)
                    for numeratorMeasure in mulDivMeasures[0]:
                        if numeratorMeasure in mulDivMeasures[1]:
                            self.modelXbrl.error("xbrl.4.8.4:measureBothNumDenom",
                                _("Unit %(unitID)s numerator measure: %(measure)s also appears as denominator measure"),
                                modelObject=unit, unitID=unit.id, measure=numeratorMeasure)
            modelXbrl.profileStat(_("validateInstance"))

            if modelXbrl.hasXDT:            
                modelXbrl.modelManager.showStatus(_("validating dimensions"))
                ''' uncomment if using otherFacts in checkFact
                dimCheckableFacts = set(f 
                                        for f in modelXbrl.factsInInstance
                                        if f.concept.isItem and f.context is not None)
                while (dimCheckableFacts): # check one and all of its compatible family members
                    f = dimCheckableFacts.pop()
                    ValidateXbrlDimensions.checkFact(self, f, dimCheckableFacts)
                del dimCheckableFacts
                '''
                self.checkFactDimensions(modelXbrl.facts) # check fact dimensions in document order
                for cntx in modelXbrl.contexts.values():
                    ValidateXbrlDimensions.checkContext(self,cntx)
                modelXbrl.profileStat(_("validateDimensions"))
                    
        # dimensional validity
        #concepts checks
        modelXbrl.modelManager.showStatus(_("validating concepts"))
        for concept in modelXbrl.qnameConcepts.values():
            conceptType = concept.type
            if XbrlConst.isStandardNamespace(concept.qname.namespaceURI) or \
               not concept.modelDocument.inDTS:
                continue
            
            if concept.isTuple:
                # must be global
                if not concept.getparent().localName == "schema":
                    self.modelXbrl.error("xbrl.4.9:tupleGloballyDeclared",
                        _("Tuple %(concept)s must be declared globally"),
                        modelObject=concept, concept=concept.qname)
                if concept.periodType:
                    self.modelXbrl.error("xbrl.4.9:tuplePeriodType",
                        _("Tuple %(concept)s must not have periodType"),
                        modelObject=concept, concept=concept.qname)
                if concept.balance:
                    self.modelXbrl.error("xbrl.4.9:tupleBalance",
                        _("Tuple %(concept)s must not have balance"),
                        modelObject=concept, concept=concept.qname)
                if conceptType is not None:
                    # check attribute declarations
                    for attribute in conceptType.attributes.values():
                        if attribute.qname.namespaceURI in (XbrlConst.xbrli, XbrlConst.link, XbrlConst.xlink, XbrlConst.xl):
                            self.modelXbrl.error("xbrl.4.9:tupleAttribute",
                                _("Tuple %(concept)s must not have attribute in this namespace %(attribute)s"),
                                modelObject=concept, concept=concept.qname, attribute=attribute.qname)
                    # check for mixed="true" or simple content
                    if XmlUtil.descendantAttr(conceptType, XbrlConst.xsd, ("complexType", "complexContent"), "mixed") == "true":
                        self.modelXbrl.error("xbrl.4.9:tupleMixedContent",
                            _("Tuple %(concept)s must not have mixed content"),
                            modelObject=concept, concept=concept.qname)
                    if XmlUtil.descendant(conceptType, XbrlConst.xsd, "simpleContent"):
                        self.modelXbrl.error("xbrl.4.9:tupleSimpleContent",
                            _("Tuple %(concept)s must not have simple content"),
                            modelObject=concept, concept=concept.qname)
                    # child elements must be item or tuple
                    for elementQname in conceptType.elements:
                        childConcept = self.modelXbrl.qnameConcepts.get(elementQname)
                        if childConcept is None:
                            self.modelXbrl.error("xbrl.4.9:tupleElementUndefined",
                                _("Tuple %(concept)s element %(tupleElement)s not defined"),
                                modelObject=concept, concept=str(concept.qname), tupleElement=elementQname)
                        elif not (childConcept.isItem or childConcept.isTuple or # isItem/isTuple do not include item or tuple itself
                                  childConcept.qname == XbrlConst.qnXbrliItem or # subs group includes item as member
                                  childConcept.qname == XbrlConst.qnXbrliTuple):
                            self.modelXbrl.error("xbrl.4.9:tupleElementItemOrTuple",
                                _("Tuple %(concept)s must not have element %(tupleElement)s not an item or tuple"),
                                modelObject=concept, concept=concept.qname, tupleElement=elementQname)
            elif concept.isItem:
                if concept.periodType not in periodTypeValues: #("instant","duration"):
                    self.modelXbrl.error("xbrl.5.1.1.1:itemPeriodType",
                        _("Item %(concept)s must have a valid periodType"),
                        modelObject=concept, concept=concept.qname)
                if concept.isMonetary:
                    if concept.balance not in balanceValues: #(None, "credit","debit"):
                        self.modelXbrl.error("xbrl.5.1.1.2:itemBalance",
                            _("Item %(concept)s must have a valid balance %(balance)s"),
                            modelObject=concept, concept=concept.qname, balance=concept.balance)
                else:
                    if concept.balance:
                        self.modelXbrl.error("xbrl.5.1.1.2:itemBalance",
                            _("Item %(concept)s may not have a balance"),
                            modelObject=concept, concept=concept.qname)
                if concept.baseXbrliType not in baseXbrliTypes:
                    self.modelXbrl.error("xbrl.5.1.1.3:itemType",
                        _("Item %(concept)s type %(itemType)s invalid"),
                        modelObject=concept, concept=concept.qname, itemType=concept.baseXbrliType)
                if modelXbrl.hasXDT:
                    if concept.isHypercubeItem and not concept.abstract == "true":
                        self.modelXbrl.error("xbrldte:HypercubeElementIsNotAbstractError",
                            _("Hypercube item %(concept)s must be abstract"),
                            modelObject=concept, concept=concept.qname)
                    elif concept.isDimensionItem and not concept.abstract == "true":
                        self.modelXbrl.error("xbrldte:DimensionElementIsNotAbstractError",
                            _("Dimension item %(concept)s must be abstract"),
                            modelObject=concept, concept=concept.qname)
            if modelXbrl.hasXDT:
                ValidateXbrlDimensions.checkConcept(self, concept)
        modelXbrl.profileStat(_("validateConcepts"))
            
        modelXbrl.modelManager.showStatus(_("validating DTS"))
        self.DTSreferenceResourceIDs = {}
        ValidateXbrlDTS.checkDTS(self, modelXbrl.modelDocument, [])
        del self.DTSreferenceResourceIDs
        
        global validateUniqueParticleAttribution
        if validateUniqueParticleAttribution is None:
            from arelle.XmlValidateParticles import validateUniqueParticleAttribution
        for modelType in modelXbrl.qnameTypes.values():
            validateUniqueParticleAttribution(modelXbrl, modelType.particlesList, modelType)
        modelXbrl.profileStat(_("validateDTS"))
        
        if self.validateCalcLB:
            modelXbrl.modelManager.showStatus(_("Validating instance calculations"))
            ValidateXbrlCalcs.validate(modelXbrl, inferDecimals=self.validateInferDecimals)
            modelXbrl.profileStat(_("validateCalculations"))
            
        if (modelXbrl.modelManager.validateUtr or
            (self.parameters and self.parameters.get(qname("forceUtrValidation",noPrefixIsNoNamespace=True),(None,"false"))[1] == "true") or
             #(self.validateEFM and 
             #any((concept.namespaceURI in self.disclosureSystem.standardTaxonomiesDict) 
             #    for concept in self.modelXbrl.nameConcepts.get("UTR",())))):
            (self.validateEFM and any(modelDoc.definesUTR for modelDoc in self.modelXbrl.urlDocs.values()))):
            ValidateUtr.validate(modelXbrl)
            modelXbrl.profileStat(_("validateUTR"))
            
        if modelXbrl.hasFormulae or modelXbrl.modelRenderingTables:
            ValidateFormula.validate(self, 
                                     statusMsg=_("compiling formulae and rendering tables") if (modelXbrl.hasFormulae and modelXbrl.modelRenderingTables)
                                     else (_("compiling formulae") if modelXbrl.hasFormulae
                                           else _("compiling rendering tables")))
            
        modelXbrl.modelManager.showStatus(_("ready"), 2000)
Example #39
0
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, *args, **kwargs):
    # check if big instance and has header with an initial incomplete tree walk (just 2 elements
    if not _streamingExtensionsCheck:
        return None

    # track whether modelXbrl has been validated by this streaming extension
    modelXbrl._streamingExtensionValidated = False

    def logSyntaxErrors(parsercontext):
        for error in parsercontext.error_log:
            modelXbrl.error(
                "xmlSchema:syntax",
                _("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"
                  ),
                modelObject=modelXbrl,
                fileName=os.path.basename(filepath),
                error=error.message,
                line=error.line,
                column=error.column,
                sourceAction="streaming")

    #### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
    #### note: iterparse wants a binary file, but file is text mode
    _file, = modelXbrl.fileSource.file(filepath, binary=True)
    startedAt = time.time()
    modelXbrl.profileActivity()
    ''' this seems twice as slow as iterparse
    class instInfoTarget():
        def __init__(self, element_factory=None, parser=None):
            self.newTree = True
            self.streamingAspects = None
            self.foundInstance = False
            self.creationSoftwareComment = ''
            self.currentEltTag = "(before xbrli:xbrl)"
            self.numRootFacts = 0
        def start(self, tag, attrib, nsmap=None):
            if self.newTree:
                if tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    self.foundInstance = True
                    self.newTree = False
                else: # break 
                    raise NotInstanceDocumentException()
            elif not tag.startswith("{http://www.xbrl.org/"):
                self.numRootFacts += 1
                if self.numRootFacts % 1000 == 0:
                    modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
            self.currentEltTag = tag
        def end(self, tag):
            pass
        def data(self, data):
            pass
        def comment(self, text):
            if not self.foundInstance: # accumulate comments before xbrli:xbrl
                self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text
            elif not self.creationSoftwareComment:
                self.creationSoftwareComment = text # or first comment after xbrli:xbrl
        def pi(self, target, data):
            if target == "xbrl-streamable-instance":
                if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl":
                    self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results
                else:
                    modelXbrl.error("streamingExtensions:headerMisplaced",
                            _("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"),
                            modelObject=modelXbrl, target=target, element=self.currentEltTag)
        def close(self):
            if not self.creationSoftwareComment:
                self.creationSoftwareComment = None
            return True
    instInfo = instInfoTarget()
    infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo)
    try:
        etree.parse(_file, parser=infoParser, base_url=filepath)
    except NotInstanceDocumentException:
        pass
    '''
    foundErrors = False
    foundInstance = False
    streamingAspects = None
    creationSoftwareComment = None
    instInfoNumRootFacts = 0
    numElts = 0
    elt = None
    instInfoContext = etree.iterparse(_file,
                                      events=("start", "end"),
                                      huge_tree=True)
    try:
        for event, elt in instInfoContext:
            if event == "start":
                if elt.getparent() is not None:
                    if elt.getparent(
                    ).tag == "{http://www.xbrl.org/2003/instance}xbrl":
                        if not foundInstance:
                            foundInstance = True
                            pi = precedingProcessingInstruction(
                                elt, "xbrl-streamable-instance")
                            if pi is None:
                                break
                            else:
                                streamingAspects = dict(pi.attrib.copy())
                                if creationSoftwareComment is None:
                                    creationSoftwareComment = precedingComment(
                                        elt)
                        if not elt.tag.startswith("{http://www.xbrl.org/"):
                            instInfoNumRootFacts += 1
                            if instInfoNumRootFacts % 1000 == 0:
                                modelXbrl.profileActivity(
                                    "... streaming tree check",
                                    minTimeToShow=20.0)
                    elif not foundInstance:
                        break
                elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    creationSoftwareComment = precedingComment(elt)
                    if precedingProcessingInstruction(
                            elt, "xbrl-streamable-instance") is not None:
                        modelXbrl.error(
                            "streamingExtensions:headerMisplaced",
                            _("Header is misplaced: %(error)s, must follow xbrli:xbrl element"
                              ),
                            modelObject=elt)
            elif event == "end":
                elt.clear()
                numElts += 1
                if numElts % 1000 == 0 and elt.getparent() is not None:
                    while elt.getprevious() is not None and elt.getparent(
                    ) is not None:
                        del elt.getparent()[0]
    except etree.XMLSyntaxError as err:
        modelXbrl.error("xmlSchema:syntax",
                        _("Unrecoverable error: %(error)s"),
                        error=err)
        _file.close()
        return err

    _file.seek(0, io.SEEK_SET)  # allow reparsing
    if not foundInstance or streamingAspects is None:
        del elt
        _file.close()
        return None
    modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
    startedAt = time.time()
    try:
        version = Decimal(streamingAspects.get("version"))
        if int(version) != 1:
            modelXbrl.error(
                "streamingExtensions:unsupportedVersion",
                _("Streaming version %(version)s, major version number must be 1"
                  ),
                modelObject=elt,
                version=version)
            foundErrors = True
    except (InvalidOperation, OverflowError):
        modelXbrl.error("streamingExtensions:versionError",
                        _("Version %(version)s, number must be 1.n"),
                        modelObject=elt,
                        version=streamingAspects.get("version", "(none)"))
        foundErrors = True
    for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
        try:
            bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
            if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
                raise InvalidOperation
            elif bufAspect == "contextBuffer":
                contextBufferLimit = bufLimit
            elif bufAspect == "unitBuffer":
                unitBufferLimit = bufLimit
            elif bufAspect == "footnoteBuffer":
                footnoteBufferLimit = bufLimit
        except InvalidOperation:
            modelXbrl.error(
                "streamingExtensions:valueError",
                _("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"
                  ),
                modelObject=elt,
                attrib=bufAspect,
                value=streamingAspects.get(bufAspect))
            foundErrors = True
    if _streamingExtensionsValidate:
        incompatibleValidations = []
        _validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
        _disclosureSystem = modelXbrl.modelManager.disclosureSystem
        if _validateDisclosureSystem and _disclosureSystem.validationType == "EFM":
            incompatibleValidations.append("EFM")
        if _validateDisclosureSystem and _disclosureSystem.validationType == "GFM":
            incompatibleValidations.append("GFM")
        if _validateDisclosureSystem and _disclosureSystem.validationType == "HMRC":
            incompatibleValidations.append("HMRC")
        if modelXbrl.modelManager.validateCalcLB:
            incompatibleValidations.append("calculation LB")
        if incompatibleValidations:
            modelXbrl.error(
                "streamingExtensions:incompatibleValidation",
                _("Streaming instance validation does not support %(incompatibleValidations)s validation"
                  ),
                modelObject=modelXbrl,
                incompatibleValidations=', '.join(incompatibleValidations))
            foundErrors = True
    if instInfoContext.error_log:
        foundErrors = True
    logSyntaxErrors(instInfoContext)
    del instInfoContext  # dereference

    for pluginMethod in pluginClassMethods("Streaming.BlockStreaming"):
        _blockingPluginName = pluginMethod(modelXbrl)
        if _blockingPluginName:  # name of blocking plugin is returned
            modelXbrl.error(
                "streamingExtensions:incompatiblePlugIn",
                _("Streaming instance not supported by plugin %(blockingPlugin)s"
                  ),
                modelObject=modelXbrl,
                blockingPlugin=_blockingPluginName)
            foundErrors = True

    if foundErrors:
        _file.close()
        return None

    _encoding = XmlUtil.encoding(_file.read(512))
    _file.seek(0, io.SEEK_SET)  # allow reparsing

    if _streamingExtensionsValidate:
        validator = Validate(modelXbrl)
        instValidator = validator.instValidator

    contextBuffer = []
    contextsToDrop = []
    unitBuffer = []
    unitsToDrop = []
    footnoteBuffer = []
    footnoteLinksToDrop = []

    _streamingFactsPlugin = any(
        True for pluginMethod in pluginClassMethods("Streaming.Facts"))
    _streamingValidateFactsPlugin = (_streamingExtensionsValidate and any(
        True
        for pluginMethod in pluginClassMethods("Streaming.ValidateFacts")))
    ''' this is very much slower than iterparse
    class modelLoaderTarget():
        def __init__(self, element_factory=None, parser=None):
            self.newTree = True
            self.currentMdlObj = None
            self.beforeInstanceStream = True
            self.beforeStartStreamingPlugin = True
            self.numRootFacts = 1
            modelXbrl.makeelementParentModelObject = None
            modelXbrl.isStreamingMode = True
            self.factsCheckVersion = None
            self.factsCheckMd5s = Md5Sum()
        def start(self, tag, attrib, nsmap=None):
            modelXbrl.makeelementParentModelObject = self.currentMdlObj # pass parent to makeelement for ModelObjectFactory
            mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
            mdlObj.sourceline = 1
            if self.newTree:
                self.newTree = False
                self.currentMdlObj = mdlObj
                modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
                modelXbrl.modelDocument = modelDocument # needed for incremental validation
                mdlObj.init(modelDocument)
                modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement 
                modelDocument.parserLookupName = _parserLookupName
                modelDocument.parserLookupClass = _parserLookupClass
                modelDocument.xmlRootElement = mdlObj
                modelDocument.schemaLocationElements.add(mdlObj)
                modelDocument.documentEncoding = _encoding
                modelDocument._creationSoftwareComment = creationSoftwareComment
                modelXbrl.info("streamingExtensions:streaming",
                               _("Stream processing this instance."),
                               modelObject = modelDocument)
            else:
                self.currentMdlObj.append(mdlObj)
                self.currentMdlObj = mdlObj
                mdlObj._init()
                ns = mdlObj.namespaceURI
                ln = mdlObj.localName
                if (self.beforeInstanceStream and (
                    (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
                    (ns == XbrlConst.xbrli and ln in ("context", "unit")) or
                    (ns not in (XbrlConst.link, XbrlConst.xbrli)))):
                    self.beforeInstanceStream = False
                    if _streamingExtensionsValidate:
                        instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters(modelXbrl.prefixedNamespaces))
                    else: # need default dimensions
                        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
                elif not self.beforeInstanceStream and self.beforeStartStreamingPlugin:
                    for pluginMethod in pluginClassMethods("Streaming.Start"):
                        pluginMethod(modelXbrl)
                    self.beforeStartStreamingPlugin = False
            return mdlObj
        def end(self, tag):
            modelDocument = modelXbrl.modelDocument
            mdlObj = self.currentMdlObj
            parentMdlObj = mdlObj.getparent()
            self.currentMdlObj = parentMdlObj
            ns = mdlObj.namespaceURI
            ln = mdlObj.localName
            if ns == XbrlConst.xbrli:
                if ln == "context":
                    if mdlObj.get("sticky"):
                        del mdlObj.attrib["sticky"]
                        XmlValidate.validate(modelXbrl, mdlObj)
                        modelDocument.contextDiscover(mdlObj)
                    else:
                        if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
                            # drop before adding as dropped may have same id as added
                            cntx = contextBuffer.pop(0)
                            if _streamingValidateFactsPlugin:
                                contextsToDrop.append(cntx)
                            else:
                                dropContext(modelXbrl, cntx)
                                del parentMdlObj[parentMdlObj.index(cntx)]
                            cntx = None
                        #>>XmlValidate.validate(modelXbrl, mdlObj)
                        #>>modelDocument.contextDiscover(mdlObj)
                        if contextBufferLimit.is_finite():
                            contextBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        contextsToCheck = (mdlObj,)
                        instValidator.checkContexts(contextsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkContextsDimensions(contextsToCheck)
                        del contextsToCheck # dereference
                elif ln == "unit":
                    if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
                        # drop before adding as dropped may have same id as added
                        unit = unitBuffer.pop(0)
                        if _streamingValidateFactsPlugin:
                            unitsToDrop.append(unit)
                        else:
                            dropUnit(modelXbrl, unit)
                            del parentMdlObj[parentMdlObj.index(unit)]
                        unit = None 
                    #>>XmlValidate.validate(modelXbrl, mdlObj)
                    #>>modelDocument.unitDiscover(mdlObj)
                    if unitBufferLimit.is_finite():
                        unitBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkUnits( (mdlObj,) )
                elif ln == "xbrl": # end of document
                    # check remaining batched facts if any
                    if _streamingValidateFactsPlugin:
                        # plugin attempts to process batch of all root facts not yet processed (not just current one)
                        # finish any final batch of facts
                        if len(modelXbrl.facts) > 0:
                            factsToCheck = modelXbrl.facts.copy()
                            factsHaveBeenProcessed = True
                            # can block facts deletion if required data not yet available, such as numeric unit for DpmDB
                            for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
                                if not pluginMethod(modelXbrl, factsToCheck):
                                    factsHaveBeenProcessed = False
                            if factsHaveBeenProcessed:
                                for fact in factsToCheck:
                                    dropFact(modelXbrl, fact, modelXbrl.facts)
                                    del parentMdlObj[parentMdlObj.index(fact)]
                                for cntx in contextsToDrop:
                                    dropContext(modelXbrl, cntx)
                                    del parentMdlObj[parentMdlObj.index(cntx)]
                                for unit in unitsToDrop:
                                    dropUnit(modelXbrl, unit)
                                    del parentMdlObj[parentMdlObj.index(unit)]
                                for footnoteLink in footnoteLinksToDrop:
                                    dropFootnoteLink(modelXbrl, footnoteLink)
                                    del parentMdlObj[parentMdlObj.index(footnoteLink)]
                                fact = cntx = unit = footnoteLink = None
                                del contextsToDrop[:]
                                del unitsToDrop[:]
                                del footnoteLinksToDrop[:]
                            del factsToCheck
                    # check remaining footnote refs
                    for footnoteLink in footnoteBuffer:
                        checkFootnoteHrefs(modelXbrl, footnoteLink)
                    for pluginMethod in pluginClassMethods("Streaming.Finish"):
                        pluginMethod(modelXbrl)
            elif ns == XbrlConst.link:
                if ln == "footnoteLink":
                    XmlValidate.validate(modelXbrl, mdlObj)
                    footnoteLinks = (mdlObj,)
                    modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
                    if footnoteBufferLimit.is_finite():
                        footnoteBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkLinks(footnoteLinks)
                        if len(footnoteBuffer) > footnoteBufferLimit:
                            # check that hrefObjects for locators were all satisfied
                                # drop before addition as dropped may have same id as added
                            footnoteLink = footnoteBuffer.pop(0)
                            checkFootnoteHrefs(modelXbrl, footnoteLink)
                            if _streamingValidateFactsPlugin:
                                footnoteLinksToDrop.append(footnoteLink)
                            else:
                                dropFootnoteLink(modelXbrl, footnoteLink)
                                del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            footnoteLink = None
                    footnoteLinks = None
                elif ln in ("schemaRef", "linkbaseRef"):
                    modelDocument.discoverHref(mdlObj)
                elif not modelXbrl.skipDTS:
                    if ln in ("roleRef", "arcroleRef"):
                        modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
            elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
                self.numRootFacts += 1
                #>>XmlValidate.validate(modelXbrl, mdlObj)
                #>>modelDocument.factDiscover(mdlObj, modelXbrl.facts)
                if self.factsCheckVersion:
                    self.factCheckFact(mdlObj)
                if _streamingExtensionsValidate or _streamingValidateFactsPlugin:
                    factsToCheck = (mdlObj,)  # validate current fact by itself
                    if _streamingExtensionsValidate:
                        instValidator.checkFacts(factsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkFactsDimensions(factsToCheck)
                    if _streamingValidateFactsPlugin:
                        # plugin attempts to process batch of all root facts not yet processed (not just current one)
                        # use batches of 1000 facts
                        if len(modelXbrl.facts) > 1000:
                            factsToCheck = modelXbrl.facts.copy()
                            factsHaveBeenProcessed = True
                            # can block facts deletion if required data not yet available, such as numeric unit for DpmDB
                            for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
                                if not pluginMethod(modelXbrl, factsToCheck):
                                    factsHaveBeenProcessed = False
                            if factsHaveBeenProcessed:
                                for fact in factsToCheck:
                                    dropFact(modelXbrl, fact, modelXbrl.facts)
                                    del parentMdlObj[parentMdlObj.index(fact)]
                                for cntx in contextsToDrop:
                                    dropContext(modelXbrl, cntx)
                                    del parentMdlObj[parentMdlObj.index(cntx)]
                                for unit in unitsToDrop:
                                    dropUnit(modelXbrl, unit)
                                    del parentMdlObj[parentMdlObj.index(unit)]
                                for footnoteLink in footnoteLinksToDrop:
                                    dropFootnoteLink(modelXbrl, footnoteLink)
                                    del parentMdlObj[parentMdlObj.index(footnoteLink)]
                                fact = cntx = unit = footnoteLink = None
                                del contextsToDrop[:]
                                del unitsToDrop[:]
                                del footnoteLinksToDrop[:]
                            del factsToCheck # dereference fact or batch of facts
                    else:
                        dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed
                        del parentMdlObj[parentMdlObj.index(mdlObj)]
                if self.numRootFacts % 1000 == 0:
                    pass
                    #modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, 
                    #                                                                          100.0 * self.numRootFacts / instInfoNumRootFacts), 
                    #                          minTimeToShow=20.0)
                    gc.collect()
                    sys.stdout.write ("\rAt fact {} of {} mem {}".format(self.numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed))
            return mdlObj
        def data(self, data):
            self.currentMdlObj.text = data
        def comment(self, text):
            pass
        def pi(self, target, data):
            if target == "xbrl-facts-check":
                _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", data)
                if _match:
                    _matchGroups = _match.groups()
                    if len(_matchGroups) == 2:
                        if _matchGroups[0] == "version":
                            self.factsCheckVersion = _matchGroups[1]
                        elif _matchGroups[0] == "sum-of-fact-md5s":
                            try:
                                expectedMd5 = Md5Sum(_matchGroups[1])
                                if self.factsCheckMd5s != expectedMd5:
                                    modelXbrl.warning("streamingExtensions:xbrlFactsCheckWarning",
                                            _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"),
                                            modelObject=modelXbrl, expectedMd5=expectedMd5, actualMd5Sum=self.factsCheckMd5s)
                                else:
                                    modelXbrl.info("info",
                                            _("Successful XBRL facts sum of md5s."),
                                            modelObject=modelXbrl)
                            except ValueError:
                                modelXbrl.error("streamingExtensions:xbrlFactsCheckError",
                                        _("Invalid sum-of-md5s %(sumOfMd5)s"),
                                        modelObject=modelXbrl, sumOfMd5=_matchGroups[1])
        def close(self):
            del modelXbrl.makeelementParentModelObject
            return None
        
        def factCheckFact(self, fact):
            self.factsCheckMd5s += fact.md5sum
            for _tupleFact in fact.modelTupleFacts:
                self.factCheckFact(_tupleFact)
        
    _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget())
    etree.parse(_file, parser=_parser, base_url=filepath)
    logSyntaxErrors(_parser)
    '''
    # replace modelLoaderTarget with iterparse (as it now supports CustomElementClassLookup)
    streamingParserContext = etree.iterparse(_file,
                                             events=("start", "end"),
                                             huge_tree=True)
    from arelle.ModelObjectFactory import setParserElementClassLookup
    modelXbrl.isStreamingMode = True  # must be set before setting element class lookup
    (_parser, _parserLookupName,
     _parserLookupClass) = setParserElementClassLookup(streamingParserContext,
                                                       modelXbrl)
    foundInstance = False
    beforeInstanceStream = beforeStartStreamingPlugin = True
    numRootFacts = 0
    factsCheckVersion = None

    def factCheckFact(fact):
        modelDocument._factsCheckMd5s += fact.md5sum
        for _tupleFact in fact.modelTupleFacts:
            factCheckFact(_tupleFact)

    for event, mdlObj in streamingParserContext:
        if event == "start":
            if mdlObj.tag == "{http://www.xbrl.org/2003/instance}xbrl":
                modelDocument = ModelDocument(modelXbrl, Type.INSTANCE,
                                              mappedUri, filepath,
                                              mdlObj.getroottree())
                modelXbrl.modelDocument = modelDocument  # needed for incremental validation
                mdlObj.init(modelDocument)
                modelDocument.parser = _parser  # needed for XmlUtil addChild's makeelement
                modelDocument.parserLookupName = _parserLookupName
                modelDocument.parserLookupClass = _parserLookupClass
                modelDocument.xmlRootElement = mdlObj
                modelDocument.schemaLocationElements.add(mdlObj)
                modelDocument.documentEncoding = _encoding
                modelDocument._creationSoftwareComment = precedingComment(
                    mdlObj)
                modelDocument._factsCheckMd5s = Md5Sum()
                modelXbrl.info("streamingExtensions:streaming",
                               _("Stream processing this instance."),
                               modelObject=modelDocument)
            elif mdlObj.getparent() is not None:
                mdlObj._init()  # requires discovery as part of start elements
                if mdlObj.getparent(
                ).tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    if not foundInstance:
                        foundInstance = True
                        pi = precedingProcessingInstruction(
                            mdlObj, "xbrl-facts-check")
                        if pi is not None:
                            factsCheckVersion = pi.attrib.get("version", None)
                elif not foundInstance:
                    break
                ns = mdlObj.qname.namespaceURI
                ln = mdlObj.qname.localName
                if beforeInstanceStream:
                    if ((ns == XbrlConst.link
                         and ln not in ("schemaRef", "linkbaseRef")) or
                        (ns == XbrlConst.xbrli and ln in ("context", "unit"))
                            or (ns not in (XbrlConst.link, XbrlConst.xbrli))):
                        beforeInstanceStream = False
                        if _streamingExtensionsValidate:
                            instValidator.validate(
                                modelXbrl,
                                modelXbrl.modelManager.formulaOptions.
                                typedParameters(modelXbrl.prefixedNamespaces))
                        else:  # need default dimensions
                            ValidateXbrlDimensions.loadDimensionDefaults(
                                modelXbrl)
                elif not beforeInstanceStream and beforeStartStreamingPlugin:
                    for pluginMethod in pluginClassMethods("Streaming.Start"):
                        pluginMethod(modelXbrl)
                    beforeStartStreamingPlugin = False
        elif event == "end":
            parentMdlObj = mdlObj.getparent()
            ns = mdlObj.namespaceURI
            ln = mdlObj.localName
            if ns == XbrlConst.xbrli:
                if ln == "context":
                    if mdlObj.get("sticky"):
                        del mdlObj.attrib["sticky"]
                        XmlValidate.validate(modelXbrl, mdlObj)
                        modelDocument.contextDiscover(mdlObj)
                    else:
                        if len(contextBuffer) >= contextBufferLimit:
                            # drop before adding as dropped may have same id as added
                            cntx = contextBuffer.pop(0)
                            if _streamingFactsPlugin or _streamingValidateFactsPlugin:
                                contextsToDrop.append(cntx)
                            else:
                                dropContext(modelXbrl, cntx)
                                #>>del parentMdlObj[parentMdlObj.index(cntx)]
                            cntx = None
                        XmlValidate.validate(modelXbrl, mdlObj)
                        modelDocument.contextDiscover(mdlObj)
                        if contextBufferLimit.is_finite():
                            contextBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        contextsToCheck = (mdlObj, )
                        instValidator.checkContexts(contextsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkContextsDimensions(
                                contextsToCheck)
                        del contextsToCheck  # dereference
                elif ln == "unit":
                    if len(unitBuffer) >= unitBufferLimit:
                        # drop before additing as dropped may have same id as added
                        unit = unitBuffer.pop(0)
                        if _streamingFactsPlugin or _streamingValidateFactsPlugin:
                            unitsToDrop.append(unit)
                        else:
                            dropUnit(modelXbrl, unit)
                            #>>del parentMdlObj[parentMdlObj.index(unit)]
                        unit = None
                    XmlValidate.validate(modelXbrl, mdlObj)
                    modelDocument.unitDiscover(mdlObj)
                    if unitBufferLimit.is_finite():
                        unitBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkUnits((mdlObj, ))
                elif ln == "xbrl":  # end of document
                    # check remaining batched facts if any
                    if _streamingFactsPlugin or _streamingValidateFactsPlugin:
                        # plugin attempts to process batch of all root facts not yet processed (not just current one)
                        # finish any final batch of facts
                        if len(modelXbrl.facts) > 0:
                            factsToCheck = modelXbrl.facts.copy()
                            # can block facts deletion if required data not yet available, such as numeric unit for DpmDB
                            if _streamingValidateFactsPlugin:
                                for pluginMethod in pluginClassMethods(
                                        "Streaming.ValidateFacts"):
                                    pluginMethod(instValidator, factsToCheck)
                            if _streamingFactsPlugin:
                                for pluginMethod in pluginClassMethods(
                                        "Streaming.Facts"):
                                    pluginMethod(modelXbrl, factsToCheck)
                            for fact in factsToCheck:
                                dropFact(modelXbrl, fact, modelXbrl.facts)
                                #>>del parentMdlObj[parentMdlObj.index(fact)]
                            for cntx in contextsToDrop:
                                dropContext(modelXbrl, cntx)
                                #>>del parentMdlObj[parentMdlObj.index(cntx)]
                            for unit in unitsToDrop:
                                dropUnit(modelXbrl, unit)
                                #>>del parentMdlObj[parentMdlObj.index(unit)]
                            for footnoteLink in footnoteLinksToDrop:
                                dropFootnoteLink(modelXbrl, footnoteLink)
                                #>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            fact = cntx = unit = footnoteLink = None
                            del contextsToDrop[:]
                            del unitsToDrop[:]
                            del footnoteLinksToDrop[:]
                            del factsToCheck
                    # check remaining footnote refs
                    for footnoteLink in footnoteBuffer:
                        checkFootnoteHrefs(modelXbrl, footnoteLink)
                    pi = childProcessingInstruction(mdlObj,
                                                    "xbrl-facts-check",
                                                    reversed=True)
                    if pi is not None:  # attrib is in .text, not attrib, no idea why!!!
                        _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']",
                                           pi.text)
                        if _match:
                            _matchGroups = _match.groups()
                            if len(_matchGroups) == 2:
                                if _matchGroups[0] == "sum-of-fact-md5s":
                                    try:
                                        expectedMd5 = Md5Sum(_matchGroups[1])
                                        if modelDocument._factsCheckMd5s != expectedMd5:
                                            modelXbrl.warning(
                                                "streamingExtensions:xbrlFactsCheckWarning",
                                                _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"
                                                  ),
                                                modelObject=modelXbrl,
                                                expectedMd5=expectedMd5,
                                                actualMd5Sum=modelDocument.
                                                _factsCheckMd5s)
                                        else:
                                            modelXbrl.info(
                                                "info",
                                                _("Successful XBRL facts sum of md5s."
                                                  ),
                                                modelObject=modelXbrl)
                                    except ValueError:
                                        modelXbrl.error(
                                            "streamingExtensions:xbrlFactsCheckError",
                                            _("Invalid sum-of-md5s %(sumOfMd5)s"
                                              ),
                                            modelObject=modelXbrl,
                                            sumOfMd5=_matchGroups[1])
                    if _streamingValidateFactsPlugin:
                        for pluginMethod in pluginClassMethods(
                                "Streaming.ValidateFinish"):
                            pluginMethod(instValidator)
                    if _streamingFactsPlugin:
                        for pluginMethod in pluginClassMethods(
                                "Streaming.Finish"):
                            pluginMethod(modelXbrl)
            elif ns == XbrlConst.link:
                if ln in ("schemaRef", "linkbaseRef"):
                    modelDocument.discoverHref(
                        mdlObj,
                        urlRewritePluginClass=
                        "ModelDocument.InstanceSchemaRefRewriter")
                elif ln in ("roleRef", "arcroleRef"):
                    modelDocument.linkbaseDiscover((mdlObj, ), inInstance=True)
                elif ln == "footnoteLink":
                    XmlValidate.validate(modelXbrl, mdlObj)
                    footnoteLinks = (mdlObj, )
                    modelDocument.linkbaseDiscover(footnoteLinks,
                                                   inInstance=True)
                    if footnoteBufferLimit.is_finite():
                        footnoteBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkLinks(footnoteLinks)
                        if len(footnoteBuffer) > footnoteBufferLimit:
                            # check that hrefObjects for locators were all satisfied
                            # drop before addition as dropped may have same id as added
                            footnoteLink = footnoteBuffer.pop(0)
                            checkFootnoteHrefs(modelXbrl, footnoteLink)
                            if _streamingValidateFactsPlugin:
                                footnoteLinksToDrop.append(footnoteLink)
                            else:
                                dropFootnoteLink(modelXbrl, footnoteLink)
                                #>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            footnoteLink = None
                    footnoteLinks = None
            elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl and isinstance(
                    mdlObj, ModelFact):
                numRootFacts += 1
                XmlValidate.validate(modelXbrl, mdlObj)
                modelDocument.factDiscover(mdlObj, modelXbrl.facts)
                if factsCheckVersion:
                    factCheckFact(mdlObj)
                if _streamingExtensionsValidate or _streamingFactsPlugin or _streamingValidateFactsPlugin:
                    factsToCheck = (mdlObj,
                                    )  # validate current fact by itself
                    if _streamingExtensionsValidate:
                        instValidator.checkFacts(factsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkFactsDimensions(factsToCheck)
                    if _streamingFactsPlugin or _streamingValidateFactsPlugin:
                        # plugin attempts to process batch of all root facts not yet processed (not just current one)
                        # use batches of 1000 facts
                        if len(modelXbrl.facts) > 1000:
                            factsToCheck = modelXbrl.facts.copy()
                            # can block facts deletion if required data not yet available, such as numeric unit for DpmDB
                            if _streamingValidateFactsPlugin:
                                for pluginMethod in pluginClassMethods(
                                        "Streaming.ValidateFacts"):
                                    pluginMethod(instValidator, factsToCheck)
                            if _streamingFactsPlugin:
                                for pluginMethod in pluginClassMethods(
                                        "Streaming.Facts"):
                                    pluginMethod(modelXbrl, factsToCheck)
                            for fact in factsToCheck:
                                dropFact(modelXbrl, fact, modelXbrl.facts)
                                #>>del parentMdlObj[parentMdlObj.index(fact)]
                            for cntx in contextsToDrop:
                                dropContext(modelXbrl, cntx)
                                #>>del parentMdlObj[parentMdlObj.index(cntx)]
                            for unit in unitsToDrop:
                                dropUnit(modelXbrl, unit)
                                #>>del parentMdlObj[parentMdlObj.index(unit)]
                            for footnoteLink in footnoteLinksToDrop:
                                dropFootnoteLink(modelXbrl, footnoteLink)
                                #>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            fact = cntx = unit = footnoteLink = None
                            del contextsToDrop[:]
                            del unitsToDrop[:]
                            del footnoteLinksToDrop[:]
                            del factsToCheck  # dereference fact or batch of facts
                    else:
                        dropFact(
                            modelXbrl, mdlObj,
                            modelXbrl.facts)  # single fact has been processed
                        #>>del parentMdlObj[parentMdlObj.index(mdlObj)]
                if numRootFacts % 1000 == 0:
                    pass
                    #modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts,
                    #                                                                          100.0 * self.numRootFacts / instInfoNumRootFacts),
                    #                          minTimeToShow=20.0)
                    #gc.collect()
                    #sys.stdout.write ("\rAt fact {} of {} mem {}".format(numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed))
    if mdlObj is not None:
        mdlObj.clear()
    del _parser, _parserLookupName, _parserLookupClass

    if _streamingExtensionsValidate and validator is not None:
        _file.close()
        del instValidator
        validator.close()
        # track that modelXbrl has been validated by this streaming extension
        modelXbrl._streamingExtensionValidated = True

    modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
    return modelXbrl.modelDocument
Example #40
0
 def checkFactDimensions(self, facts): # check fact dimensions in document order
     for f in facts:
         if f.concept.isItem and f.context is not None:
             ValidateXbrlDimensions.checkFact(self, f)
         elif f.modelTupleFacts:
             self.checkFactDimensions(f.modelTupleFacts)
Example #41
0
def createTargetInstance(modelXbrl,
                         targetUrl,
                         targetDocumentSchemaRefs,
                         filingFiles,
                         baseXmlLang=None,
                         defaultXmlLang=None):
    def addLocallyReferencedFile(elt, filingFiles):
        if elt.tag in ("a", "img"):
            for attrTag, attrValue in elt.items():
                if attrTag in ("href", "src") and not isHttpUrl(
                        attrValue) and not os.path.isabs(attrValue):
                    attrValue = attrValue.partition('#')[0]  # remove anchor
                    if attrValue:  # ignore anchor references to base document
                        attrValue = os.path.normpath(
                            attrValue
                        )  # change url path separators to host separators
                        file = os.path.join(sourceDir, attrValue)
                        if modelXbrl.fileSource.isInArchive(
                                file,
                                checkExistence=True) or os.path.exists(file):
                            filingFiles.add(file)

    targetInstance = ModelXbrl.create(
        modelXbrl.modelManager,
        newDocumentType=Type.INSTANCE,
        url=targetUrl,
        schemaRefs=targetDocumentSchemaRefs,
        isEntry=True,
        discover=False)  # don't attempt to load DTS
    ixTargetRootElt = modelXbrl.ixTargetRootElements[getattr(
        modelXbrl, "ixdsTarget", None)]
    langIsSet = False
    # copy ix resources target root attributes
    for attrName, attrValue in ixTargetRootElt.items():
        if attrName != "target":  # ix:references target is not mapped to xbrli:xbrl
            targetInstance.modelDocument.xmlRootElement.set(
                attrName, attrValue)
        if attrName == "{http://www.w3.org/XML/1998/namespace}lang":
            langIsSet = True
            defaultXmlLang = attrValue
        if attrName.startswith("{"):
            ns, _sep, ln = attrName[1:].rpartition("}")
            if ns:
                prefix = xmlnsprefix(ixTargetRootElt, ns)
                if prefix not in (None, "xml"):
                    setXmlns(targetInstance.modelDocument, prefix, ns)

    if not langIsSet and baseXmlLang:
        targetInstance.modelDocument.xmlRootElement.set(
            "{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
        if defaultXmlLang is None:
            defaultXmlLang = baseXmlLang  # allows facts/footnotes to override baseXmlLang
    ValidateXbrlDimensions.loadDimensionDefaults(
        targetInstance)  # need dimension defaults
    # roleRef and arcroleRef (of each inline document)
    for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
        for roleRefElt in sourceRefs.values():
            addChild(targetInstance.modelDocument.xmlRootElement,
                     roleRefElt.qname,
                     attributes=roleRefElt.items())

    # contexts
    for context in sorted(modelXbrl.contexts.values(),
                          key=lambda c: c.objectIndex
                          ):  # contexts may come from multiple IXDS files
        ignore = targetInstance.createContext(
            context.entityIdentifier[0],
            context.entityIdentifier[1],
            'instant' if context.isInstantPeriod else
            'duration' if context.isStartEndPeriod else 'forever',
            context.startDatetime,
            context.endDatetime,
            None,
            context.qnameDims, [], [],
            id=context.id)
    for unit in sorted(modelXbrl.units.values(), key=lambda u: u.objectIndex
                       ):  # units may come from multiple IXDS files
        measures = unit.measures
        ignore = targetInstance.createUnit(measures[0],
                                           measures[1],
                                           id=unit.id)

    modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
    newFactForOldObjId = {}

    def createFacts(facts, parent):
        for fact in facts:
            if fact.isItem:  # HF does not de-duplicate, which is currently-desired behavior
                attrs = {"contextRef": fact.contextID}
                if fact.id:
                    attrs["id"] = fact.id
                if fact.isNumeric:
                    attrs["unitRef"] = fact.unitID
                    if fact.get("decimals"):
                        attrs["decimals"] = fact.get("decimals")
                    if fact.get("precision"):
                        attrs["precision"] = fact.get("precision")
                if fact.isNil:
                    attrs[XbrlConst.qnXsiNil] = "true"
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.textValue
                for attrName, attrValue in fact.items():
                    if attrName.startswith("{"):
                        attrs[qname(
                            attrName, fact.nsmap
                        )] = attrValue  # using qname allows setting prefix in extracted instance
                newFact = targetInstance.createFact(fact.qname,
                                                    attributes=attrs,
                                                    text=text,
                                                    parent=parent)
                # if fact.isFraction, create numerator and denominator
                newFactForOldObjId[fact.objectIndex] = newFact
                if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
                    # check for img and other filing references so that referenced files are included in the zip.
                    for xmltext in [text] + CDATApattern.findall(text):
                        try:
                            for elt in XML("<body>\n{0}\n</body>\n".format(
                                    xmltext)).iter():
                                addLocallyReferencedFile(elt, filingFiles)
                        except (XMLSyntaxError, UnicodeDecodeError):
                            pass  # TODO: Why ignore UnicodeDecodeError?
            elif fact.isTuple:
                attrs = {}
                if fact.id:
                    attrs["id"] = fact.id
                if fact.isNil:
                    attrs[XbrlConst.qnXsiNil] = "true"
                for attrName, attrValue in fact.items():
                    if attrName.startswith("{"):
                        attrs[qname(attrName, fact.nsmap)] = attrValue
                newTuple = targetInstance.createFact(fact.qname,
                                                     attributes=attrs,
                                                     parent=parent)
                newFactForOldObjId[fact.objectIndex] = newTuple
                createFacts(fact.modelTupleFacts, newTuple)

    createFacts(modelXbrl.facts, None)
    modelXbrl.modelManager.showStatus(
        _("Creating and validating footnotes and relationships"))
    HREF = "{http://www.w3.org/1999/xlink}href"
    footnoteLinks = defaultdict(list)
    footnoteIdCount = {}
    for linkKey, linkPrototypes in modelXbrl.baseSets.items():
        arcrole, linkrole, linkqname, arcqname = linkKey
        if (linkrole and linkqname and arcqname and  # fully specified roles
                arcrole != "XBRL-footnotes" and any(
                    lP.modelDocument.type == Type.INLINEXBRL
                    for lP in linkPrototypes)):
            for linkPrototype in linkPrototypes:
                if linkPrototype not in footnoteLinks[linkrole]:
                    footnoteLinks[linkrole].append(linkPrototype)
    for linkrole in sorted(footnoteLinks.keys()):
        for linkPrototype in footnoteLinks[linkrole]:
            newLink = addChild(targetInstance.modelDocument.xmlRootElement,
                               linkPrototype.qname,
                               attributes=linkPrototype.attributes)
            for linkChild in linkPrototype:
                attributes = linkChild.attributes
                if isinstance(linkChild, LocPrototype):
                    if HREF not in linkChild.attributes:
                        linkChild.attributes[HREF] = \
                        "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
                    addChild(newLink, linkChild.qname, attributes=attributes)
                elif isinstance(linkChild, ArcPrototype):
                    addChild(newLink, linkChild.qname, attributes=attributes)
                elif isinstance(linkChild, ModelInlineFootnote):
                    idUseCount = footnoteIdCount.get(linkChild.footnoteID,
                                                     0) + 1
                    if idUseCount > 1:  # if footnote with id in other links bump the id number
                        attributes = linkChild.attributes.copy()
                        attributes["id"] = "{}_{}".format(
                            attributes["id"], idUseCount)
                    footnoteIdCount[linkChild.footnoteID] = idUseCount
                    newChild = addChild(newLink,
                                        linkChild.qname,
                                        attributes=attributes)
                    xmlLang = linkChild.xmlLang
                    if xmlLang is not None and xmlLang != defaultXmlLang:  # default
                        newChild.set(
                            "{http://www.w3.org/XML/1998/namespace}lang",
                            xmlLang)
                    copyIxFootnoteHtml(
                        linkChild,
                        newChild,
                        targetModelDocument=targetInstance.modelDocument,
                        withText=True)

                    if filingFiles and linkChild.textValue:
                        footnoteHtml = XML("<body/>")
                        copyIxFootnoteHtml(linkChild, footnoteHtml)
                        for elt in footnoteHtml.iter():
                            addLocallyReferencedFile(elt, filingFiles)
    return targetInstance
Example #42
0
def validate(logMessage, sphinxContext):
    modelXbrl = sphinxContext.modelXbrl
    hasDTS = modelXbrl is not None
    
    
    if hasDTS:
        # if no formulas loaded, set
        if not hasattr(modelXbrl, "modelFormulaEqualityDefinitions"):
            modelXbrl.modelFormulaEqualityDefinitions = {}
    
        import logging
        initialErrorCount = modelXbrl.logCount.get(logging.getLevelName('ERROR'), 0)

        # must also have default dimensions loaded
        from arelle import ValidateXbrlDimensions
        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
        
    sphinxContext.ruleBasePreconditionNodes = []
    sphinxContext.preconditionNodes = {}
    
    # accumulate definitions
    for prog in sphinxContext.sphinxProgs:
        for node in prog:
            if isinstance(node, astRuleBasePrecondition):
                sphinxContext.ruleBasePreconditionNodes.append(node)
            elif isinstance(node, astPreconditionDeclaration):
                sphinxContext.preconditionNodes[node.name] = node    
            elif isinstance(node, astFunctionDeclaration):
                sphinxContext.functions[node.name] = node
            elif isinstance(node, astConstant):
                sphinxContext.constants[node.constantName] = node
                node.value = None   # compute dynamically on first reference
                if node.tagName:
                    sphinxContext.taggedConstants[node.tagName] = node

    # check references            
    def checkNodes(nodes, inMacro=False):
        if not nodes: return
        for node in nodes:
            if node is None:
                continue
            elif isinstance(node, (list,set)):
                checkNodes(node, inMacro)
            elif isinstance(node, astPreconditionReference):
                for name in node.names:
                    if name not in sphinxContext.preconditionNodes:
                        logMessage("ERROR", "sphinxCompiler:preconditionReferenceI",
                            _("Precondition reference is not defined %(name)s"),
                            sourceFileLine=node.sourceFileLine,
                            name=name)
            elif isinstance(node, (astFormulaRule, astReportRule, astValidationRule)):
                checkNodes((node.precondition, node.severity, 
                            node.variableAssignments, 
                            node.expr, node.message), inMacro)
                sphinxContext.rules.append(node)
                if node.severity:
                    severity = node.severity
                    if isinstance(severity, astFunctionReference):
                        severity = severity.name
                    if (severity not in ("error", "warning", "info") and
                        (isinstance(node, astFormulaRule) and severity not in sphinxContext.functions)):
                        logMessage("ERROR", "sphinxCompiler:ruleSeverity",
                            _("Rule %(name)s severity is not recognized: %(severity)s"),
                            sourceFileLine=node.sourceFileLine,
                            name=node.name,
                            severity=node.severity)
                if isinstance(node, astFormulaRule) and not hasFormulaOp(node):
                    logMessage("ERROR", "sphinxCompiler:formulaSyntax",
                        _("Formula %(name)s missing \":=\" operation"),
                        sourceFileLine=node.sourceFileLine,
                        name=node.name)
            elif isinstance(node, astHyperspaceExpression) and hasDTS:
                # check axes
                for axis in node.axes:
                    if isinstance(axis.aspect, QName):
                        concept = modelXbrl.qnameConcepts.get(axis)
                        if concept is None or not concept.isDimensionItem:
                            logMessage("ERROR", "sphinxCompiler:axisNotDimension",
                                _("Axis is not a dimension in the DTS %(qname)s"),
                                sourceFileLine=node.sourceFileLine,
                                qname=axis)
                        elif axis not in sphinxContext.dimensionIsExplicit:
                            sphinxContext.dimensionIsExplicit[axis] = concept.isExplicitDimension
                    elif isinstance(axis.aspect, astNode):
                        if not inMacro:
                            logMessage("ERROR", "sphinxCompiler:axisDisallowed",
                                _("Hypercube axis aspect not static %(aspect)s"),
                                sourceFileLine=node.sourceFileLine,
                                aspect=axis.aspect)
                    elif (axis.aspect not in {"unit", "segment", "scenario"} and
                        isinstance(axis.restriction, (list, tuple))):
                        for restrictionValue in axis.restriction:
                            if isinstance(restrictionValue, QName) and not restrictionValue in modelXbrl.qnameConcepts: 
                                logMessage("ERROR", "sphinxCompiler:axisNotDimension",
                                    _("Hypercube value not in the DTS %(qname)s"),
                                    sourceFileLine=node.sourceFileLine,
                                    qname=restrictionValue)
                        checkNodes((axis.whereExpr,), inMacro)
            elif isinstance(node, astWith):
                node.axes = {}
                def checkWithAxes(withNode):
                    if isinstance(withNode, astHyperspaceExpression):
                        checkNodes((withNode,), inMacro)
                        node.axes.update(withNode.axes)
                    else:
                        logMessage("ERROR", "sphinxCompiler:withRestrictionError",
                            _("With restriction is not a single hyperspace expression"),
                            sourceFileLine=withNode.sourceFileLine)
                checkWithAxes(node.restrictionExpr)
                checkNodes((node.variableAssignments, node.bodyExpr,), inMacro)
            elif isinstance(node, astNode):
                nestedMacro = inMacro or (isinstance(node, astFunctionDeclaration) and 
                                          node.functionType == "macro")
                checkNodes([expr
                            for expr in node.__dict__.values() 
                            if isinstance(expr, (astNode, list, set))],
                           nestedMacro)
                
    for prog in sphinxContext.sphinxProgs:
        checkNodes(prog)
                    
    if len(sphinxContext.ruleBasePreconditionNodes) > 1:
        logMessage("ERROR", "sphinxCompiler:multipleRuleBaseDeclarations",
            _("Multiple rule-base declarations %(preconditions)s"),
            sourceFileLines=[node.sourceFileLine for node in sphinxContext.ruleBasePreconditionNodes],
            preconditions=", ".join(str(r) for r in sphinxContext.ruleBasePreconditionNodes)) 
        
               
    if hasDTS:
        # if no errors in checking sphinx
        if initialErrorCount == modelXbrl.logCount.get(logging.getLevelName('ERROR'), 0):
            from .SphinxEvaluator import evaluateRuleBase
            evaluateRuleBase(sphinxContext)
Example #43
0
def init(modelXbrl):
    # setup modelXbrl for rendering evaluation

    # dimension defaults required in advance of validation
    from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument
    ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
    
    hasXbrlTables = False
    
    # validate table linkbase dimensions
    for baseSetKey in modelXbrl.baseSets.keys():
        arcrole, ELR, linkqname, arcqname = baseSetKey
        if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(arcrole):
            ValidateFormula.checkBaseSet(modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname))
            if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011):
                hasXbrlTables = True

    # provide context for view
    if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE:
        instance = None # use instance of the entry pont
    else: # need dummy instance
        instance = ModelDocument.create(modelXbrl, ModelDocument.Type.INSTANCE, 
                                        "dummy.xml",  # fake URI and fake schemaRef 
                                        ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd",))
        
    if hasXbrlTables:
        # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables
        modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance)
        
        modelXbrl.profileStat(None)
        
        # setup fresh parameters from formula options
        modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters()
        
        # validate parameters and custom function signatures
        ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables"))
        
        # deprecated as of 2013-05-17
        # check and extract message expressions into compilable programs
        for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301, XbrlConst.tableDefinitionNodeSelectionMessage201301,
                           XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011):
            for msgRel in modelXbrl.relationshipSet(msgArcrole).modelRelationships:
                ValidateFormula.checkMessageExpressions(modelXbrl, msgRel.toModelObject)
                
        # compile and validate tables
        for modelTable in modelXbrl.modelRenderingTables:
            modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining
            modelTable.compile()

            hasNsWithAspectModel = modelTable.namespaceURI in (XbrlConst.euRend, XbrlConst.table2011, XbrlConst.table201301, XbrlConst.table201305) 
        
            # check aspectModel  (attribute removed 2013-06, now always dimensional)
            if modelTable.aspectModel not in ("non-dimensional", "dimensional") and hasNsWithAspectModel:
                modelXbrl.error("xbrlte:unknownAspectModel",
                    _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"),
                    modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel)
            else:
                modelTable.priorAspectAxisDisposition = {}
                # check ordinate aspects against aspectModel
                oppositeAspectModel = (_DICT_SET({'dimensional','non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop()
                if hasNsWithAspectModel:
                    uncoverableAspects = aspectModels[oppositeAspectModel] - aspectModels[modelTable.aspectModel]
                else:
                    uncoverableAspects = ()
                aspectsCovered = set()
                for tblAxisRel in modelXbrl.relationshipSet((XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301,XbrlConst.tableAxis2011)).fromModelObject(modelTable):
                    breakdownAspectsCovered = set()
                    hasCoveredAspect = checkBreakdownDefinitionNode(modelXbrl, modelTable, tblAxisRel, tblAxisRel.axisDisposition, uncoverableAspects, breakdownAspectsCovered)
                    ''' removed 2013-10
                    if not hasCoveredAspect:
                        definitionNode = tblAxisRel.toModelObject
                        modelXbrl.error("xbrlte:breakdownDefinesNoAspects",
                            _("Breakdown %(xlinkLabel)s has no participating aspects"),
                            modelObject=(modelTable,definitionNode), xlinkLabel=definitionNode.xlinkLabel, axis=definitionNode.localName)
                    '''
                    aspectsCovered |= breakdownAspectsCovered
                    checkBreakdownLeafNodeAspects(modelXbrl, modelTable, tblAxisRel, set(), breakdownAspectsCovered)
                if Aspect.CONCEPT not in aspectsCovered and not hasNsWithAspectModel:
                    modelXbrl.error("xbrlte:tableMissingConceptAspect",
                        _("Table %(xlinkLabel)s does not include the concept aspect as one of its participating aspects"),
                        modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel)
                del modelTable.priorAspectAxisDisposition
                # check for table-parameter name clash
                parameterNames = {}
                for tblParamRel in modelXbrl.relationshipSet((XbrlConst.tableParameter, XbrlConst.tableParameterMMDD)).fromModelObject(modelTable):
                    parameterName = tblParamRel.variableQname
                    if parameterName in parameterNames:
                        modelXbrl.error("xbrlte:tableParameterNameClash ",
                            _("Table %(xlinkLabel)s has parameter name clash for variable %(name)s"),
                            modelObject=(modelTable,tblParamRel,parameterNames[parameterName]), xlinkLabel=modelTable.xlinkLabel, name=parameterName)
                    else:
                        parameterNames[parameterName] = tblParamRel
    
        modelXbrl.profileStat(_("compileTables"))
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None):
    targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
    targetUrlParts = targetUrl.rpartition(".")
    targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
    modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
    targetInstance = ModelXbrl.create(modelXbrl.modelManager, 
                                      newDocumentType=Type.INSTANCE,
                                      url=targetUrl,
                                      schemaRefs=targetDocumentSchemaRefs,
                                      isEntry=True)
    ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults 
    # roleRef and arcroleRef (of each inline document)
    for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
        for roleRefElt in sourceRefs.values():
            addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, 
                     attributes=roleRefElt.items())
    
    # contexts
    for context in modelXbrl.contexts.values():
        newCntx = targetInstance.createContext(context.entityIdentifier[0],
                                               context.entityIdentifier[1],
                                               'instant' if context.isInstantPeriod else
                                               'duration' if context.isStartEndPeriod
                                               else 'forever',
                                               context.startDatetime,
                                               context.endDatetime,
                                               None, 
                                               context.qnameDims, [], [],
                                               id=context.id)
    for unit in modelXbrl.units.values():
        measures = unit.measures
        newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id)

    modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
    newFactForOldObjId = {}
    def createFacts(facts, parent):
        for fact in facts:
            if fact.isItem:
                attrs = {"contextRef": fact.contextID}
                if fact.id:
                    attrs["id"] = fact.id
                if fact.isNumeric:
                    attrs["unitRef"] = fact.unitID
                    if fact.get("decimals"):
                        attrs["decimals"] = fact.get("decimals")
                    if fact.get("precision"):
                        attrs["precision"] = fact.get("precision")
                if fact.isNil:
                    attrs[XbrlConst.qnXsiNil] = "true"
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.textValue
                newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newFact
                if filingFiles and fact.concept is not None and fact.concept.isTextBlock:
                    # check for img and other filing references
                    for xmltext in [text] + CDATApattern.findall(text):
                        try:
                            for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)):
                                if elt.tag in ("a", "img") and not isHttpUrl(attrValue) and not os.path.isabs(attrvalue):
                                    for attrTag, attrValue in elt.items():
                                        if attrTag in ("href", "src"):
                                            filingFiles.add(attrValue)
                        except (XMLSyntaxError, UnicodeDecodeError):
                            pass
            elif fact.isTuple:
                newTuple = targetInstance.createFact(fact.qname, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newTuple
                createFacts(fact.modelTupleFacts, newTuple)
                
    createFacts(modelXbrl.facts, None)
    # footnote links
    footnoteIdCount = {}
    modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships"))
    HREF = "{http://www.w3.org/1999/xlink}href"
    footnoteLinks = defaultdict(list)
    for linkKey, linkPrototypes in modelXbrl.baseSets.items():
        arcrole, linkrole, linkqname, arcqname = linkKey
        if (linkrole and linkqname and arcqname and # fully specified roles
            arcrole != "XBRL-footnotes" and
            any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
            for linkPrototype in linkPrototypes:
                if linkPrototype not in footnoteLinks[linkrole]:
                    footnoteLinks[linkrole].append(linkPrototype)
    for linkrole in sorted(footnoteLinks.keys()):
        for linkPrototype in footnoteLinks[linkrole]:
            newLink = addChild(targetInstance.modelDocument.xmlRootElement, 
                               linkPrototype.qname, 
                               attributes=linkPrototype.attributes)
            for linkChild in linkPrototype:
                attributes = linkChild.attributes
                if isinstance(linkChild, LocPrototype):
                    if HREF not in linkChild.attributes:
                        linkChild.attributes[HREF] = \
                        "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
                    addChild(newLink, linkChild.qname, 
                             attributes=attributes)
                elif isinstance(linkChild, ArcPrototype):
                    addChild(newLink, linkChild.qname, attributes=attributes)
                elif isinstance(linkChild, ModelInlineFootnote):
                    idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
                    if idUseCount > 1: # if footnote with id in other links bump the id number
                        attributes = linkChild.attributes.copy()
                        attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
                    footnoteIdCount[linkChild.footnoteID] = idUseCount
                    newChild = addChild(newLink, linkChild.qname, 
                                        attributes=attributes)
                    copyIxFootnoteHtml(linkChild, newChild, withText=True)
                    if filingFiles and linkChild.textValue:
                        footnoteHtml = XML("<body/>")
                        copyIxFootnoteHtml(linkChild, footnoteHtml)
                        for elt in footnoteHtml.iter():
                            if elt.tag in ("a", "img"):
                                for attrTag, attrValue in elt.items():
                                    if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrvalue):
                                        filingFiles.add(attrValue)
        
    targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip)
    modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
Example #45
0
def init(modelXbrl):
    # setup modelXbrl for rendering evaluation

    # dimension defaults required in advance of validation
    from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument
    ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)

    hasXbrlTables = False

    # validate table linkbase dimensions
    for baseSetKey in modelXbrl.baseSets.keys():
        arcrole, ELR, linkqname, arcqname = baseSetKey
        if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(
                arcrole):
            ValidateFormula.checkBaseSet(
                modelXbrl, arcrole, ELR,
                modelXbrl.relationshipSet(arcrole, ELR, linkqname, arcqname))
            if arcrole in (XbrlConst.tableBreakdown,
                           XbrlConst.tableBreakdownMMDD,
                           XbrlConst.tableBreakdown201305,
                           XbrlConst.tableBreakdown201301,
                           XbrlConst.tableAxis2011):
                hasXbrlTables = True

    # provide context for view
    if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE:
        instance = None  # use instance of the entry pont
    else:  # need dummy instance
        instance = ModelDocument.create(
            modelXbrl,
            ModelDocument.Type.INSTANCE,
            "dummy.xml",  # fake URI and fake schemaRef 
            ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd", ))

    if hasXbrlTables:
        # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables
        modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance)

        modelXbrl.profileStat(None)

        # setup fresh parameters from formula options
        modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters(
        )

        # validate parameters and custom function signatures
        ValidateFormula.validate(modelXbrl,
                                 xpathContext=modelXbrl.rendrCntx,
                                 parametersOnly=True,
                                 statusMsg=_("compiling rendering tables"))

        # deprecated as of 2013-05-17
        # check and extract message expressions into compilable programs
        for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301,
                           XbrlConst.tableDefinitionNodeSelectionMessage201301,
                           XbrlConst.tableAxisMessage2011,
                           XbrlConst.tableAxisSelectionMessage2011):
            for msgRel in modelXbrl.relationshipSet(
                    msgArcrole).modelRelationships:
                ValidateFormula.checkMessageExpressions(
                    modelXbrl, msgRel.toModelObject)

        # compile and validate tables
        for modelTable in modelXbrl.modelRenderingTables:
            modelTable.fromInstanceQnames = None  # required if referred to by variables scope chaining
            modelTable.compile()

            hasNsWithAspectModel = modelTable.namespaceURI in (
                XbrlConst.euRend, XbrlConst.table2011, XbrlConst.table201301,
                XbrlConst.table201305)

            # check aspectModel  (attribute removed 2013-06, now always dimensional)
            if modelTable.aspectModel not in (
                    "non-dimensional", "dimensional") and hasNsWithAspectModel:
                modelXbrl.error(
                    "xbrlte:unknownAspectModel",
                    _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"
                      ),
                    modelObject=modelTable,
                    xlinkLabel=modelTable.xlinkLabel,
                    aspectModel=modelTable.aspectModel)
            else:
                modelTable.priorAspectAxisDisposition = {}
                # check ordinate aspects against aspectModel
                oppositeAspectModel = (
                    _DICT_SET({'dimensional', 'non-dimensional'}) -
                    _DICT_SET({modelTable.aspectModel})).pop()
                if hasNsWithAspectModel:
                    uncoverableAspects = aspectModels[
                        oppositeAspectModel] - aspectModels[
                            modelTable.aspectModel]
                else:
                    uncoverableAspects = ()
                aspectsCovered = set()
                for tblAxisRel in modelXbrl.relationshipSet(
                    (XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD,
                     XbrlConst.tableBreakdown201305,
                     XbrlConst.tableBreakdown201301,
                     XbrlConst.tableAxis2011)).fromModelObject(modelTable):
                    breakdownAspectsCovered = set()
                    hasCoveredAspect = checkBreakdownDefinitionNode(
                        modelXbrl, modelTable, tblAxisRel,
                        tblAxisRel.axisDisposition, uncoverableAspects,
                        breakdownAspectsCovered)
                    ''' removed 2013-10
                    if not hasCoveredAspect:
                        definitionNode = tblAxisRel.toModelObject
                        modelXbrl.error("xbrlte:breakdownDefinesNoAspects",
                            _("Breakdown %(xlinkLabel)s has no participating aspects"),
                            modelObject=(modelTable,definitionNode), xlinkLabel=definitionNode.xlinkLabel, axis=definitionNode.localName)
                    '''
                    aspectsCovered |= breakdownAspectsCovered
                    checkBreakdownLeafNodeAspects(modelXbrl, modelTable,
                                                  tblAxisRel, set(),
                                                  breakdownAspectsCovered)
                if Aspect.CONCEPT not in aspectsCovered and not hasNsWithAspectModel:
                    modelXbrl.error(
                        "xbrlte:tableMissingConceptAspect",
                        _("Table %(xlinkLabel)s does not include the concept aspect as one of its participating aspects"
                          ),
                        modelObject=modelTable,
                        xlinkLabel=modelTable.xlinkLabel)
                del modelTable.priorAspectAxisDisposition
                # check for table-parameter name clash
                parameterNames = {}
                for tblParamRel in modelXbrl.relationshipSet(
                    (XbrlConst.tableParameter, XbrlConst.tableParameterMMDD
                     )).fromModelObject(modelTable):
                    parameterName = tblParamRel.variableQname
                    if parameterName in parameterNames:
                        modelXbrl.error(
                            "xbrlte:tableParameterNameClash ",
                            _("Table %(xlinkLabel)s has parameter name clash for variable %(name)s"
                              ),
                            modelObject=(modelTable, tblParamRel,
                                         parameterNames[parameterName]),
                            xlinkLabel=modelTable.xlinkLabel,
                            name=parameterName)
                    else:
                        parameterNames[parameterName] = tblParamRel

        modelXbrl.profileStat(_("compileTables"))
Example #46
0
def saveTargetDocument(modelXbrl, targetDocumentFilename,
                       targetDocumentSchemaRefs):
    targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
        targetDocumentFilename, modelXbrl.modelDocument.filepath)
    targetUrlParts = targetUrl.rpartition(".")
    targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
    modelXbrl.modelManager.showStatus(
        _("Extracting instance ") + os.path.basename(targetUrl))
    targetInstance = ModelXbrl.create(modelXbrl.modelManager,
                                      newDocumentType=Type.INSTANCE,
                                      url=targetUrl,
                                      schemaRefs=targetDocumentSchemaRefs,
                                      isEntry=True)
    ValidateXbrlDimensions.loadDimensionDefaults(
        targetInstance)  # need dimension defaults
    # roleRef and arcroleRef (of each inline document)
    for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
        for roleRefElt in sourceRefs.values():
            XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement,
                             roleRefElt.qname,
                             attributes=roleRefElt.items())

    # contexts
    for context in modelXbrl.contexts.values():
        newCntx = targetInstance.createContext(
            context.entityIdentifier[0],
            context.entityIdentifier[1],
            'instant' if context.isInstantPeriod else
            'duration' if context.isStartEndPeriod else 'forever',
            context.startDatetime,
            context.endDatetime,
            None,
            context.qnameDims, [], [],
            id=context.id)
    for unit in modelXbrl.units.values():
        measures = unit.measures
        newUnit = targetInstance.createUnit(measures[0],
                                            measures[1],
                                            id=unit.id)

    modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
    newFactForOldObjId = {}

    def createFacts(facts, parent):
        for fact in modelXbrl.facts:
            if fact.isItem:
                attrs = {"contextRef": fact.contextID}
                if fact.isNumeric:
                    attrs["unitRef"] = fact.unitID
                    if fact.get("decimals"):
                        attrs["decimals"] = fact.get("decimals")
                    if fact.get("precision"):
                        attrs["precision"] = fact.get("precision")
                if fact.isNil:
                    attrs[XbrlConst.qnXsiNil] = "true"
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.textValue
                newFact = targetInstance.createFact(fact.qname,
                                                    attributes=attrs,
                                                    text=text,
                                                    parent=parent)
                newFactForOldObjId[fact.objectIndex] = newFact
            elif fact.isTuple:
                newTuple = targetInstance.createFact(fact.qname, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newTuple
                createFacts(fact.modelTupleFacts, newTuple)

    createFacts(modelXbrl.facts, None)
    # footnote links
    modelXbrl.modelManager.showStatus(
        _("Creating and validating footnotes & relationships"))
    for linkKey, linkPrototypes in modelXbrl.baseSets.items():
        arcrole, linkrole, linkqname, arcqname = linkKey
        if (linkrole and linkqname and arcqname and  # fully specified roles
                any(lP.modelDocument.type == Type.INLINEXBRL
                    for lP in linkPrototypes)):
            for linkPrototype in linkPrototypes:
                newLink = XmlUtil.addChild(
                    targetInstance.modelDocument.xmlRootElement,
                    linkqname,
                    attributes=linkPrototype.attributes)
                for linkChild in linkPrototype:
                    if isinstance(
                            linkChild, LocPrototype
                    ) and "{http://www.w3.org/1999/xlink}href" not in linkChild.attributes:
                        linkChild.attributes["{http://www.w3.org/1999/xlink}href"] = \
                        "#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
                    XmlUtil.addChild(newLink,
                                     linkChild.qname,
                                     attributes=linkChild.attributes,
                                     text=linkChild.textValue)

    targetInstance.saveInstance(overrideFilepath=targetUrl)
    modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
Example #47
0
def init(modelXbrl):
    # setup modelXbrl for rendering evaluation

    # dimension defaults required in advance of validation
    from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument
    ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)

    hasXbrlTables = False

    # validate table linkbase dimensions
    for baseSetKey in modelXbrl.baseSets.keys():
        arcrole, ELR, linkqname, arcqname = baseSetKey
        if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(
                arcrole):
            ValidateFormula.checkBaseSet(
                modelXbrl, arcrole, ELR,
                modelXbrl.relationshipSet(arcrole, ELR, linkqname, arcqname))
            if arcrole in (XbrlConst.tableBreakdown,
                           XbrlConst.tableBreakdown201301,
                           XbrlConst.tableAxis2011):
                hasXbrlTables = True

    # provide context for view
    if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE:
        instance = None  # use instance of the entry pont
    else:  # need dummy instance
        instance = ModelDocument.create(
            modelXbrl,
            ModelDocument.Type.INSTANCE,
            "dummy.xml",  # fake URI and fake schemaRef 
            ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd", ))

    if hasXbrlTables:
        # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables
        modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance)

        modelXbrl.profileStat(None)

        # setup fresh parameters from formula optoins
        modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters(
        )

        # validate parameters and custom function signatures
        ValidateFormula.validate(modelXbrl,
                                 xpathContext=modelXbrl.rendrCntx,
                                 parametersOnly=True,
                                 statusMsg=_("compiling rendering tables"))

        # deprecated as of 2013-05-17
        # check and extract message expressions into compilable programs
        for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301,
                           XbrlConst.tableDefinitionNodeSelectionMessage201301,
                           XbrlConst.tableAxisMessage2011,
                           XbrlConst.tableAxisSelectionMessage2011):
            for msgRel in modelXbrl.relationshipSet(
                    msgArcrole).modelRelationships:
                ValidateFormula.checkMessageExpressions(
                    modelXbrl, msgRel.toModelObject)

        # compile and validate tables
        for modelTable in modelXbrl.modelRenderingTables:
            modelTable.fromInstanceQnames = None  # required if referred to by variables scope chaining
            modelTable.compile()

            # check aspectModel
            if modelTable.aspectModel not in ("non-dimensional",
                                              "dimensional"):
                modelXbrl.error(
                    "xbrlte:unknownAspectModel",
                    _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"
                      ),
                    modelObject=modelTable,
                    xlinkLabel=modelTable.xlinkLabel,
                    aspectModel=modelTable.aspectModel)
            else:
                modelTable.priorAspectAxisDisposition = {}
                # check ordinate aspects against aspectModel
                oppositeAspectModel = (
                    _DICT_SET({'dimensional', 'non-dimensional'}) -
                    _DICT_SET({modelTable.aspectModel})).pop()
                uncoverableAspects = aspectModels[
                    oppositeAspectModel] - aspectModels[modelTable.aspectModel]
                for tblAxisRel in modelXbrl.relationshipSet(
                    (XbrlConst.tableBreakdown, XbrlConst.tableBreakdown201301,
                     XbrlConst.tableAxis2011)).fromModelObject(modelTable):
                    checkDefinitionNodeAspectModel(modelXbrl, modelTable,
                                                   tblAxisRel,
                                                   uncoverableAspects)
                del modelTable.priorAspectAxisDisposition

        modelXbrl.profileStat(_("compileTables"))
Example #48
0
    def validate(self, modelXbrl, parameters=None):
        self.parameters = parameters
        self.NCnamePattern = re.compile(
            "^[_A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]"
            r"[_\-\."
            "\xB7A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040]*$"
        )
        self.precisionPattern = re.compile("^([0-9]+|INF)$")
        self.decimalsPattern = re.compile("^(-?[0-9]+|INF)$")
        self.isoCurrencyPattern = re.compile(r"^[A-Z]{3}$")
        self.modelXbrl = modelXbrl
        self.validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
        self.disclosureSystem = modelXbrl.modelManager.disclosureSystem
        self.validateEFM = self.validateDisclosureSystem and self.disclosureSystem.EFM
        self.validateGFM = self.validateDisclosureSystem and self.disclosureSystem.GFM
        self.validateEFMorGFM = self.validateDisclosureSystem and self.disclosureSystem.EFMorGFM
        self.validateHMRC = self.validateDisclosureSystem and self.disclosureSystem.HMRC
        self.validateSBRNL = self.validateDisclosureSystem and self.disclosureSystem.SBRNL
        self.validateXmlLang = self.validateDisclosureSystem and self.disclosureSystem.xmlLangPattern
        self.validateCalcLB = modelXbrl.modelManager.validateCalcLB
        self.validateInferDecimals = modelXbrl.modelManager.validateInferDecimals

        # xlink validation
        modelXbrl.modelManager.showStatus(_("validating links"))
        modelLinks = set()
        self.remoteResourceLocElements = set()
        self.genericArcArcroles = set()
        for baseSetExtLinks in modelXbrl.baseSets.values():
            for baseSetExtLink in baseSetExtLinks:
                modelLinks.add(
                    baseSetExtLink)  # ext links are unique (no dups)
        for modelLink in modelLinks:
            fromToArcs = {}
            locLabels = {}
            resourceLabels = {}
            resourceArcTos = []
            for arcElt in modelLink.element.childNodes:
                if arcElt.nodeType == 1:
                    xlinkType = arcElt.getAttributeNS(XbrlConst.xlink, "type")
                    # locator must have an href
                    if xlinkType == "locator":
                        if not arcElt.hasAttributeNS(XbrlConst.xlink, "href"):
                            modelXbrl.error(
                                _("Linkbase {0} extended link {1} locator {2} missing href"
                                  ).format(
                                      modelLink.modelDocument.basename,
                                      modelLink.role,
                                      arcElt.hasAttributeNS(
                                          XbrlConst.xlink, "label")), "err",
                                "xlink:locatorHref")
                        locLabels[arcElt.getAttributeNS(
                            XbrlConst.xlink, "label")] = arcElt
                    elif xlinkType == "resource":
                        resourceLabels[arcElt.getAttributeNS(
                            XbrlConst.xlink, "label")] = arcElt
                    # can be no duplicated arcs between same from and to
                    elif xlinkType == "arc":
                        fromLabel = arcElt.getAttributeNS(
                            XbrlConst.xlink, "from")
                        toLabel = arcElt.getAttributeNS(XbrlConst.xlink, "to")
                        fromTo = (fromLabel, toLabel)
                        if fromTo in fromToArcs:
                            modelXbrl.error(
                                _("Linkbase {0} extended link {1} duplicate arcs from {2} to {3}"
                                  ).format(modelLink.modelDocument.basename,
                                           modelLink.role, fromLabel, toLabel),
                                "err", "xlink:dupArcs")
                        else:
                            fromToArcs[fromTo] = arcElt
                        if arcElt.namespaceURI == XbrlConst.link:
                            if arcElt.localName in arcNamesTo21Resource:  #("labelArc","referenceArc"):
                                resourceArcTos.append(
                                    (toLabel, arcElt.getAttribute("use")))
                        elif self.isGenericArc(arcElt):
                            arcrole = arcElt.getAttributeNS(
                                XbrlConst.xlink, "arcrole")
                            self.genericArcArcroles.add(arcrole)
                            if arcrole in (XbrlConst.elementLabel,
                                           XbrlConst.elementReference):
                                resourceArcTos.append((toLabel, arcrole))
                    # values of type (not needed for validating parsers)
                    if xlinkType not in xlinkTypeValues:  # ("", "simple", "extended", "locator", "arc", "resource", "title", "none"):
                        modelXbrl.error(
                            _("Linkbase {0} extended link {1} type {2} invalid"
                              ).format(modelLink.modelDocument.basename,
                                       modelLink.role, xlinkType), "err",
                            "xlink:type")
                    # values of actuate (not needed for validating parsers)
                    xlinkActuate = arcElt.getAttributeNS(
                        XbrlConst.xlink, "actuate")
                    if xlinkActuate not in xlinkActuateValues:  # ("", "onLoad", "onRequest", "other", "none"):
                        modelXbrl.error(
                            _("Linkbase {0} extended link {1} actuate {2} invalid"
                              ).format(modelLink.modelDocument.basename,
                                       modelLink.role, xlinkActuate), "err",
                            "xlink:actuate")
                    # values of show (not needed for validating parsers)
                    xlinkShow = arcElt.getAttributeNS(XbrlConst.xlink, "show")
                    if xlinkShow not in xlinkShowValues:  # ("", "new", "replace", "embed", "other", "none"):
                        modelXbrl.error(
                            _("Linkbase {0} extended link {1} show {2} invalid"
                              ).format(modelLink.modelDocument.basename,
                                       modelLink.role, xlinkShow), "err",
                            "xlink:show")
                    # values of label, from, to (not needed for validating parsers)
                    for name in xlinkLabelAttributes:  # ("label", "from", "to"):
                        value = arcElt.getAttributeNS(XbrlConst.xlink, name)
                        if value != "" and not self.NCnamePattern.match(value):
                            modelXbrl.error(
                                _("Linkbase {0} extended link {1} element {2} {3} '{4}' not an NCname"
                                  ).format(modelLink.modelDocument.basename,
                                           modelLink.role, arcElt.tagName,
                                           name, value), "err",
                                "xlink:{0}".format(name))
            # check from, to of arcs have a resource or loc
            for fromTo, arcElt in fromToArcs.items():
                fromLabel, toLabel in fromTo
                for name, value, sect in (("from", fromLabel, "3.5.3.9.2"),
                                          ("to", toLabel, "3.5.3.9.3")):
                    if value not in locLabels and value not in resourceLabels:
                        modelXbrl.error(
                            _("Arc in linkbase {0} extended link {1} from {2} to {3} attribute \"{4}\" has no matching loc or resource label"
                              ).format(modelLink.modelDocument.basename,
                                       modelLink.role, fromLabel, toLabel,
                                       name), "err",
                            "xbrl.{0}:arcResource".format(sect))
                if arcElt.localName == "footnoteArc" and arcElt.namespaceURI == XbrlConst.link and \
                   arcElt.getAttributeNS(XbrlConst.xlink,"arcrole") == XbrlConst.factFootnote:
                    if fromLabel not in locLabels:
                        modelXbrl.error(
                            _("FootnoteArc in {0} extended link {1} from {2} to {3} \"from\" is not a loc"
                              ).format(modelLink.modelDocument.basename,
                                       modelLink.role, fromLabel, toLabel),
                            "err", "xbrl.4.11.1.3.1:factFootnoteArcFrom")
                    if toLabel not in resourceLabels or qname(
                            resourceLabels[toLabel]
                    ) != XbrlConst.qnLinkFootnote:
                        modelXbrl.error(
                            _("FootnoteArc in {0} extended link {1} from {2} to {3} \"to\" is not a footnote resource"
                              ).format(modelLink.modelDocument.basename,
                                       modelLink.role, fromLabel, toLabel),
                            "err", "xbrl.4.11.1.3.1:factFootnoteArcTo")
            # check unprohibited label arcs to remote locs
            for resourceArcTo in resourceArcTos:
                resourceArcToLabel, resourceArcUse = resourceArcTo
                if resourceArcToLabel in locLabels:
                    toLabel = locLabels[resourceArcToLabel]
                    if resourceArcUse == "prohibited":
                        self.remoteResourceLocElements.add(toLabel)
                    else:
                        modelXbrl.error(
                            _("Unprohibited labelArc in linkbase {0} extended link {1} has illegal remote resource loc labeled {2} href {3}"
                              ).format(
                                  modelLink.modelDocument.basename,
                                  modelLink.role, resourceArcToLabel,
                                  toLabel.getAttributeNS(
                                      XbrlConst.xlink, "href")), "err",
                            "xbrl.5.2.2.3:labelArcRemoteResource")
                elif resourceArcToLabel in resourceLabels:
                    toResource = resourceLabels[resourceArcToLabel]
                    if resourceArcUse == XbrlConst.elementLabel:
                        if not self.isGenericLabel(toResource):
                            modelXbrl.error(
                                _("Generic label arc in linkbase {0} extended link {1} to {2} must target a generic label"
                                  ).format(modelLink.modelDocument.basename,
                                           modelLink.role, resourceArcToLabel),
                                "err", "xbrlle.2.1.1:genericLabelTarget")
                    elif resourceArcUse == XbrlConst.elementReference:
                        if not self.isGenericReference(toResource):
                            modelXbrl.error(
                                _("Generic reference arc in linkbase {0} extended link {1} to {2} must target a generic reference"
                                  ).format(modelLink.modelDocument.basename,
                                           modelLink.role, resourceArcToLabel),
                                "err", "xbrlre.2.1.1:genericReferenceTarget")

        self.dimensionDefaults = {}
        modelXbrl.qnameDimensionDefaults = {}
        modelXbrl.qnameDimensionContextElement = {}
        # check base set cycles, dimensions
        modelXbrl.modelManager.showStatus(_("validating relationship sets"))
        for baseSetKey in modelXbrl.baseSets.keys():
            arcrole, ELR, linkqname, arcqname = baseSetKey
            if arcrole.startswith("XBRL-") or ELR is None or \
                linkqname is None or arcqname is None:
                continue
            elif arcrole in XbrlConst.standardArcroleCyclesAllowed:
                cyclesAllowed, specSect = XbrlConst.standardArcroleCyclesAllowed[
                    arcrole]
            elif arcrole in self.modelXbrl.arcroleTypes and len(
                    self.modelXbrl.arcroleTypes[arcrole]) > 0:
                cyclesAllowed = self.modelXbrl.arcroleTypes[arcrole][
                    0].cyclesAllowed
                if arcrole in self.genericArcArcroles:
                    specSect = "xbrlgene:violatedCyclesConstraint"
                else:
                    specSect = "xbrl.5.1.4.3:cycles"
            else:
                cyclesAllowed = "any"
                specSect = None
            if cyclesAllowed != "any" or arcrole in (XbrlConst.summationItem,) \
                                      or arcrole in self.genericArcArcroles  \
                                      or arcrole.startswith(XbrlConst.formulaStartsWith):
                relsSet = modelXbrl.relationshipSet(arcrole, ELR, linkqname,
                                                    arcqname)
            if cyclesAllowed != "any" and \
                   (XbrlConst.isStandardExtLinkQname(linkqname) and XbrlConst.isStandardArcQname(arcqname)) \
                   or arcrole in self.genericArcArcroles:
                noUndirected = cyclesAllowed == "none"
                fromRelationships = relsSet.fromModelObjects()
                for relFrom, rels in fromRelationships.items():
                    cycleFound = self.fwdCycle(relsSet, rels, noUndirected,
                                               {relFrom})
                    if cycleFound:
                        modelXbrl.error(
                            _("Relationships have a {0} cycle in arcrole {1} link role {2} link {3}, arc {4} starting from {5}"
                              ).format(cycleFound, arcrole, ELR, linkqname,
                                       arcqname, relFrom.qname), "err",
                            "{0}".format(specSect))
                        break

            # check calculation arcs for weight issues (note calc arc is an "any" cycles)
            if arcrole == XbrlConst.summationItem:
                for modelRel in relsSet.modelRelationships:
                    weight = modelRel.weight
                    fromConcept = modelRel.fromModelObject
                    toConcept = modelRel.toModelObject
                    if fromConcept and toConcept:
                        if weight == 0:
                            modelXbrl.error(
                                _("Calculation relationship has zero weight from {0} to {1} in link role {2}"
                                  ).format(fromConcept.qname, toConcept.qname,
                                           ELR), "err",
                                "xbrl.5.2.5.2.1:zeroWeight")
                        fromBalance = fromConcept.balance
                        toBalance = toConcept.balance
                        if fromBalance and toBalance:
                            if (fromBalance == toBalance and weight < 0) or \
                               (fromBalance != toBalance and weight > 0):
                                modelXbrl.error(
                                    _("Calculation relationship has illegal weight {0} from {1}, {2}, to {3}, {4} in link role {5} (per 5.1.1.2 Table 6)"
                                      ).format(weight, fromConcept.qname,
                                               fromBalance, toConcept.qname,
                                               toBalance, ELR), "err",
                                    "xbrl.5.1.1.2:balanceCalcWeight")
                        if not fromConcept.isNumeric or not toConcept.isNumeric:
                            modelXbrl.error(
                                _("Calculation relationship has illegal concept from {0}{1} to {2}{3} in link role {4}"
                                  ).format(
                                      fromConcept.qname,
                                      "" if fromConcept.isNumeric else
                                      " (non-numeric)", toConcept.qname,
                                      "" if fromConcept.isNumeric else
                                      " (non-numeric)", ELR), "err",
                                "xbrl.5.2.5.2:nonNumericCalc")
            # check presentation relationships for preferredLabel issues
            elif arcrole == XbrlConst.parentChild:
                for modelRel in relsSet.modelRelationships:
                    preferredLabel = modelRel.preferredLabel
                    toConcept = modelRel.toModelObject
                    if preferredLabel and toConcept and \
                       toConcept.label(preferredLabel=preferredLabel,fallbackToQname=False) is None:
                        modelXbrl.error(
                            _("Presentation relationship from {0} to {1} in link role {2} missing preferredLabel {3}"
                              ).format(modelRel.fromModelObject.qname,
                                       toConcept.qname, ELR, preferredLabel),
                            "err", "xbrl.5.2.4.2.1:preferredLabelMissing")
            # check essence-alias relationships
            elif arcrole == XbrlConst.essenceAlias:
                for modelRel in relsSet.modelRelationships:
                    fromConcept = modelRel.fromModelObject
                    toConcept = modelRel.toModelObject
                    if fromConcept and toConcept:
                        if fromConcept.type != toConcept.type or fromConcept.periodType != toConcept.periodType:
                            modelXbrl.error(
                                _("Essence-alias relationship from {0} to {1} in link role {2} has different types or periodTypes"
                                  ).format(fromConcept.qname, toConcept.qname,
                                           ELR), "err",
                                "xbrl.5.2.6.2.2:essenceAliasTypes")
                        fromBalance = fromConcept.balance
                        toBalance = toConcept.balance
                        if fromBalance and toBalance:
                            if fromBalance and toBalance and fromBalance != toBalance:
                                modelXbrl.error(
                                    _("Essence-alias relationship from {0} to {1} in link role {2} has different balances"
                                      ).format(fromConcept.qname,
                                               toConcept.qname, ELR), "err",
                                    "xbrl.5.2.6.2.2:essenceAliasBalance")
            elif modelXbrl.hasXDT and arcrole.startswith(
                    XbrlConst.dimStartsWith):
                ValidateXbrlDimensions.checkBaseSet(self, arcrole, ELR,
                                                    relsSet)
            elif modelXbrl.hasFormulae and arcrole.startswith(
                    XbrlConst.formulaStartsWith):
                ValidateFormula.checkBaseSet(self, arcrole, ELR, relsSet)

        # instance checks
        modelXbrl.modelManager.showStatus(_("validating instance"))
        self.footnoteRefs = set()
        if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE or \
           modelXbrl.modelDocument.type == ModelDocument.Type.INLINEXBRL:
            for f in modelXbrl.facts:
                concept = f.concept
                if concept:
                    if concept.isNumeric:
                        unit = f.unit
                        if f.unitID is None or unit is None:
                            self.modelXbrl.error(
                                _("Fact {0} context {1} is numeric and must have a unit"
                                  ).format(modelXbrl.modelDocument.basename,
                                           f.qname, f.contextID), "err",
                                "xbrl.4.6.2:numericUnit")
                        else:
                            if concept.isMonetary:
                                measures = unit.measures
                                if not measures or len(measures[0]) != 1 or len(measures[1]) != 0 or \
                                    measures[0][0].namespaceURI != XbrlConst.iso4217 or \
                                    not self.isoCurrencyPattern.match(measures[0][0].localName):
                                    self.modelXbrl.error(
                                        _("Fact {0} context {1} must have a monetary unit {2}"
                                          ).format(
                                              modelXbrl.modelDocument.basename,
                                              f.qname, f.contextID, f.unitID),
                                        "err", "xbrl.4.8.2:monetaryFactUnit")
                            elif concept.isShares:
                                measures = unit.measures
                                if not measures or len(measures[0]) != 1 or len(measures[1]) != 0 or \
                                    measures[0][0] != XbrlConst.qnXbrliShares:
                                    self.modelXbrl.error(
                                        _("Fact {0} context {1} must have a xbrli:shares unit {2}"
                                          ).format(f.qname, f.contextID,
                                                   f.unitID), "err",
                                        "xbrl.4.8.2:sharesFactUnit")
                    precision = f.precision
                    hasPrecision = precision is not None
                    if hasPrecision and precision != "INF" and not precision.isdigit(
                    ):
                        self.modelXbrl.error(
                            _("Fact {0} context {1} precision {2} is invalid"
                              ).format(f.qname, f.contextID, precision), "err",
                            "xbrl.4.6.4:precision")
                    decimals = f.decimals
                    hasDecimals = decimals is not None
                    if hasPrecision and not self.precisionPattern.match(
                            precision):
                        self.modelXbrl.error(
                            _("Fact {0} context {1} precision {2} is invalid"
                              ).format(f.qname, f.contextID, precision), "err",
                            "xbrl.4.6.4:precision")
                    if hasPrecision and hasDecimals:
                        self.modelXbrl.error(
                            _("Fact {0} context {1} can not have both precision and decimals"
                              ).format(f.qname, f.contextID), "err",
                            "xbrl.4.6.3:bothPrecisionAndDecimals")
                    if hasDecimals and not self.decimalsPattern.match(
                            decimals):
                        self.modelXbrl.error(
                            _("Fact {0} context {1} decimals {2} is invalid"
                              ).format(f.qname, f.contextID, decimals), "err",
                            "xbrl.4.6.5:decimals")
                    if concept.isItem:
                        context = f.context
                        if context is None:
                            self.modelXbrl.error(
                                _("Item {0} must have a context").format(
                                    f.qname), "err",
                                "xbrl.4.6.1:itemContextRef")
                        else:
                            periodType = concept.periodType
                            if (periodType == "instant" and not context.isInstantPeriod) or \
                               (periodType == "duration" and not (context.isStartEndPeriod or context.isForeverPeriod)):
                                self.modelXbrl.error(
                                    _("Fact {0} context {1} has period type {2} conflict with context"
                                      ).format(f.qname, f.contextID,
                                               periodType), "err",
                                    "xbrl.4.7.2:contextPeriodType")
                            if modelXbrl.hasXDT:
                                ValidateXbrlDimensions.checkFact(self, f)
                        # check precision and decimals
                        if f.xsiNil == "true":
                            if hasPrecision or hasDecimals:
                                self.modelXbrl.error(
                                    _("Fact {0} context {1} can not be nil and have either precision or decimals"
                                      ).format(f.qname, f.contextID), "err",
                                    "xbrl.4.6.3:nilPrecisionDecimals")
                        elif concept.isFraction:
                            if hasPrecision or hasDecimals:
                                self.modelXbrl.error(
                                    _("Fact {0} context {1} is a fraction concept and cannot have either precision or decimals"
                                      ).format(f.qname, f.contextID), "err",
                                    "xbrl.4.6.3:fractionPrecisionDecimals")
                                numerator, denominator = f.fractionValue
                                if not (numerator == "INF"
                                        or numerator.isnumeric()):
                                    self.modelXbrl.error(
                                        _("Fact {0} context {1} is a fraction with invalid numerator {2}"
                                          ).format(f.qname, f.contextID,
                                                   numerator), "err",
                                        "xbrl.5.1.1:fractionPrecisionDecimals")
                                if not denominator.isnumeric() or int(
                                        denominator) == 0:
                                    self.modelXbrl.error(
                                        _("Fact {0} context {1} is a fraction with invalid denominator {2}"
                                          ).format(f.qname, f.contextID,
                                                   denominator), "err",
                                        "xbrl.5.1.1:fractionPrecisionDecimals")
                        else:
                            if modelXbrl.modelDocument.type != ModelDocument.Type.INLINEXBRL:
                                for child in f.element.childNodes:
                                    if child.nodeType == 1:
                                        self.modelXbrl.error(
                                            _("Fact {0} context {1} may not have child elements {2}"
                                              ).format(f.qname, f.contextID,
                                                       child.tagName), "err",
                                            "xbrl.5.1.1:itemMixedContent")
                                        break
                            if concept.isNumeric and not hasPrecision and not hasDecimals:
                                self.modelXbrl.error(
                                    _("Fact {0} context {1} is a numeric concept and must have either precision or decimals"
                                      ).format(f.qname, f.contextID), "err",
                                    "xbrl.4.6.3:missingPrecisionDecimals")
                    elif concept.isTuple:
                        if f.contextID:
                            self.modelXbrl.error(
                                _("Tuple {0} must not have a context").format(
                                    f.qname), "err",
                                "xbrl.4.6.1:tupleContextRef")
                        if hasPrecision or hasDecimals:
                            self.modelXbrl.error(
                                _("Fact {0} is a tuple and cannot have either precision or decimals"
                                  ).format(f.qname), "err",
                                "xbrl.4.6.3:tuplePrecisionDecimals")
                        # custom attributes may be allowed by anyAttribute but not by 2.1
                        for attrQname, attrValue in XbrlUtil.attributes(
                                self.modelXbrl, concept, f.element):
                            if attrQname.namespaceURI in (XbrlConst.xbrli,
                                                          XbrlConst.link,
                                                          XbrlConst.xlink,
                                                          XbrlConst.xl):
                                self.modelXbrl.error(
                                    _("Fact {0} is a tuple and must not have attribute in this namespace {1}"
                                      ).format(f.qname, attrQname), "err",
                                    "xbrl.4.9:tupleAttribute")

                    else:
                        self.modelXbrl.error(
                            _("Fact {0} must be an item or tuple").format(
                                f.qname), "err", "xbrl.4.6:notItemOrTuple")

                from arelle.ModelObject import ModelInlineFact
                if isinstance(f, ModelInlineFact):
                    self.footnoteRefs.update(f.footnoteRefs)

            #instance checks
            for cntx in modelXbrl.contexts.values():
                if cntx.isStartEndPeriod:
                    try:
                        if cntx.endDatetime <= cntx.startDatetime:
                            self.modelXbrl.error(
                                _("Context {0} must have startDate less than endDate"
                                  ).format(cntx.id), "err",
                                "xbrl.4.7.2:periodStartBeforeEnd")
                    except ValueError as err:
                        self.modelXbrl.error(
                            _("Context {0} startDate or endDate: {1}").format(
                                cntx.id, err), "err",
                            "xbrl.4.7.2:contextDateError")
                elif cntx.isInstantPeriod:
                    try:
                        cntx.instantDatetime  #parse field
                    except ValueError as err:
                        self.modelXbrl.error(
                            _("Context {0} instant date: {1}").format(
                                cntx.id, err), "err",
                            "xbrl.4.7.2:contextDateError")
                self.segmentScenario(cntx.segment, cntx.id, "segment",
                                     "4.7.3.2")
                self.segmentScenario(cntx.scenario, cntx.id, "scenario",
                                     "4.7.4")
                if modelXbrl.hasXDT:
                    ValidateXbrlDimensions.checkContext(self, cntx)

            for unit in modelXbrl.units.values():
                mulDivMeasures = unit.measures
                if mulDivMeasures:
                    for measures in mulDivMeasures:
                        for measure in measures:
                            if measure.namespaceURI == XbrlConst.xbrli and not \
                                measure in (XbrlConst.qnXbrliPure, XbrlConst.qnXbrliShares):
                                self.modelXbrl.error(
                                    _("Unit {0} illegal measure: {1}").format(
                                        unit.id, measure), "err",
                                    "xbrl.4.8.2:measureElement")
                    for numeratorMeasure in mulDivMeasures[0]:
                        if numeratorMeasure in mulDivMeasures[1]:
                            self.modelXbrl.error(
                                _("Unit {0} numerator measure: {1} also appears as denominator measure"
                                  ).format(unit.id, numeratorMeasure), "err",
                                "xbrl.4.8.4:measureBothNumDenom")

        #concepts checks
        modelXbrl.modelManager.showStatus(_("validating concepts"))
        for concept in modelXbrl.qnameConcepts.values():
            conceptType = concept.type
            if XbrlConst.isStandardNamespace(concept.namespaceURI) or \
               not concept.modelDocument.inDTS:
                continue

            if concept.isTuple:
                # must be global
                if not concept.element.parentNode.localName == "schema":
                    self.modelXbrl.error(
                        _("Tuple {0} must be declared globally").format(
                            concept.qname), "err",
                        "xbrl.4.9:tupleGloballyDeclared")
                if concept.periodType:
                    self.modelXbrl.error(
                        _("Tuple {0} must not have periodType").format(
                            concept.qname), "err", "xbrl.4.9:tuplePeriodType")
                if concept.balance:
                    self.modelXbrl.error(
                        _("Tuple {0} must not have balance").format(
                            concept.qname), "err", "xbrl.4.9:tupleBalance")
                # check attribute declarations
                for attributeQname in conceptType.attributes:
                    if attributeQname.namespaceURI in (XbrlConst.xbrli,
                                                       XbrlConst.link,
                                                       XbrlConst.xlink,
                                                       XbrlConst.xl):
                        self.modelXbrl.error(
                            _("Tuple {0} must not have attribute in this namespace {1}"
                              ).format(concept.qname, attributeQname), "err",
                            "xbrl.4.9:tupleAttribute")
                # check for mixed="true" or simple content
                if XmlUtil.descendantAttr(conceptType.element, XbrlConst.xsd,
                                          ("complexType", "complexContent"),
                                          "mixed") == "true":
                    self.modelXbrl.error(
                        _("Tuple {0} must not have mixed content").format(
                            concept.qname), "err",
                        "xbrl.4.9:tupleMixedContent")
                if XmlUtil.descendant(conceptType.element, XbrlConst.xsd,
                                      "simpleContent"):
                    self.modelXbrl.error(
                        _("Tuple {0} must not have simple content").format(
                            concept.qname), "err",
                        "xbrl.4.9:tupleSimpleContent")
                # child elements must be item or tuple
                for elementQname in conceptType.elements:
                    childConcept = self.modelXbrl.qnameConcepts.get(
                        elementQname)
                    if childConcept is None:
                        self.modelXbrl.error(
                            _("Tuple {0} element {1} not defined").format(
                                concept.qname, elementQname), "err",
                            "xbrl.4.9:tupleElementUndefined")
                    elif not (
                            childConcept.isItem or childConcept.isTuple
                            or  # isItem/isTuple do not include item or tuple itself
                            childConcept.qname == XbrlConst.qnXbrliItem
                            or  # subs group includes item as member
                            childConcept.qname == XbrlConst.qnXbrliTuple):
                        self.modelXbrl.error(
                            _("Tuple {0} must not have element {1} not an item or tuple"
                              ).format(concept.qname, elementQname), "err",
                            "xbrl.4.9:tupleElementItemOrTuple")
            elif concept.isItem:
                if concept.periodType not in periodTypeValues:  #("instant","duration"):
                    self.modelXbrl.error(
                        _("Item {0} must have a valid periodType").format(
                            concept.qname), "err",
                        "xbrl.5.1.1.1:itemPeriodType")
                if concept.isMonetary:
                    if concept.balance not in balanceValues:  #(None, "credit","debit"):
                        self.modelXbrl.error(
                            _("Item {0} must have a valid balance {1}").format(
                                concept.qname, concept.balance), "err",
                            "xbrl.5.1.1.2:itemBalance")
                else:
                    if concept.balance:
                        self.modelXbrl.error(
                            _("Item {0} may not have a balance").format(
                                concept.qname), "err",
                            "xbrl.5.1.1.2:itemBalance")
                if concept.baseXbrliType not in baseXbrliTypes:
                    self.modelXbrl.error(
                        _("Item {0} type {1} invalid").format(
                            concept.qname, concept.baseXbrliType), "err",
                        "xbrl.5.1.1.3:itemType")
                if modelXbrl.hasXDT:
                    if concept.isHypercubeItem and not concept.abstract == "true":
                        self.modelXbrl.error(
                            _("Hypercube item {0} must be abstract").format(
                                concept.qname), "err",
                            "xbrldte:HypercubeElementIsNotAbstractError")
                    elif concept.isDimensionItem and not concept.abstract == "true":
                        self.modelXbrl.error(
                            _("Dimension item {0} must be abstract").format(
                                concept.qname), "err",
                            "xbrldte:DimensionElementIsNotAbstractError")
            if modelXbrl.hasXDT:
                ValidateXbrlDimensions.checkConcept(self, concept)

        modelXbrl.modelManager.showStatus(_("validating DTS"))
        self.DTSreferenceResourceIDs = {}
        ValidateXbrlDTS.checkDTS(self, modelXbrl.modelDocument, [])
        del self.DTSreferenceResourceIDs

        if self.validateCalcLB:
            modelXbrl.modelManager.showStatus(
                _("Validating instance calculations"))
            ValidateXbrlCalcs.validate(
                modelXbrl, inferPrecision=(not self.validateInferDecimals))

        if modelXbrl.modelManager.validateUtr:
            ValidateUtr.validate(modelXbrl)

        if modelXbrl.hasFormulae:
            ValidateFormula.validate(self)

        modelXbrl.modelManager.showStatus(_("ready"), 2000)