def backgroundProfileFormula(cntlr, profileReportFile, maxRunTime, excludeCompileTime): from arelle import Locale, XPathParser, ValidateXbrlDimensions, ValidateFormula # build grammar before profiling (if this is the first pass, so it doesn't count in profile statistics) XPathParser.initializeParser(cntlr.modelManager) # load dimension defaults ValidateXbrlDimensions.loadDimensionDefaults(cntlr.modelManager) import cProfile, pstats, sys, time # a minimal validation class for formula validator parameters that are needed class Validate: def __init__(self, modelXbrl, maxRunTime): self.modelXbrl = modelXbrl self.parameters = None self.validateSBRNL = False self.maxFormulaRunTime = maxRunTime def close(self): self.__dict__.clear() val = Validate(cntlr.modelManager.modelXbrl, maxRunTime) formulaOptions = val.modelXbrl.modelManager.formulaOptions if excludeCompileTime: startedAt = time.time() cntlr.addToLog(_("pre-compiling formulas before profiling")) val.validateFormulaCompileOnly = True ValidateFormula.validate(val) del val.validateFormulaCompileOnly cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale, _("formula pre-compiling completed in %.2f secs"), time.time() - startedAt)) cntlr.addToLog(_("executing formulas for profiling")) else: cntlr.addToLog(_("compiling and executing formulas for profiling")) startedAt = time.time() statsFile = profileReportFile + ".bin" cProfile.runctx("ValidateFormula.validate(val)", globals(), locals(), statsFile) cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale, _("formula profiling completed in %.2f secs"), time.time() - startedAt)) # dereference val val.close() # specify a file for log priorStdOut = sys.stdout sys.stdout = open(profileReportFile, "w") statObj = pstats.Stats(statsFile) statObj.strip_dirs() statObj.sort_stats("time") statObj.print_stats() statObj.print_callees() statObj.print_callers() sys.stdout.flush() sys.stdout.close() del statObj sys.stdout = priorStdOut os.remove(statsFile)
def insertXbrl(self, rssItem): try: # must also have default dimensions loaded from arelle import ValidateXbrlDimensions ValidateXbrlDimensions.loadDimensionDefaults(self.modelXbrl) # must have a valid XBRL instance or document if self.modelXbrl.modelDocument is None: raise XPDBException("xpgDB:MissingXbrlDocument", _("No XBRL instance or schema loaded for this filing.")) # at this point we determine what's in the database and provide new tables # requires locking most of the table structure self.lockTables(("dAvailableTable", "dInstance", "dFact", "dFilingIndicator", "dProcessingContext", "dProcessingFact")) self.dropTemporaryTable() startedAt = time.time() self.insertInstance() self.insertDataPoints() self.modelXbrl.profileStat(_("XbrlSqlDB: instance insertion"), time.time() - startedAt) startedAt = time.time() self.showStatus("Committing entries") self.commit() self.modelXbrl.profileStat(_("XbrlSqlDB: insertion committed"), time.time() - startedAt) self.showStatus("DB insertion completed", clearAfter=5000) except Exception as ex: self.showStatus("DB insertion failed due to exception", clearAfter=5000) raise
def viewRenderedGrid(modelXbrl, tabWin, lang=None): modelXbrl.modelManager.showStatus(_("viewing rendering")) view = ViewRenderedGrid(modelXbrl, tabWin, lang) # dimension defaults required in advance of validation from arelle import ValidateXbrlDimensions ValidateXbrlDimensions.loadDimensionDefaults(view) # context menu menu = view.contextMenu() optionsMenu = Menu(view.viewFrame, tearoff=0) view.ignoreDimValidity = BooleanVar(value=True) view.ignoreDimValidity.trace("w", view.viewReloadDueToMenuAction) optionsMenu.add_checkbutton(label=_("Ignore Dimensional Validity"), underline=0, variable=view.ignoreDimValidity, onvalue=True, offvalue=False) view.xAxisChildrenFirst = BooleanVar(value=True) view.xAxisChildrenFirst.trace("w", view.viewReloadDueToMenuAction) optionsMenu.add_checkbutton(label=_("X-Axis Children First"), underline=0, variable=view.xAxisChildrenFirst, onvalue=True, offvalue=False) view.yAxisChildrenFirst = BooleanVar(value=False) view.yAxisChildrenFirst.trace("w", view.viewReloadDueToMenuAction) optionsMenu.add_checkbutton(label=_("Y-Axis Children First"), underline=0, variable=view.yAxisChildrenFirst, onvalue=True, offvalue=False) menu.add_cascade(label=_("Options"), menu=optionsMenu, underline=0) view.tablesMenu = Menu(view.viewFrame, tearoff=0) menu.add_cascade(label=_("Tables"), menu=view.tablesMenu, underline=0) view.tablesMenuLength = 0 view.menuAddLangs() view.view() view.blockSelectEvent = 1 view.blockViewModelObject = 0 view.viewFrame.bind("<Enter>", view.cellEnter, '+') view.viewFrame.bind("<Leave>", view.cellLeave, '+')
def insertXbrl(self, rssItem): try: # must also have default dimensions loaded from arelle import ValidateXbrlDimensions ValidateXbrlDimensions.loadDimensionDefaults(self.modelXbrl) # must have a valid XBRL instance or document if self.modelXbrl.modelDocument is None: raise XPDBException("xpgDB:MissingXbrlDocument", _("No XBRL instance or schema loaded for this filing.")) # at this point we determine what's in the database and provide new tables # requires locking most of the table structure self.lockTables(("dAvailableTable", "dInstance", "dFact", "dFilingIndicator", # "dProcessingContext", "dProcessingFact" )) self.dropTemporaryTable() startedAt = time.time() self.insertInstance() self.insertDataPoints() self.modelXbrl.profileStat(_("XbrlSqlDB: instance insertion"), time.time() - startedAt) startedAt = time.time() self.showStatus("Committing entries") self.commit() self.modelXbrl.profileStat(_("XbrlSqlDB: insertion committed"), time.time() - startedAt) self.showStatus("DB insertion completed", clearAfter=5000) except Exception as ex: self.showStatus("DB insertion failed due to exception", clearAfter=5000) raise
def insertXbrl(self, rssItem): try: # must also have default dimensions loaded from arelle import ValidateXbrlDimensions ValidateXbrlDimensions.loadDimensionDefaults(self.modelXbrl) startedAt = time.time() # self.load() this done in the verify step self.insertAccession(rssItem) self.insertDocuments() self.insertDataDictionary() # XML namespaces types aspects #self.insertRelationshipTypeSets() #self.insertResourceRoleSets() #self.insertAspectValues() #self.insertResources() self.modelXbrl.profileStat(_("XbrlPublicDB: DTS insertion"), time.time() - startedAt) startedAt = time.time() self.insertDataPoints() self.modelXbrl.profileStat(_("XbrlPublicDB: data points insertion"), time.time() - startedAt) startedAt = time.time() self.insertRelationshipSets() self.modelXbrl.profileStat(_("XbrlPublicDB: Relationships insertion"), time.time() - startedAt) #startedAt = time.time() #self.insertValidCombinations() #self.modelXbrl.profileStat(_("XbrlPublicDB: Valid Combinations insertion"), time.time() - startedAt) self.showStatus("Committing entries") self.commit() self.modelXbrl.profileStat(_("XbrlPublicDB: insertion committed"), time.time() - startedAt) self.showStatus("DB insertion completed", clearAfter=5000) except Exception as ex: self.showStatus("DB insertion failed due to exception", clearAfter=5000) raise
def start(self, tag, attrib, nsmap=None): mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap) mdlObj.sourceline = 1 if self.newTree: self.newTree = False self.currentMdlObj = mdlObj modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree()) modelXbrl.modelDocument = modelDocument # needed for incremental validation mdlObj.init(modelDocument) modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement modelDocument.parserLookupName = _parserLookupName modelDocument.parserLookupClass = _parserLookupClass modelDocument.xmlRootElement = mdlObj modelDocument.schemaLocationElements.add(mdlObj) modelDocument.documentEncoding = _encoding modelDocument._creationSoftwareComment = creationSoftwareComment modelXbrl.info("streamingExtensions:streaming", _("Stream processing this instance."), modelObject = modelDocument) else: self.currentMdlObj.append(mdlObj) self.currentMdlObj = mdlObj mdlObj._init() ns = mdlObj.namespaceURI ln = mdlObj.localName if (self.beforeInstanceStream and ( (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or (ns == XbrlConst.xbrli and ln in ("context", "unit")) or (ns not in (XbrlConst.link, XbrlConst.xbrli)))): self.beforeInstanceStream = False if _streamingExtensionsValidate: instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters()) else: # need default dimensions ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) return mdlObj
def viewRenderedGrid(modelXbrl, tabWin, lang=None): modelXbrl.modelManager.showStatus(_("viewing rendering")) view = ViewRenderedGrid(modelXbrl, tabWin, lang) # dimension defaults required in advance of validation from arelle import ValidateXbrlDimensions ValidateXbrlDimensions.loadDimensionDefaults(view) # context menu setDefaults(view) menu = view.contextMenu() optionsMenu = Menu(view.viewFrame, tearoff=0) view.ignoreDimValidity.trace("w", view.viewReloadDueToMenuAction) optionsMenu.add_checkbutton(label=_("Ignore Dimensional Validity"), underline=0, variable=view.ignoreDimValidity, onvalue=True, offvalue=False) view.xAxisChildrenFirst.trace("w", view.viewReloadDueToMenuAction) optionsMenu.add_checkbutton(label=_("X-Axis Children First"), underline=0, variable=view.xAxisChildrenFirst, onvalue=True, offvalue=False) view.yAxisChildrenFirst.trace("w", view.viewReloadDueToMenuAction) optionsMenu.add_checkbutton(label=_("Y-Axis Children First"), underline=0, variable=view.yAxisChildrenFirst, onvalue=True, offvalue=False) menu.add_cascade(label=_("Options"), menu=optionsMenu, underline=0) view.tablesMenu = Menu(view.viewFrame, tearoff=0) menu.add_cascade(label=_("Tables"), menu=view.tablesMenu, underline=0) view.tablesMenuLength = 0 view.menuAddLangs() view.menu.add_command(label=_("Save html file"), underline=0, command=lambda: view.modelXbrl.modelManager.cntlr.fileSave(view=view)) view.view() view.blockSelectEvent = 1 view.blockViewModelObject = 0 view.viewFrame.bind("<Enter>", view.cellEnter, '+') view.viewFrame.bind("<Leave>", view.cellLeave, '+')
def insertXbrl(self, rssItem): try: # must also have default dimensions loaded from arelle import ValidateXbrlDimensions ValidateXbrlDimensions.loadDimensionDefaults(self.modelXbrl) # find pre-existing documents in server database self.identifyPreexistingDocuments() self.identifyConceptsUsed() startedAt = time.time() self.insertAccession(rssItem) self.insertUris() self.insertQnames() self.insertNamespaces() self.insertDocuments() self.insertCustomArcroles() self.insertCustomRoles() self.insertElements() self.insertResources() self.insertNetworks() self.modelXbrl.profileStat(_("XbrlPublicDB: DTS insertion"), time.time() - startedAt) startedAt = time.time() self.insertFacts() self.modelXbrl.profileStat(_("XbrlPublicDB: instance insertion"), time.time() - startedAt) startedAt = time.time() self.showStatus("Committing entries") self.commit() self.modelXbrl.profileStat(_("XbrlPublicDB: insertion committed"), time.time() - startedAt) self.showStatus("DB insertion completed", clearAfter=5000) except Exception as ex: self.showStatus("DB insertion failed due to exception", clearAfter=5000) raise
def saveTargetDocument(self): targetUrl = self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl( self.targetDocumentPreferredFilename, self.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] self.modelXbrl.modelManager.showStatus( _("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create( self.modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=self.targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults( targetInstance) # need dimension defaults for context in self.modelXbrl.contexts.values(): newCntx = targetInstance.createContext( context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in self.modelXbrl.units.values(): measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) self.modelXbrl.modelManager.showStatus( _("Creating and validating facts")) for fact in self.modelXbrl.facts: if fact.isItem: attrs = [("contextRef", fact.contextID)] if fact.isNumeric: attrs.append(("unitRef", fact.unitID)) if fact.get("decimals"): attrs.append(("decimals", fact.get("decimals"))) if fact.get("precision"): attrs.append(("precision", fact.get("precision"))) if fact.isNil: attrs.append( ("{http://www.w3.org/2001/XMLSchema-instance}nil", "true")) text = None else: text = fact.xValue if fact.xValid else fact.elementText newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text) targetInstance.saveInstance(overrideFilepath=targetUrl) self.modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def viewRenderedGrid(modelXbrl, tabWin, lang=None): modelXbrl.modelManager.showStatus(_("viewing rendering")) view = ViewRenderedGrid(modelXbrl, tabWin, lang) # dimension defaults required in advance of validation from arelle import ValidateXbrlDimensions ValidateXbrlDimensions.loadDimensionDefaults(view) # context menu setDefaults(view) menu = view.contextMenu() optionsMenu = Menu(view.viewFrame, tearoff=0) optionsMenu.add_command( label=_("New fact item options"), underline=0, command=lambda: getNewFactItemOptions(modelXbrl.modelManager.cntlr, view.newFactItemOptions)) view.ignoreDimValidity.trace("w", view.viewReloadDueToMenuAction) optionsMenu.add_checkbutton(label=_("Ignore Dimensional Validity"), underline=0, variable=view.ignoreDimValidity, onvalue=True, offvalue=False) view.xAxisChildrenFirst.trace("w", view.viewReloadDueToMenuAction) optionsMenu.add_checkbutton(label=_("X-Axis Children First"), underline=0, variable=view.xAxisChildrenFirst, onvalue=True, offvalue=False) view.yAxisChildrenFirst.trace("w", view.viewReloadDueToMenuAction) optionsMenu.add_checkbutton(label=_("Y-Axis Children First"), underline=0, variable=view.yAxisChildrenFirst, onvalue=True, offvalue=False) menu.add_cascade(label=_("Options"), menu=optionsMenu, underline=0) view.tablesMenu = Menu(view.viewFrame, tearoff=0) menu.add_cascade(label=_("Tables"), menu=view.tablesMenu, underline=0) view.tablesMenuLength = 0 view.menuAddLangs() saveMenu = Menu(view.viewFrame, tearoff=0) saveMenu.add_command(label=_("HTML file"), underline=0, command=lambda: view.modelXbrl.modelManager.cntlr. fileSave(view=view, fileType="html")) saveMenu.add_command(label=_("XBRL instance"), underline=0, command=view.saveInstance) menu.add_cascade(label=_("Save"), menu=saveMenu, underline=0) view.view() view.blockSelectEvent = 1 view.blockViewModelObject = 0 view.viewFrame.bind("<Enter>", view.cellEnter, '+') view.viewFrame.bind("<Leave>", view.cellLeave, '+')
def viewRenderedGrid(modelXbrl, outfile, lang=None, viewTblELR=None, sourceView=None): modelXbrl.modelManager.showStatus(_("viewing rendering")) view = ViewRenderedGrid(modelXbrl, outfile, lang) # dimension defaults required in advance of validation from arelle import ValidateXbrlDimensions ValidateXbrlDimensions.loadDimensionDefaults(view) setDefaults(view) if sourceView is not None: viewTblELR = sourceView.tblELR view.ignoreDimValidity.set(sourceView.ignoreDimValidity.get()) view.xAxisChildrenFirst.set(sourceView.xAxisChildrenFirst.get()) view.yAxisChildrenFirst.set(sourceView.yAxisChildrenFirst.get()) view.view(viewTblELR) view.close()
def insertXbrl(self, rssItem): try: # must also have default dimensions loaded from arelle import ValidateXbrlDimensions ValidateXbrlDimensions.loadDimensionDefaults(self.modelXbrl) # obtain supplementaion entity information self.entityInformation = loadEntityInformation( self.modelXbrl, rssItem) # identify table facts (table datapoints) (prior to locked database transaction self.tableFacts = tableFacts( self.modelXbrl ) # for EFM & HMRC this is ( (roleType, table_code, fact) ) loadPrimaryDocumentFacts( self.modelXbrl, rssItem, self.entityInformation ) # load primary document facts for SEC filing # find pre-existing documents in server database self.identifyPreexistingDocuments() self.identifyConceptsUsed() startedAt = time.time() self.insertAccession(rssItem) self.insertUris() self.insertQnames() self.insertNamespaces() self.insertDocuments() self.insertCustomArcroles() self.insertCustomRoles() self.insertElements() self.insertResources() self.insertNetworks() self.modelXbrl.profileStat(_("XbrlPublicDB: DTS insertion"), time.time() - startedAt) startedAt = time.time() self.insertFacts() self.modelXbrl.profileStat(_("XbrlPublicDB: instance insertion"), time.time() - startedAt) startedAt = time.time() self.showStatus("Committing entries") self.commit() self.modelXbrl.profileStat(_("XbrlPublicDB: insertion committed"), time.time() - startedAt) self.showStatus("DB insertion completed", clearAfter=5000) except Exception as ex: self.showStatus("DB insertion failed due to exception", clearAfter=5000) raise
def insertXbrl(self, rssItem): try: # must also have default dimensions loaded from arelle import ValidateXbrlDimensions ValidateXbrlDimensions.loadDimensionDefaults(self.modelXbrl) #initialVcount, initialEcount = self.getDBsize() # don't include in timing, very slow startedAt = time.time() # find pre-existing documents in server database self.identifyPreexistingDocuments() g = {FILINGS:{}, DOCUMENTS:{}} self.insertSchema(g) # self.load() this done in the verify step self.insertFiling(rssItem,g) self.insertDocuments(g) self.insertDataDictionary() # XML namespaces types aspects #self.insertRelationshipTypeSets() #self.insertResourceRoleSets() #self.insertAspectValues() self.modelXbrl.profileStat(_("XbrlSemanticJsonDB: DTS insertion"), time.time() - startedAt) startedAt = time.time() self.insertDataPoints() self.modelXbrl.profileStat(_("XbrlSemanticJsonDB: data points insertion"), time.time() - startedAt) startedAt = time.time() self.insertRelationshipSets() self.modelXbrl.profileStat(_("XbrlSemanticJsonDB: Relationships insertion"), time.time() - startedAt) self.insertValidationResults() self.modelXbrl.profileStat(_("XbrlSemanticJsonDB: Validation results insertion"), time.time() - startedAt) #startedAt = time.time() #self.insertValidCombinations() #self.modelXbrl.profileStat(_("XbrlSemanticJsonDB: Valid Combinations insertion"), time.time() - startedAt) self.showStatus("Committing entries") self.commit(g) self.modelXbrl.profileStat(_("XbrlSemanticJsonDB: insertion committed"), time.time() - startedAt) #finalVcount, finalEcount = self.getDBsize() #self.modelXbrl.modelManager.addToLog("added vertices: {0}, edges: {1}, total vertices: {2}, edges: {3}".format( # finalVcount - initialVcount, finalEcount - initialEcount, finalVcount, finalEcount)) self.showStatus("DB insertion completed", clearAfter=5000) except Exception as ex: self.showStatus("DB insertion failed due to exception", clearAfter=5000) raise
def saveTargetDocument(self): targetUrl = self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(self.targetDocumentPreferredFilename, self.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] self.modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create(self.modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=self.targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults for context in self.modelXbrl.contexts.values(): newCntx = targetInstance.createContext(context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in self.modelXbrl.units.values(): measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) self.modelXbrl.modelManager.showStatus(_("Creating and validating facts")) for fact in self.modelXbrl.facts: if fact.isItem: attrs = [("contextRef", fact.contextID)] if fact.isNumeric: attrs.append(("unitRef", fact.unitID)) if fact.get("decimals"): attrs.append(("decimals", fact.get("decimals"))) if fact.get("precision"): attrs.append(("precision", fact.get("precision"))) if fact.isNil: attrs.append(("{http://www.w3.org/2001/XMLSchema-instance}nil","true")) text = None else: text = fact.xValue if fact.xValid else fact.elementText newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text) targetInstance.saveInstance(overrideFilepath=targetUrl) self.modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def insertXbrl(self, rssItem): try: # must also have default dimensions loaded from arelle import ValidateXbrlDimensions ValidateXbrlDimensions.loadDimensionDefaults(self.modelXbrl) # get logging entries (needed to find which aspects to identify) self.loggingEntries = [] for handler in logging.getLogger("arelle").handlers: if hasattr(handler, "dbHandlerLogEntries"): self.loggingEntries = handler.dbHandlerLogEntries() break # find pre-existing documents in server database self.identifyPreexistingDocuments() self.identifyAspectsUsed() startedAt = time.time() self.dropTemporaryTable() self.insertFiling(rssItem) self.insertDocuments() self.insertAspects() self.insertArcroleTypes() self.insertRoleTypes() self.insertResources() self.insertRelationships() self.modelXbrl.profileStat(_("XbrlSqlDB: DTS insertion"), time.time() - startedAt) startedAt = time.time() self.insertDataPoints() self.modelXbrl.profileStat(_("XbrlSqlDB: instance insertion"), time.time() - startedAt) self.insertValidationResults() self.modelXbrl.profileStat(_("XbrlSqlDB: Validation results insertion"), time.time() - startedAt) startedAt = time.time() self.showStatus("Committing entries") self.commit() self.modelXbrl.profileStat(_("XbrlSqlDB: insertion committed"), time.time() - startedAt) self.showStatus("DB insertion completed", clearAfter=5000) except Exception as ex: self.showStatus("DB insertion failed due to exception", clearAfter=5000) raise
def insertXbrl(self, rssItem): try: # must also have default dimensions loaded from arelle import ValidateXbrlDimensions ValidateXbrlDimensions.loadDimensionDefaults(self.modelXbrl) # obtain supplementaion entity information self.entityInformation = loadEntityInformation(self.modelXbrl, rssItem) # identify table facts (table datapoints) (prior to locked database transaction self.tableFacts = tableFacts(self.modelXbrl) # for EFM & HMRC this is ( (roleType, table_code, fact) ) loadPrimaryDocumentFacts(self.modelXbrl, rssItem, self.entityInformation) # load primary document facts for SEC filing # find pre-existing documents in server database self.identifyPreexistingDocuments() self.identifyConceptsUsed() startedAt = time.time() self.insertAccession(rssItem) self.insertUris() self.insertQnames() self.insertNamespaces() self.insertDocuments() self.insertCustomArcroles() self.insertCustomRoles() self.insertElements() self.insertResources() self.insertNetworks() self.modelXbrl.profileStat(_("XbrlPublicDB: DTS insertion"), time.time() - startedAt) startedAt = time.time() self.insertFacts() self.modelXbrl.profileStat(_("XbrlPublicDB: instance insertion"), time.time() - startedAt) startedAt = time.time() self.showStatus("Committing entries") self.commit() self.modelXbrl.profileStat(_("XbrlPublicDB: insertion committed"), time.time() - startedAt) self.showStatus("DB insertion completed", clearAfter=5000) except Exception as ex: self.showStatus("DB insertion failed due to exception", clearAfter=5000) raise
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri): from openpyxl import load_workbook from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname try: currentAction = "initializing" startingErrorCount = len(modelXbrl.errors) startedAt = time.time() if os.path.isabs(oimFile): # allow relative filenames to loading directory priorCWD = os.getcwd() os.chdir(os.path.dirname(oimFile)) else: priorCWD = None currentAction = "determining file type" isJSON = oimFile.endswith( ".json") and not oimFile.endswith("-metadata.json") isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json") isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls") isCSVorXL = isCSV or isXL instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl" if isJSON: currentAction = "loading and parsing JSON OIM file" def loadDict(keyValuePairs): _dict = OrderedDict( ) # preserve fact order in resulting instance for key, value in keyValuePairs: if isinstance(value, dict): if DUPJSONKEY in value: for _errKey, _errValue, _otherValue in value[ DUPJSONKEY]: if key == "prefixes": modelXbrl.error( "oime:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s" ), modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue) del value[DUPJSONKEY] if key in _dict: if DUPJSONKEY not in _dict: _dict[DUPJSONKEY] = [] _dict[DUPJSONKEY].append((key, value, _dict[key])) else: _dict[key] = value return _dict with io.open(oimFile, 'rt', encoding='utf-8') as f: oimObject = json.load(f, object_pairs_hook=loadDict) missing = [ t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject ] if missing: raise OIMException( "oime:missingJSONElements", _("Required element(s) are missing from JSON input: %(missing)s" ), missing=", ".join(missing)) currentAction = "identifying JSON objects" dtsReferences = oimObject["dtsReferences"] prefixesList = oimObject["prefixes"].items() facts = oimObject["facts"] footnotes = oimObject["facts"] # shares this object elif isCSV: currentAction = "identifying CSV input tables" if sys.version[0] >= '3': csvOpenMode = 'w' csvOpenNewline = '' else: csvOpenMode = 'wb' # for 2.7 csvOpenNewline = None oimFileBase = None if "-facts" in oimFile: oimFileBase = oimFile.partition("-facts")[0] else: for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"): if oimFile.endswith(suffix): oimFileBase = oimFile[:-len(suffix)] break if oimFileBase is None: raise OIMException( "oime:missingCSVTables", _("Unable to identify CSV tables file name pattern")) if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or not os.path.exists(oimFileBase + "-prefixes.csv")): raise OIMException( "oime:missingCSVTables", _("Unable to identify CSV tables for dtsReferences or prefixes" )) instanceFileName = oimFileBase + ".xbrl" currentAction = "loading CSV dtsReferences table" dtsReferences = [] with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: dtsReferences.append( dict( (header[j], col) for j, col in enumerate(row))) currentAction = "loading CSV prefixes table" prefixesList = [] with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = dict((col, i) for i, col in enumerate(row)) else: prefixesList.append( (row[header["prefix"]], row[header["URI"]])) defaults = {} if os.path.exists(oimFileBase + "-defaults.csv"): currentAction = "loading CSV defaults table" with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row fileCol = row.index("file") else: defaults[row[fileCol]] = dict( (header[j], col) for j, col in enumerate(row) if j != fileCol) currentAction = "loading CSV facts tables" facts = [] _dir = os.path.dirname(oimFileBase) factsFileBasename = os.path.basename(oimFileBase) + "-facts" for filename in os.listdir(_dir): filepath = os.path.join(_dir, filename) if filename.startswith(factsFileBasename): currentAction = "loading CSV facts table {}".format( filename) tableDefaults = defaults.get(filename, {}) with io.open(filepath, 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col is not None: if header[ j]: # skip cols with no header if header[j].endswith("Value"): if col: # ignore empty columns (= null CSV value) fact["value"] = col else: fact[header[j]] = col facts.append(fact) footnotes = [] if os.path.exists(oimFileBase + "-footnotes.csv"): currentAction = "loading CSV footnotes table" with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: footnotes.append( dict((header[j], col) for j, col in enumerate(row) if col)) elif isXL: currentAction = "identifying workbook input worksheets" oimWb = load_workbook(oimFile, read_only=True, data_only=True) sheetNames = oimWb.get_sheet_names() if (not any(sheetName == "prefixes" for sheetName in sheetNames) or not any(sheetName == "dtsReferences" for sheetName in sheetNames) or not any("facts" in sheetName for sheetName in sheetNames)): raise OIMException( "oime:missingWorkbookWorksheets", _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts" )) currentAction = "loading worksheet: dtsReferences" dtsReferences = [] for i, row in enumerate(oimWb["dtsReferences"]): if i == 0: header = [col.value for col in row] else: dtsReferences.append( dict((header[j], col.value) for j, col in enumerate(row))) currentAction = "loading worksheet: prefixes" prefixesList = [] for i, row in enumerate(oimWb["prefixes"]): if i == 0: header = dict((col.value, i) for i, col in enumerate(row)) else: prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value)) defaults = {} if "defaults" in sheetNames: currentAction = "loading worksheet: defaults" for i, row in enumerate(oimWb["defaults"]): if i == 0: header = dict( (col.value, i) for i, col in enumerate(row)) fileCol = header["file"] else: defaults[row[fileCol].value] = dict( (header[j], col.value) for j, col in enumerate(row) if j != fileCol) facts = [] for sheetName in sheetNames: if sheetName == "facts" or "-facts" in sheetName: currentAction = "loading worksheet: {}".format(sheetName) tableDefaults = defaults.get(sheetName, {}) for i, row in enumerate(oimWb[sheetName]): if i == 0: header = [col.value for col in row] else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col.value is not None: if header[j]: # skip cols with no header if header[j].endswith("Value"): fact["value"] = str(col.value) else: fact[header[j]] = str(col.value) facts.append(fact) footnotes = [] if "footnotes" in sheetNames: currentAction = "loading worksheet: footnotes" for i, row in enumerate(oimWb["footnotes"]): if i == 0: header = dict((col.value, i) for i, col in enumerate(row) if col.value) else: footnotes.append( dict((header[j], col.value) for j, col in enumerate(row) if col.value)) currentAction = "identifying default dimensions" ValidateXbrlDimensions.loadDimensionDefaults( modelXbrl) # needs dimension defaults currentAction = "validating OIM" prefixes = {} prefixedUris = {} for _prefix, _uri in prefixesList: if not _prefix: modelXbrl.error( "oime:emptyPrefix", _("The empty string must not be used as a prefix, uri %(uri)s" ), modelObject=modelXbrl, uri=_uri) elif not NCNamePattern.match(_prefix): modelXbrl.error( "oime:prefixPattern", _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s" ), modelObject=modelXbrl, prefix=_prefix, uri=_uri) elif _prefix in prefixes: modelXbrl.error( "oime:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s" ), modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri) elif _uri in prefixedUris: modelXbrl.error( "oime:duplicatedUri", _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s" ), modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix) else: prefixes[_prefix] = _uri prefixedUris[_uri] = _prefix oimPrefix = None for _nsOim in nsOim: if _nsOim in prefixedUris: oimPrefix = prefixedUris[_nsOim] if not oimPrefix: raise OIMException( "oime:noOimPrefix", _("The oim namespace must have a declared prefix")) oimConcept = "{}:concept".format(oimPrefix) oimEntity = "{}:entity".format(oimPrefix) oimPeriod = "{}:period".format(oimPrefix) oimPeriodStart = "{}:periodStart".format(oimPrefix) oimPeriodEnd = "{}:periodEnd".format(oimPrefix) oimUnit = "{}:unit".format(oimPrefix) oimPrefix = "{}:".format(oimPrefix) # create the instance document currentAction = "creating instance document" modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument( modelXbrl, Type.INSTANCE, instanceFileName, schemaRefs=[ dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema" ], isEntry=True, initialComment="extracted from OIM {}".format(mappedUri), documentEncoding="utf-8") cntxTbl = {} unitTbl = {} currentAction = "creating facts" for fact in facts: conceptQn = qname(fact[oimConcept], prefixes) concept = modelXbrl.qnameConcepts.get(conceptQn) entityAsQn = qname(fact[oimEntity], prefixes) if oimPeriod in fact: periodStart = fact[oimPeriod]["start"] periodEnd = fact[oimPeriod]["end"] else: periodStart = fact[oimPeriodStart] periodEnd = fact[oimPeriodEnd] cntxKey = ( # hashable context key ("periodType", concept.periodType), ("entity", entityAsQn), ("periodStart", periodStart), ("periodEnd", periodEnd)) + tuple( sorted( (dimName, dimVal) for dimName, dimVal in fact.items() if ":" in dimName and not dimName.startswith(oimPrefix))) if cntxKey in cntxTbl: _cntx = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) qnameDims = {} for dimName, dimVal in fact.items(): if ":" in dimName and not dimName.startswith( oimPrefix) and dimVal: dimQname = qname(dimName, prefixes) dimConcept = modelXbrl.qnameConcepts.get(dimQname) if ":" in dimVal and dimVal.partition( ':')[0] in prefixes: mem = qname(dimVal, prefixes) # explicit dim elif dimConcept.isTypedDimension: # a modelObject xml element is needed for all of the instance functions to manage the typed dim mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False) qnameDims[dimQname] = DimValuePrototype( modelXbrl, None, dimQname, mem, "segment") _cntx = modelXbrl.createContext( entityAsQn.namespaceURI, entityAsQn.localName, concept.periodType, None if concept.periodType == "instant" else dateTime( periodStart, type=DATETIME), dateTime(periodEnd, type=DATETIME), None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) cntxTbl[cntxKey] = _cntx if oimUnit in fact: unitKey = fact[oimUnit] if unitKey in unitTbl: _unit = unitTbl[unitKey] else: _unit = None # validate unit unitKeySub = PrefixedQName.sub( UnitPrefixedQNameSubstitutionChar, unitKey) if not UnitPattern.match(unitKeySub): modelXbrl.error( "oime:unitStringRepresentation", _("Unit string representation is lexically invalid, %(unit)s" ), modelObject=modelXbrl, unit=unitKey) else: _mul, _sep, _div = unitKey.partition('/') if _mul.startswith('('): _mul = _mul[1:-1] _muls = [u for u in _mul.split('*') if u] if _div.startswith('('): _div = _div[1:-1] _divs = [u for u in _div.split('*') if u] if _muls != sorted(_muls) or _divs != sorted(_divs): modelXbrl.error( "oime:unitStringRepresentation", _("Unit string representation measures are not in alphabetical order, %(unit)s" ), modelObject=modelXbrl, unit=unitKey) try: mulQns = [ qname( u, prefixes, prefixException=OIMException( "oime:unitPrefix", _("Unit prefix is not declared: %(unit)s" ), unit=u)) for u in _muls ] divQns = [ qname( u, prefixes, prefixException=OIMException( "oime:unitPrefix", _("Unit prefix is not declared: %(unit)s" ), unit=u)) for u in _divs ] unitId = 'u-{:02}'.format(len(unitTbl) + 1) for _measures in mulQns, divQns: for _measure in _measures: addQnameValue(modelXbrl.modelDocument, _measure) _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId) except OIMException as ex: modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) unitTbl[unitKey] = _unit else: _unit = None attrs = {"contextRef": _cntx.id} if fact.get("value") is None: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact["value"] if fact.get("id"): attrs["id"] = fact["id"] if concept.isNumeric: if _unit is None: continue # skip creating fact because unit was invalid attrs["unitRef"] = _unit.id if "accuracy" in fact: attrs["decimals"] = fact["accuracy"] # is value a QName? if concept.baseXbrliType == "QName": addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes)) f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text) currentAction = "creating footnotes" footnoteLinks = {} # ELR elements factLocs = {} # index by (linkrole, factId) footnoteNbr = 0 locNbr = 0 for factOrFootnote in footnotes: if "factId" in factOrFootnote: factId = factOrFootnote["factId"] factFootnotes = (factOrFootnote, ) # CSV or XL elif "id" in factOrFootnote and "footnotes" in factOrFootnote: factId = factOrFootnote["id"] factFootnotes = factOrFootnote["footnotes"] else: factFootnotes = () for footnote in factFootnotes: linkrole = footnote.get("group") arcrole = footnote.get("footnoteType") if not factId or not linkrole or not arcrole or not ( footnote.get("factRef") or footnote.get("footnote")): # invalid footnote continue if linkrole not in footnoteLinks: footnoteLinks[linkrole] = addChild( modelXbrl.modelDocument.xmlRootElement, XbrlConst.qnLinkFootnoteLink, attributes={ "{http://www.w3.org/1999/xlink}type": "extended", "{http://www.w3.org/1999/xlink}role": linkrole }) footnoteLink = footnoteLinks[linkrole] if (linkrole, factId) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factId)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={ XLINKTYPE: "locator", XLINKHREF: "#" + factId, XLINKLABEL: locLabel }) locLabel = factLocs[(linkrole, factId)] if footnote.get("footnote"): footnoteNbr += 1 footnoteLabel = "f_{:02}".format(footnoteNbr) attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel} if footnote.get("language"): attrs[XMLLANG] = footnote["language"] # note, for HTML will need to build an element structure addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"]) elif footnote.get("factRef"): factRef = footnote.get("factRef") if (linkrole, factRef) not in factLocs: locNbr += 1 locLabel = "f_{:02}".format(footnoteNbr) factLoc[(linkrole, factRef)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={ XLINKTYPE: "locator", XLINKHREF: "#" + factRef, XLINKLABEL: locLabel }) footnoteLabel = factLoc[(linkrole, factId)] footnoteArc = addChild(footnoteLink, XbrlConst.qnLinkFootnoteArc, attributes={ XLINKTYPE: "arc", XLINKARCROLE: arcrole, XLINKFROM: locLabel, XLINKTO: footnoteLabel }) currentAction = "done loading facts and footnotes" #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") except Exception as ex: if isinstance(ex, OIMException): modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) else: modelXbrl.error( "arelleOIMloader:error", "Error while %(action)s, error %(error)s\ntraceback %(traceback)s", modelObject=modelXbrl, action=currentAction, error=ex, traceback=traceback.format_tb(sys.exc_info()[2])) if priorCWD: os.chdir( priorCWD ) # restore prior current working directory startingErrorCount = len(modelXbrl.errors) if startingErrorCount < len(modelXbrl.errors): # had errors, don't allow ModelDocument.load to continue return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors") return getattr(modelXbrl, "modelDocument", None) # none if returning from exception
def evaluateTableIndex(modelXbrl): disclosureSystem = modelXbrl.modelManager.disclosureSystem if disclosureSystem.EFM: COVER = "1Cover" STMTS = "2Financial Statements" NOTES = "3Notes to Financial Statements" POLICIES = "4Accounting Policies" TABLES = "5Notes Tables" DETAILS = "6Notes Details" UNCATEG = "7Uncategorized" roleDefinitionPattern = re.compile(r"([0-9]+) - (Statement|Disclosure|Schedule|Document) - (.+)") # build EFM rendering-compatible index definitionElrs = dict((roleType.definition, roleType) for roleURI in modelXbrl.relationshipSet(XbrlConst.parentChild).linkRoleUris for roleType in modelXbrl.roleTypes.get(roleURI,())) isRR = any(ns.startswith("http://xbrl.sec.gov/rr/") for ns in modelXbrl.namespaceDocs.keys()) tableGroup = None firstTableLinkroleURI = None firstDocumentLinkroleURI = None sortedRoleTypes = sorted(definitionElrs.items(), key=lambda item: item[0]) for roleDefinition, roleType in sortedRoleTypes: roleType._tableChildren = [] match = roleDefinitionPattern.match(roleDefinition) if roleDefinition else None if not match: roleType._tableIndex = (UNCATEG, "", roleType.roleURI) continue seq, tblType, tblName = match.groups() if isRR: tableGroup = COVER elif not tableGroup: tableGroup = ("Paren" in tblName and COVER or tblType == "Statement" and STMTS or "(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or COVER) elif tableGroup == COVER: tableGroup = (tblType == "Statement" and STMTS or "Paren" in tblName and COVER or "(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or NOTES) elif tableGroup == STMTS: tableGroup = ((tblType == "Statement" or "Paren" in tblName) and STMTS or "(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or NOTES) elif tableGroup == NOTES: tableGroup = ("(Polic" in tblName and POLICIES or "(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or tblType == "Disclosure" and NOTES or UNCATEG) elif tableGroup == POLICIES: tableGroup = ("(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or ("Paren" in tblName or "(Polic" in tblName) and POLICIES or UNCATEG) elif tableGroup == TABLES: tableGroup = ("(Detail" in tblName and DETAILS or ("Paren" in tblName or "(Table" in tblName) and TABLES or UNCATEG) elif tableGroup == DETAILS: tableGroup = (("Paren" in tblName or "(Detail" in tblName) and DETAILS or UNCATEG) else: tableGroup = UNCATEG if firstTableLinkroleURI is None and tableGroup == COVER: firstTableLinkroleURI = roleType.roleURI if tblType == "Document" and not firstDocumentLinkroleURI: firstDocumentLinkroleURI = roleType.roleURI roleType._tableIndex = (tableGroup, seq, tblName) # flow allocate facts to roles (SEC presentation groups) if not modelXbrl.qnameDimensionDefaults: # may not have run validatino yet from arelle import ValidateXbrlDimensions ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) reportedFacts = set() # facts which were shown in a higher-numbered ELR table factsByQname = modelXbrl.factsByQname reportingPeriods = set() nextEnd = None deiFact = {} for conceptName in ("DocumentPeriodEndDate", "DocumentType", "CurrentFiscalPeriodEndDate"): for concept in modelXbrl.nameConcepts[conceptName]: for fact in factsByQname[concept.qname]: deiFact[conceptName] = fact if fact.context is not None: reportingPeriods.add((None, fact.context.endDatetime)) # for instant reportingPeriods.add((fact.context.startDatetime, fact.context.endDatetime)) # for startEnd nextEnd = fact.context.startDatetime duration = (fact.context.endDatetime - fact.context.startDatetime).days + 1 break if "DocumentType" in deiFact: fact = deiFact["DocumentType"] if "-Q" in fact.xValue: # need quarterly and yr to date durations endDatetime = fact.context.endDatetime # if within 2 days of end of month use last day of month endDatetimeMonth = endDatetime.month if (endDatetime + timedelta(2)).month != endDatetimeMonth: # near end of month endOfMonth = True while endDatetime.month == endDatetimeMonth: endDatetime += timedelta(1) # go forward to next month else: endOfMonth = False startYr = endDatetime.year startMo = endDatetime.month - 3 if startMo <= 0: startMo += 12 startYr -= 1 startDatetime = datetime(startYr, startMo, endDatetime.day, endDatetime.hour, endDatetime.minute, endDatetime.second) if endOfMonth: startDatetime -= timedelta(1) endDatetime -= timedelta(1) reportingPeriods.add((startDatetime, endDatetime)) duration = 91 # find preceding compatible default context periods while (nextEnd is not None): thisEnd = nextEnd prevMaxStart = thisEnd - timedelta(duration * .9) prevMinStart = thisEnd - timedelta(duration * 1.1) nextEnd = None for cntx in modelXbrl.contexts.values(): if (cntx.isStartEndPeriod and not cntx.qnameDims and thisEnd == cntx.endDatetime and prevMinStart <= cntx.startDatetime <= prevMaxStart): reportingPeriods.add((None, cntx.endDatetime)) reportingPeriods.add((cntx.startDatetime, cntx.endDatetime)) nextEnd = cntx.startDatetime break elif (cntx.isInstantPeriod and not cntx.qnameDims and thisEnd == cntx.endDatetime): reportingPeriods.add((None, cntx.endDatetime)) stmtReportingPeriods = set(reportingPeriods) sortedRoleTypes.reverse() # now in descending order for i, roleTypes in enumerate(sortedRoleTypes): roleDefinition, roleType = roleTypes # find defined non-default axes in pre hierarchy for table tableFacts = set() tableGroup, tableSeq, tableName = roleType._tableIndex roleURIdims, priItemQNames = EFMlinkRoleURIstructure(modelXbrl, roleType.roleURI) for priItemQName in priItemQNames: for fact in factsByQname[priItemQName]: cntx = fact.context # non-explicit dims must be default if (cntx is not None and all(dimQn in modelXbrl.qnameDimensionDefaults for dimQn in (roleURIdims.keys() - cntx.qnameDims.keys())) and all(mdlDim.memberQname in roleURIdims[dimQn] for dimQn, mdlDim in cntx.qnameDims.items() if dimQn in roleURIdims)): # the flow-up part, drop cntxStartDatetime = cntx.startDatetime cntxEndDatetime = cntx.endDatetime if (tableGroup != STMTS or (cntxStartDatetime, cntxEndDatetime) in stmtReportingPeriods and (fact not in reportedFacts or all(dimQn not in cntx.qnameDims # unspecified dims are all defaulted if reported elsewhere for dimQn in (cntx.qnameDims.keys() - roleURIdims.keys())))): tableFacts.add(fact) reportedFacts.add(fact) roleType._tableFacts = tableFacts # find parent if any closestParentType = None closestParentMatchLength = 0 for _parentRoleDefinition, parentRoleType in sortedRoleTypes[i+1:]: matchLen = parentNameMatchLen(tableName, parentRoleType) if matchLen > closestParentMatchLength: closestParentMatchLength = matchLen closestParentType = parentRoleType if closestParentType is not None: closestParentType._tableChildren.insert(0, roleType) # remove lesser-matched children if there was a parent match unmatchedChildRoles = set() longestChildMatchLen = 0 numChildren = 0 for childRoleType in roleType._tableChildren: matchLen = parentNameMatchLen(tableName, childRoleType) if matchLen < closestParentMatchLength: unmatchedChildRoles.add(childRoleType) elif matchLen > longestChildMatchLen: longestChildMatchLen = matchLen numChildren += 1 if numChildren > 1: # remove children that don't have the full match pattern length to parent for childRoleType in roleType._tableChildren: if (childRoleType not in unmatchedChildRoles and parentNameMatchLen(tableName, childRoleType) < longestChildMatchLen): unmatchedChildRoles.add(childRoleType) for unmatchedChildRole in unmatchedChildRoles: roleType._tableChildren.remove(unmatchedChildRole) for childRoleType in roleType._tableChildren: childRoleType._tableParent = roleType unmatchedChildRoles = None # dereference global UGT_TOPICS if UGT_TOPICS is None: try: from arelle import FileSource fh = FileSource.openFileStream(modelXbrl.modelManager.cntlr, os.path.join(modelXbrl.modelManager.cntlr.configDir, "ugt-topics.zip/ugt-topics.json"), 'r', 'utf-8') UGT_TOPICS = json.load(fh) fh.close() for topic in UGT_TOPICS: topic[6] = set(topic[6]) # change concept abstracts list into concept abstracts set topic[7] = set(topic[7]) # change concept text blocks list into concept text blocks set topic[8] = set(topic[8]) # change concept names list into concept names set except Exception as ex: UGT_TOPICS = None if UGT_TOPICS is not None: def roleUgtConcepts(roleType): roleConcepts = set() for rel in modelXbrl.relationshipSet(XbrlConst.parentChild, roleType.roleURI).modelRelationships: if rel.toModelObject is not None: roleConcepts.add(rel.toModelObject.name) if rel.fromModelObject is not None: roleConcepts.add(rel.fromModelObject.name) if hasattr(roleType, "_tableChildren"): for _tableChild in roleType._tableChildren: roleConcepts |= roleUgtConcepts(_tableChild) return roleConcepts topicMatches = {} # topicNum: (best score, roleType) for roleDefinition, roleType in sortedRoleTypes: roleTopicType = 'S' if roleDefinition.startswith('S') else 'D' if getattr(roleType, "_tableParent", None) is None: # rooted tables in reverse order concepts = roleUgtConcepts(roleType) for i, ugtTopic in enumerate(UGT_TOPICS): if ugtTopic[0] == roleTopicType: countAbstracts = len(concepts & ugtTopic[6]) countTextBlocks = len(concepts & ugtTopic[7]) countLineItems = len(concepts & ugtTopic[8]) if countAbstracts or countTextBlocks or countLineItems: _score = (10 * countAbstracts + 1000 * countTextBlocks + countLineItems / len(concepts)) if i not in topicMatches or _score > topicMatches[i][0]: topicMatches[i] = (_score, roleType) for topicNum, scoredRoleType in topicMatches.items(): _score, roleType = scoredRoleType if _score > getattr(roleType, "_tableTopicScore", 0): ugtTopic = UGT_TOPICS[topicNum] roleType._tableTopicScore = _score roleType._tableTopicType = ugtTopic[0] roleType._tableTopicName = ugtTopic[3] roleType._tableTopicCode = ugtTopic[4] # print ("Match score {:.2f} topic {} preGrp {}".format(_score, ugtTopic[3], roleType.definition)) return firstTableLinkroleURI or firstDocumentLinkroleURI # did build _tableIndex attributes return None
def run(self, options, sourceZipStream=None): """Process command line arguments or web service request, such as to load and validate an XBRL document, or start web server. When a web server has been requested, this method may be called multiple times, once for each web service (REST) request that requires processing. Otherwise (when called for a command line request) this method is called only once for the command line arguments request. :param options: OptionParser options from parse_args of main argv arguments (when called from command line) or corresponding arguments from web service (REST) request. :type options: optparse.Values """ if options.showOptions: # debug options for optName, optValue in sorted(options.__dict__.items(), key=lambda optItem: optItem[0]): self.addToLog("Option {0}={1}".format(optName, optValue), messageCode="info") self.addToLog("sys.argv {0}".format(sys.argv), messageCode="info") if options.uiLang: # set current UI Lang (but not config setting) self.setUiLanguage(options.uiLang) if options.proxy: if options.proxy != "show": proxySettings = proxyTuple(options.proxy) self.webCache.resetProxies(proxySettings) self.config["proxySettings"] = proxySettings self.saveConfig() self.addToLog(_("Proxy configuration has been set."), messageCode="info") useOsProxy, urlAddr, urlPort, user, password = self.config.get("proxySettings", proxyTuple("none")) if useOsProxy: self.addToLog(_("Proxy configured to use {0}.").format( _('Microsoft Windows Internet Settings') if sys.platform.startswith("win") else (_('Mac OS X System Configuration') if sys.platform in ("darwin", "macos") else _('environment variables'))), messageCode="info") elif urlAddr: self.addToLog(_("Proxy setting: http://{0}{1}{2}{3}{4}").format( user if user else "", ":****" if password else "", "@" if (user or password) else "", urlAddr, ":{0}".format(urlPort) if urlPort else ""), messageCode="info") else: self.addToLog(_("Proxy is disabled."), messageCode="info") if options.plugins: from arelle import PluginManager resetPlugins = False savePluginChanges = True showPluginModules = False for pluginCmd in options.plugins.split('|'): cmd = pluginCmd.strip() if cmd == "show": showPluginModules = True elif cmd == "temp": savePluginChanges = False elif cmd.startswith("+"): moduleInfo = PluginManager.addPluginModule(cmd[1:]) if moduleInfo: self.addToLog(_("Addition of plug-in {0} successful.").format(moduleInfo.get("name")), messageCode="info", file=moduleInfo.get("moduleURL")) resetPlugins = True else: self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:]) elif cmd.startswith("~"): if PluginManager.reloadPluginModule(cmd[1:]): self.addToLog(_("Reload of plug-in successful."), messageCode="info", file=cmd[1:]) resetPlugins = True else: self.addToLog(_("Unable to reload plug-in."), messageCode="info", file=cmd[1:]) elif cmd.startswith("-"): if PluginManager.removePluginModule(cmd[1:]): self.addToLog(_("Deletion of plug-in successful."), messageCode="info", file=cmd[1:]) resetPlugins = True else: self.addToLog(_("Unable to delete plug-in."), messageCode="info", file=cmd[1:]) else: # assume it is a module or package savePluginChanges = False moduleInfo = PluginManager.addPluginModule(cmd) if moduleInfo: self.addToLog(_("Activation of plug-in {0} successful.").format(moduleInfo.get("name")), messageCode="info", file=moduleInfo.get("moduleURL")) resetPlugins = True else: self.addToLog(_("Unable to load {0} as a plug-in or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd) if resetPlugins: PluginManager.reset() if savePluginChanges: PluginManager.save(self) if showPluginModules: self.addToLog(_("Plug-in modules:"), messageCode="info") for i, moduleItem in enumerate(sorted(PluginManager.pluginConfig.get("modules", {}).items())): moduleInfo = moduleItem[1] self.addToLog(_("Plug-in: {0}; author: {1}; version: {2}; status: {3}; date: {4}; description: {5}; license {6}.").format( moduleItem[0], moduleInfo.get("author"), moduleInfo.get("version"), moduleInfo.get("status"), moduleInfo.get("fileDate"), moduleInfo.get("description"), moduleInfo.get("license")), messageCode="info", file=moduleInfo.get("moduleURL")) # run utility command line options that don't depend on entrypoint Files hasUtilityPlugin = False for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Utility.Run"): hasUtilityPlugin = True pluginXbrlMethod(self, options) # if no entrypointFile is applicable, quit now if options.proxy or options.plugins or hasUtilityPlugin: if not options.entrypointFile: return True # success self.username = options.username self.password = options.password self.entrypointFile = options.entrypointFile if self.entrypointFile: filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream) else: filesource = None if options.validateEFM: if options.disclosureSystemName: self.addToLog(_("both --efm and --disclosureSystem validation are requested, proceeding with --efm only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("efm") elif options.disclosureSystemName: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select(options.disclosureSystemName) elif options.validateHMRC: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("hmrc") else: self.modelManager.disclosureSystem.select(None) # just load ordinary mappings self.modelManager.validateDisclosureSystem = False if options.utrUrl: # override disclosureSystem utrUrl self.modelManager.disclosureSystem.utrUrl = options.utrUrl # can be set now because the utr is first loaded at validation time # disclosure system sets logging filters, override disclosure filters, if specified by command line if options.logLevelFilter: self.setLogLevelFilter(options.logLevelFilter) if options.logCodeFilter: self.setLogCodeFilter(options.logCodeFilter) if options.calcDecimals: if options.calcPrecision: self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateInferDecimals = True self.modelManager.validateCalcLB = True elif options.calcPrecision: self.modelManager.validateInferDecimals = False self.modelManager.validateCalcLB = True if options.utrValidate: self.modelManager.validateUtr = True if options.infosetValidate: self.modelManager.validateInfoset = True if options.abortOnMajorError: self.modelManager.abortOnMajorError = True if options.collectProfileStats: self.modelManager.collectProfileStats = True if options.internetConnectivity == "offline": self.webCache.workOffline = True elif options.internetConnectivity == "online": self.webCache.workOffline = False if options.internetTimeout is not None: self.webCache.timeout = (options.internetTimeout or None) # use None if zero specified to disable timeout fo = FormulaOptions() if options.parameters: parameterSeparator = (options.parameterSeparator or ',') fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value)) for param in options.parameters.split(parameterSeparator) for key,sep,value in (param.partition('='),) ) ) if options.formulaParamExprResult: fo.traceParameterExpressionResult = True if options.formulaParamInputValue: fo.traceParameterInputValue = True if options.formulaCallExprSource: fo.traceCallExpressionSource = True if options.formulaCallExprCode: fo.traceCallExpressionCode = True if options.formulaCallExprEval: fo.traceCallExpressionEvaluation = True if options.formulaCallExprResult: fo.traceCallExpressionResult = True if options.formulaVarSetExprEval: fo.traceVariableSetExpressionEvaluation = True if options.formulaVarSetExprResult: fo.traceVariableSetExpressionResult = True if options.formulaAsserResultCounts: fo.traceAssertionResultCounts = True if options.formulaFormulaRules: fo.traceFormulaRules = True if options.formulaVarsOrder: fo.traceVariablesOrder = True if options.formulaVarExpressionSource: fo.traceVariableExpressionSource = True if options.formulaVarExpressionCode: fo.traceVariableExpressionCode = True if options.formulaVarExpressionEvaluation: fo.traceVariableExpressionEvaluation = True if options.formulaVarExpressionResult: fo.traceVariableExpressionResult = True if options.timeVariableSetEvaluation: fo.timeVariableSetEvaluation = True if options.formulaVarFilterWinnowing: fo.traceVariableFilterWinnowing = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True self.modelManager.formulaOptions = fo timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) firstStartedAt = startedAt = time.time() modelDiffReport = None success = True modelXbrl = None try: if filesource: modelXbrl = self.modelManager.load(filesource, _("views loading")) except ModelDocument.LoadingException: pass except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl and modelXbrl.modelDocument: loadTime = time.time() - startedAt modelXbrl.profileStat(_("load"), loadTime) self.addToLog(format_string(self.modelManager.locale, _("loaded in %.2f secs at %s"), (loadTime, timeNow)), messageCode="info", file=self.entrypointFile) if options.importFiles: for importFile in options.importFiles.split("|"): fileName = importFile.strip() if sourceZipStream is not None and not (fileName.startswith('http://') or os.path.isabs(fileName)): fileName = os.path.dirname(modelXbrl.uri) + os.sep + fileName # make relative to sourceZipStream ModelDocument.load(modelXbrl, fileName) loadTime = time.time() - startedAt self.addToLog(format_string(self.modelManager.locale, _("import in %.2f secs at %s"), (loadTime, timeNow)), messageCode="info", file=importFile) modelXbrl.profileStat(_("import"), loadTime) if modelXbrl.errors: success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl.modelDocument.type in ModelDocument.Type.TESTCASETYPES: for pluginXbrlMethod in pluginClassMethods("Testcases.Start"): pluginXbrlMethod(self, options, modelXbrl) else: # not a test case, probably instance or DTS for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Loaded"): pluginXbrlMethod(self, options, modelXbrl) else: success = False if success and options.diffFile and options.versReportFile: try: diffFilesource = FileSource.FileSource(options.diffFile,self) startedAt = time.time() modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading")) if modelXbrl2.errors: if not options.keepOpen: modelXbrl2.close() success = False else: loadTime = time.time() - startedAt modelXbrl.profileStat(_("load"), loadTime) self.addToLog(format_string(self.modelManager.locale, _("diff comparison DTS loaded in %.2f secs"), loadTime), messageCode="info", file=self.entrypointFile) startedAt = time.time() modelDiffReport = self.modelManager.compareDTSes(options.versReportFile) diffTime = time.time() - startedAt modelXbrl.profileStat(_("diff"), diffTime) self.addToLog(format_string(self.modelManager.locale, _("compared in %.2f secs"), diffTime), messageCode="info", file=self.entrypointFile) except ModelDocument.LoadingException: success = False except Exception as err: success = False self.addToLog(_("[Exception] Failed to doad diff file: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) if success: try: modelXbrl = self.modelManager.modelXbrl hasFormulae = modelXbrl.hasFormulae if options.validate: startedAt = time.time() if options.formulaAction: # don't automatically run formulas modelXbrl.hasFormulae = False self.modelManager.validate() if options.formulaAction: # restore setting modelXbrl.hasFormulae = hasFormulae self.addToLog(format_string(self.modelManager.locale, _("validated in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if options.formulaAction in ("validate", "run"): # do nothing here if "none" from arelle import ValidateXbrlDimensions, ValidateFormula startedAt = time.time() if not options.validate: ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # setup fresh parameters from formula optoins modelXbrl.parameters = fo.typedParameters() ValidateFormula.validate(modelXbrl, compileOnly=(options.formulaAction != "run")) self.addToLog(format_string(self.modelManager.locale, _("formula validation and execution in %.2f secs") if options.formulaAction == "run" else _("formula validation only in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if options.testReport: ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport, options.testReportCols) if options.rssReport: ViewFileRssFeed.viewRssFeed(self.modelManager.modelXbrl, options.rssReport, options.rssReportCols) if options.DTSFile: ViewFileDTS.viewDTS(modelXbrl, options.DTSFile) if options.factsFile: ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols) if options.factTableFile: ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang) if options.conceptsFile: ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang) if options.preFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang) if options.calFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang) if options.dimFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang) if options.formulaeFile: ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang) if options.viewArcrole and options.viewFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.viewFile, os.path.basename(options.viewArcrole), options.viewArcrole, labelrole=options.labelRole, lang=options.labelLang) for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"): pluginXbrlMethod(self, options, modelXbrl) except (IOError, EnvironmentError) as err: self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err)) success = False except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False if modelXbrl: modelXbrl.profileStat(_("total"), time.time() - firstStartedAt) if options.collectProfileStats and modelXbrl: modelXbrl.logProfileStats() if not options.keepOpen: if modelDiffReport: self.modelManager.close(modelDiffReport) elif modelXbrl: self.modelManager.close(modelXbrl) self.username = self.password = None #dereference password return success
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None, *args, **kwargs): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( targetDocumentFilename, modelXbrl.modelDocument.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus( _("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults( targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in modelXbrl.contexts.values(): newCntx = targetInstance.createContext( context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) newFactForOldObjId[fact.objectIndex] = newFact if filingFiles and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML( "<body>\n{0}\n</body>\n".format(xmltext)): if elt.tag in ("a", "img") and not isHttpUrl( attrValue) and not os.path.isabs( attrvalue): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src"): filingFiles.add(attrValue) except (XMLSyntaxError, UnicodeDecodeError): pass elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) # footnote links footnoteIdCount = {} modelXbrl.modelManager.showStatus( _("Creating and validating footnotes & relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any( lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format( attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) copyIxFootnoteHtml( linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ( "href", "src") and not isHttpUrl( attrValue ) and not os.path.isabs(attrvalue): filingFiles.add(attrValue) targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip) if getattr(modelXbrl, "isTestcaseVariation", False): modelXbrl.extractedInlineInstance = True # for validation comparison modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None, *args, **kwargs): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( targetDocumentFilename, modelXbrl.modelDocument.filepath) def addLocallyReferencedFile(elt, filingFiles): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src") and not isHttpUrl( attrValue) and not os.path.isabs(attrValue): attrValue = attrValue.partition('#')[0] # remove anchor if attrValue: # ignore anchor references to base document attrValue = os.path.normpath( attrValue ) # change url path separators to host separators file = os.path.join(sourceDir, attrValue) if modelXbrl.fileSource.isInArchive( file, checkExistence=True) or os.path.exists(file): filingFiles.add(file) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus( _("Extracting instance ") + os.path.basename(targetUrl)) rootElt = modelXbrl.modelDocument.xmlRootElement # take baseXmlLang from <html> or <base> baseXmlLang = rootElt.get( "{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang") for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants( tag="{http://www.w3.org/1999/xhtml}body"): baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang" ) or rootElt.get("lang") or baseXmlLang targetInstance = ModelXbrl.create( modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True, discover=False) # don't attempt to load DTS if baseXmlLang: targetInstance.modelDocument.xmlRootElement.set( "{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang) ValidateXbrlDimensions.loadDimensionDefaults( targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in sorted(modelXbrl.contexts.values(), key=lambda c: elementChildSequence(c)): ignore = targetInstance.createContext( context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue if fact.concept is not None and fact.concept.baseXsdType in ( "string", "normalizedString"): # default xmlLang = fact.xmlLang if xmlLang is not None and xmlLang != baseXmlLang: attrs[ "{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) # if fact.isFraction, create numerator and denominator newFactForOldObjId[fact.objectIndex] = newFact if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references so that referenced files are included in the zip. for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML("<body>\n{0}\n</body>\n".format( xmltext)).iter(): addLocallyReferencedFile(elt, filingFiles) except (XMLSyntaxError, UnicodeDecodeError): pass # TODO: Why ignore UnicodeDecodeError? elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) modelXbrl.modelManager.showStatus( _("Creating and validating footnotes and relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) footnoteIdCount = {} for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any( lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format( attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) xmlLang = linkChild.xmlLang if xmlLang is not None and xmlLang != baseXmlLang: # default newChild.set( "{http://www.w3.org/XML/1998/namespace}lang", xmlLang) copyIxFootnoteHtml( linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): addLocallyReferencedFile(elt, filingFiles) targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip) if getattr(modelXbrl, "isTestcaseVariation", False): modelXbrl.extractedInlineInstance = True # for validation comparison modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri): from openpyxl import load_workbook from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname try: currentAction = "initializing" startingErrorCount = len(modelXbrl.errors) startedAt = time.time() if os.path.isabs(oimFile): # allow relative filenames to loading directory priorCWD = os.getcwd() os.chdir(os.path.dirname(oimFile)) else: priorCWD = None currentAction = "determining file type" isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json") isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json") isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls") isCSVorXL = isCSV or isXL instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl" if isJSON: currentAction = "loading and parsing JSON OIM file" def loadDict(keyValuePairs): _dict = OrderedDict() # preserve fact order in resulting instance for key, value in keyValuePairs: if isinstance(value, dict): if DUPJSONKEY in value: for _errKey, _errValue, _otherValue in value[DUPJSONKEY]: if key == "prefixes": modelXbrl.error("oime:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"), modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue) del value[DUPJSONKEY] if key in _dict: if DUPJSONKEY not in _dict: _dict[DUPJSONKEY] = [] _dict[DUPJSONKEY].append((key, value, _dict[key])) else: _dict[key] = value return _dict with io.open(oimFile, 'rt', encoding='utf-8') as f: oimObject = json.load(f, object_pairs_hook=loadDict) missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject] if missing: raise OIMException("oime:missingJSONElements", _("Required element(s) are missing from JSON input: %(missing)s"), missing = ", ".join(missing)) currentAction = "identifying JSON objects" dtsReferences = oimObject["dtsReferences"] prefixesList = oimObject["prefixes"].items() facts = oimObject["facts"] footnotes = oimObject["facts"] # shares this object elif isCSV: currentAction = "identifying CSV input tables" if sys.version[0] >= '3': csvOpenMode = 'w' csvOpenNewline = '' else: csvOpenMode = 'wb' # for 2.7 csvOpenNewline = None oimFileBase = None if "-facts" in oimFile: oimFileBase = oimFile.partition("-facts")[0] else: for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"): if oimFile.endswith(suffix): oimFileBase = oimFile[:-len(suffix)] break if oimFileBase is None: raise OIMException("oime:missingCSVTables", _("Unable to identify CSV tables file name pattern")) if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or not os.path.exists(oimFileBase + "-prefixes.csv")): raise OIMException("oime:missingCSVTables", _("Unable to identify CSV tables for dtsReferences or prefixes")) instanceFileName = oimFileBase + ".xbrl" currentAction = "loading CSV dtsReferences table" dtsReferences = [] with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: dtsReferences.append(dict((header[j], col) for j, col in enumerate(row))) currentAction = "loading CSV prefixes table" prefixesList = [] with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = dict((col,i) for i,col in enumerate(row)) else: prefixesList.append((row[header["prefix"]], row[header["URI"]])) defaults = {} if os.path.exists(oimFileBase + "-defaults.csv"): currentAction = "loading CSV defaults table" with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row fileCol = row.index("file") else: defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol) currentAction = "loading CSV facts tables" facts = [] _dir = os.path.dirname(oimFileBase) factsFileBasename = os.path.basename(oimFileBase) + "-facts" for filename in os.listdir(_dir): filepath = os.path.join(_dir, filename) if filename.startswith(factsFileBasename): currentAction = "loading CSV facts table {}".format(filename) tableDefaults = defaults.get(filename, {}) with io.open(filepath, 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col is not None: if header[j]: # skip cols with no header if header[j].endswith("Value"): if col: # ignore empty columns (= null CSV value) fact["value"] = col else: fact[header[j]] = col facts.append(fact) footnotes = [] if os.path.exists(oimFileBase + "-footnotes.csv"): currentAction = "loading CSV footnotes table" with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col)) elif isXL: currentAction = "identifying workbook input worksheets" oimWb = load_workbook(oimFile, read_only=True, data_only=True) sheetNames = oimWb.get_sheet_names() if (not any(sheetName == "prefixes" for sheetName in sheetNames) or not any(sheetName == "dtsReferences" for sheetName in sheetNames) or not any("facts" in sheetName for sheetName in sheetNames)): raise OIMException("oime:missingWorkbookWorksheets", _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts")) currentAction = "loading worksheet: dtsReferences" dtsReferences = [] for i, row in enumerate(oimWb["dtsReferences"]): if i == 0: header = [col.value for col in row] else: dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row))) currentAction = "loading worksheet: prefixes" prefixesList = [] for i, row in enumerate(oimWb["prefixes"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) else: prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value)) defaults = {} if "defaults" in sheetNames: currentAction = "loading worksheet: defaults" for i, row in enumerate(oimWb["defaults"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) fileCol = header["file"] else: defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol) facts = [] for sheetName in sheetNames: if sheetName == "facts" or "-facts" in sheetName: currentAction = "loading worksheet: {}".format(sheetName) tableDefaults = defaults.get(sheetName, {}) for i, row in enumerate(oimWb[sheetName]): if i == 0: header = [col.value for col in row] else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col.value is not None: if header[j]: # skip cols with no header if header[j].endswith("Value"): fact["value"] = str(col.value) else: fact[header[j]] = str(col.value) facts.append(fact) footnotes = [] if "footnotes" in sheetNames: currentAction = "loading worksheet: footnotes" for i, row in enumerate(oimWb["footnotes"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row) if col.value) else: footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value)) currentAction = "identifying default dimensions" ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults currentAction = "validating OIM" prefixes = {} prefixedUris = {} for _prefix, _uri in prefixesList: if not _prefix: modelXbrl.error("oime:emptyPrefix", _("The empty string must not be used as a prefix, uri %(uri)s"), modelObject=modelXbrl, uri=_uri) elif not NCNamePattern.match(_prefix): modelXbrl.error("oime:prefixPattern", _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"), modelObject=modelXbrl, prefix=_prefix, uri=_uri) elif _prefix in prefixes: modelXbrl.error("oime:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"), modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri) elif _uri in prefixedUris: modelXbrl.error("oime:duplicatedUri", _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"), modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix) else: prefixes[_prefix] = _uri prefixedUris[_uri] = _prefix oimPrefix = None for _nsOim in nsOim: if _nsOim in prefixedUris: oimPrefix = prefixedUris[_nsOim] if not oimPrefix: raise OIMException("oime:noOimPrefix", _("The oim namespace must have a declared prefix")) oimConcept = "{}:concept".format(oimPrefix) oimEntity = "{}:entity".format(oimPrefix) oimPeriod = "{}:period".format(oimPrefix) oimPeriodStart = "{}:periodStart".format(oimPrefix) oimPeriodEnd = "{}:periodEnd".format(oimPrefix) oimUnit = "{}:unit".format(oimPrefix) oimPrefix = "{}:".format(oimPrefix) # create the instance document currentAction = "creating instance document" modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument( modelXbrl, Type.INSTANCE, instanceFileName, schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"], isEntry=True, initialComment="extracted from OIM {}".format(mappedUri), documentEncoding="utf-8") cntxTbl = {} unitTbl = {} currentAction = "creating facts" for fact in facts: conceptQn = qname(fact[oimConcept], prefixes) concept = modelXbrl.qnameConcepts.get(conceptQn) entityAsQn = qname(fact[oimEntity], prefixes) if oimPeriod in fact: periodStart = fact[oimPeriod]["start"] periodEnd = fact[oimPeriod]["end"] else: periodStart = fact[oimPeriodStart] periodEnd = fact[oimPeriodEnd] cntxKey = ( # hashable context key ("periodType", concept.periodType), ("entity", entityAsQn), ("periodStart", periodStart), ("periodEnd", periodEnd)) + tuple(sorted( (dimName, dimVal) for dimName, dimVal in fact.items() if ":" in dimName and not dimName.startswith(oimPrefix))) if cntxKey in cntxTbl: _cntx = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) qnameDims = {} for dimName, dimVal in fact.items(): if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal: dimQname = qname(dimName, prefixes) dimConcept = modelXbrl.qnameConcepts.get(dimQname) if ":" in dimVal and dimVal.partition(':')[0] in prefixes: mem = qname(dimVal, prefixes) # explicit dim elif dimConcept.isTypedDimension: # a modelObject xml element is needed for all of the instance functions to manage the typed dim mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment") _cntx = modelXbrl.createContext( entityAsQn.namespaceURI, entityAsQn.localName, concept.periodType, None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME), dateTime(periodEnd, type=DATETIME), None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) cntxTbl[cntxKey] = _cntx if oimUnit in fact: unitKey = fact[oimUnit] if unitKey in unitTbl: _unit = unitTbl[unitKey] else: _unit = None # validate unit unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey) if not UnitPattern.match(unitKeySub): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation is lexically invalid, %(unit)s"), modelObject=modelXbrl, unit=unitKey) else: _mul, _sep, _div = unitKey.partition('/') if _mul.startswith('('): _mul = _mul[1:-1] _muls = [u for u in _mul.split('*') if u] if _div.startswith('('): _div = _div[1:-1] _divs = [u for u in _div.split('*') if u] if _muls != sorted(_muls) or _divs != sorted(_divs): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation measures are not in alphabetical order, %(unit)s"), modelObject=modelXbrl, unit=unitKey) try: mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _muls] divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _divs] unitId = 'u-{:02}'.format(len(unitTbl) + 1) for _measures in mulQns, divQns: for _measure in _measures: addQnameValue(modelXbrl.modelDocument, _measure) _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId) except OIMException as ex: modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) unitTbl[unitKey] = _unit else: _unit = None attrs = {"contextRef": _cntx.id} if fact.get("value") is None: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact["value"] if fact.get("id"): attrs["id"] = fact["id"] if concept.isNumeric: if _unit is None: continue # skip creating fact because unit was invalid attrs["unitRef"] = _unit.id if "accuracy" in fact: attrs["decimals"] = fact["accuracy"] # is value a QName? if concept.baseXbrliType == "QName": addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes)) f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text) currentAction = "creating footnotes" footnoteLinks = {} # ELR elements factLocs = {} # index by (linkrole, factId) footnoteNbr = 0 locNbr = 0 for factOrFootnote in footnotes: if "factId" in factOrFootnote: factId = factOrFootnote["factId"] factFootnotes = (factOrFootnote,) # CSV or XL elif "id" in factOrFootnote and "footnotes" in factOrFootnote: factId = factOrFootnote["id"] factFootnotes = factOrFootnote["footnotes"] else: factFootnotes = () for footnote in factFootnotes: linkrole = footnote.get("group") arcrole = footnote.get("footnoteType") if not factId or not linkrole or not arcrole or not ( footnote.get("factRef") or footnote.get("footnote")): # invalid footnote continue if linkrole not in footnoteLinks: footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, XbrlConst.qnLinkFootnoteLink, attributes={"{http://www.w3.org/1999/xlink}type": "extended", "{http://www.w3.org/1999/xlink}role": linkrole}) footnoteLink = footnoteLinks[linkrole] if (linkrole, factId) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factId)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factId, XLINKLABEL: locLabel}) locLabel = factLocs[(linkrole, factId)] if footnote.get("footnote"): footnoteNbr += 1 footnoteLabel = "f_{:02}".format(footnoteNbr) attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel} if footnote.get("language"): attrs[XMLLANG] = footnote["language"] # note, for HTML will need to build an element structure addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"]) elif footnote.get("factRef"): factRef = footnote.get("factRef") if (linkrole, factRef) not in factLocs: locNbr += 1 locLabel = "f_{:02}".format(footnoteNbr) factLoc[(linkrole, factRef)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factRef, XLINKLABEL: locLabel}) footnoteLabel = factLoc[(linkrole, factId)] footnoteArc = addChild(footnoteLink, XbrlConst.qnLinkFootnoteArc, attributes={XLINKTYPE: "arc", XLINKARCROLE: arcrole, XLINKFROM: locLabel, XLINKTO: footnoteLabel}) currentAction = "done loading facts and footnotes" #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") except Exception as ex: if isinstance(ex, OIMException): modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) else: modelXbrl.error("arelleOIMloader:error", "Error while %(action)s, error %(error)s\ntraceback %(traceback)s", modelObject=modelXbrl, action=currentAction, error=ex, traceback=traceback.format_tb(sys.exc_info()[2])) if priorCWD: os.chdir(priorCWD) # restore prior current working directory startingErrorCount = len(modelXbrl.errors) if startingErrorCount < len(modelXbrl.errors): # had errors, don't allow ModelDocument.load to continue return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors") return getattr(modelXbrl, "modelDocument", None) # none if returning from exception
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None, *args, **kwargs): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath) def addLocallyReferencedFile(elt,filingFiles): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrValue): attrValue = attrValue.partition('#')[0] # remove anchor if attrValue: # ignore anchor references to base document attrValue = os.path.normpath(attrValue) # change url path separators to host separators file = os.path.join(sourceDir,attrValue) if modelXbrl.fileSource.isInArchive(file, checkExistence=True) or os.path.exists(file): filingFiles.add(file) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl)) rootElt = modelXbrl.modelDocument.xmlRootElement # take baseXmlLang from <html> or <base> baseXmlLang = rootElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang") for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/1999/xhtml}body"): baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang") or baseXmlLang targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True, discover=False) # don't attempt to load DTS if baseXmlLang: targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang) ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in sorted(modelXbrl.contexts.values(), key=lambda c: elementChildSequence(c)): ignore = targetInstance.createContext(context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default xmlLang = fact.xmlLang if xmlLang is not None and xmlLang != baseXmlLang: attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) # if fact.isFraction, create numerator and denominator newFactForOldObjId[fact.objectIndex] = newFact if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references so that referenced files are included in the zip. for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter(): addLocallyReferencedFile(elt, filingFiles) except (XMLSyntaxError, UnicodeDecodeError): pass # TODO: Why ignore UnicodeDecodeError? elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) footnoteIdCount = {} for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format(attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) xmlLang = linkChild.xmlLang if xmlLang is not None and xmlLang != baseXmlLang: # default newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang) copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): addLocallyReferencedFile(elt,filingFiles) targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip) if getattr(modelXbrl, "isTestcaseVariation", False): modelXbrl.extractedInlineInstance = True # for validation comparison modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri): from openpyxl import load_workbook from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname _return = None # modelDocument or an exception try: currentAction = "initializing" startingErrorCount = len(modelXbrl.errors) startedAt = time.time() if os.path.isabs(oimFile): # allow relative filenames to loading directory priorCWD = os.getcwd() os.chdir(os.path.dirname(oimFile)) else: priorCWD = None currentAction = "determining file type" isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") # this option is not currently supported isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls") isCSVorXL = isCSV or isXL instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl" _csvwContext = None if isJSON: errPrefix = "xbrlje" currentAction = "loading and parsing JSON OIM file" def loadDict(keyValuePairs): _dict = OrderedDict() # preserve fact order in resulting instance for key, value in keyValuePairs: if isinstance(value, dict): if DUPJSONKEY in value: for _errKey, _errValue, _otherValue in value[DUPJSONKEY]: if key == "prefixes": modelXbrl.error("xbrlje:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"), modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue) del value[DUPJSONKEY] if key in _dict: if DUPJSONKEY not in _dict: _dict[DUPJSONKEY] = [] _dict[DUPJSONKEY].append((key, value, _dict[key])) else: _dict[key] = value return _dict with io.open(oimFile, 'rt', encoding='utf-8') as f: oimObject = json.load(f, object_pairs_hook=loadDict) # check if it's a CSVW metadata _csvwContext = oimObject.get("@context") if _csvwContext == "http://www.w3.org/ns/csvw" or ( isinstance(_csvwContext, list) and "http://www.w3.org/ns/csvw" in _csvwContext): isJSON = False isCSV = True else: if oimObject.get("documentType", None) != "http://www.xbrl.org/WGWD/YYYY-MM-DD/xbrl-json": # is it likely to be an OIM document? if any(t in oimObject for t in ("dtsReferences", "prefixes", "facts")): raise OIMException("xbrlje:missingJSONdocumentType", _("Required documentType is missing from JSON input")) else: raise NotOIMException() # return None, not an OIM document missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject] if missing: raise OIMException("xbrlje:missingJSONElements", _("Required element(s) are missing from JSON input: %(missing)s"), missing = ", ".join(missing)) currentAction = "identifying JSON objects" dtsReferences = oimObject["dtsReferences"] prefixesList = oimObject["prefixes"].items() facts = oimObject["facts"] footnotes = oimObject["facts"] # shares this object if isCSV: errPrefix = "xbrlce" currentAction = "compiling metadata" if sys.version[0] >= '3': csvOpenMode = 'w' csvOpenNewline = '' else: csvOpenMode = 'wb' # for 2.7 csvOpenNewline = None if not _csvwContext: # json metadata wasn't specified, try to find it raise OIMException("xbrlce:missingCSVMetadata", _("Unable to identify CSV metadata file")) # process CSV metadata # mandatory sections of metadata file oimMetadata = oimObject.get(CSVmetadata) if not oimMetadata: raise OIMException("xbrlce:missingOIMMetadata", _("Unable to identify OIM metadata infile")) if oimMetadata.get("documentType") != CSVdocumentType: raise OIMException("xbrlce:documentType", _("Document type %(documentType)s not recognized, expecting %(expectedDocumentType)s"), documentType=oimMetadata.get("documentType"), expectedDocumentType=CSVdocumentType) dtsReferences = oimMetadata.get("dtsReferences", {}) prefixesList = oimMetadata.get("prefixes", {}).items() topLevelAspects = oimObject.get(CSVaspects, {}) currentAction = "loading CSV facts tables" facts = [] footnotes = [] _dir = os.path.dirname(oimFile) for oimTable in oimObject.get("tables", []): tableType = oimTable.get(CSVtableType) tableLevelAspects = oimTable.get(CSVaspects, {}) tableUrl = oimTable.get("url") tableColumns = oimTable.get("tableSchema",{}).get("columns", []) # compile column dependencies aspectCols = [] factCols = [] for iCol, col in enumerate(tableColumns): if CSVcolumnAspect in col: aspectCols.append(iCol) if CSVtupleFactAspects in col or CSVsimpleFactAspects in col: factCols.append(iCol) if tableType not in ("fact", "footnote"): modelXbrl.error("xbrlce:unrecognizedTableType", _("Table type %(tableType)s was not recognized, table URI %(uri)s."), modelObject=modelXbrl, uri=oimFile, tableType=tableType) continue if not tableColumns: modelXbrl.error("xbrlce:noTableColumns", _("Table has no columns, table URI %(uri)s."), modelObject=modelXbrl, uri=_uri) continue tableAspects = topLevelAspects.copy() tableAspects.update(tableLevelAspects) filepath = os.path.join(_dir, tableUrl) tupleIds = set() with io.open(filepath, 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: if tableType == "fact": colAspects = tableAspects.copy() specificColAspects = defaultdict(dict) for iCol in aspectCols: value = row[iCol] aspect = tableColumns[iCol][CSVcolumnAspect] if isinstance(aspect, str): # applies to all cols colAspects[aspect] = value elif isinstance(aspect, dict): # applies to specific cols for _aspect, _colNames in aspect.items(): for _colName in _colNames: specificColAspects[_colName][_aspect] = value for iCol in factCols: fact = {"aspects": {}} cellValue = row[iCol] if cellValue == "": # no fact produced for this cell continue tableCol = tableColumns[iCol] if CSVtupleFactAspects in tableCol: aspectsObjectName = CSVtupleFactAspects else: aspectsObjectName = CSVsimpleFactAspects cellAspects = (colAspects, specificColAspects.get(tableCol.get("name"), EMPTYDICT), tableColumns[iCol].get(aspectsObjectName), EMPTYDICT) inapplicableAspects = set() if tableCol.get(CSVtupleReferenceId) == "true": if cellValue: if cellValue in tupleIds: continue # don't duplicate parent tuple fact["id"] = cellValue tupleIds.add(cellValue) # prevent tuple duplication elif CSVsimpleFactAspects in tableCol: fact["value"] = csvCellValue(cellValue) if CSVtupleFactAspects in tableCol: inapplicableAspects.update(oimSimpleFactAspects) for _aspects in cellAspects: for aspectName, aspectValue in _aspects.items(): if not aspectName.startswith(oimPrefix): inapplicableAspects.add(aspectName) # block any row aspect produced by this column from this column's fact _colAspect = tableCol.get(CSVcolumnAspect) if isinstance(_colAspect, str): # applies to all cols inapplicableAspects.add(_colAspect) for _aspects in cellAspects: if _aspects: for aspectName, aspectValue in _aspects.items(): if aspectName not in inapplicableAspects and aspectValue != "": if ":" in aspectName: fact["aspects"][aspectName] = csvCellValue(aspectValue) else: fact[aspectName] = aspectValue facts.append(fact) elif tableType == "footnote": footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col)) del tupleIds elif isXL: errPrefix = "xbrlwe" currentAction = "identifying workbook input worksheets" oimWb = load_workbook(oimFile, read_only=True, data_only=True) sheetNames = oimWb.get_sheet_names() if (not any(sheetName == "prefixes" for sheetName in sheetNames) or not any(sheetName == "dtsReferences" for sheetName in sheetNames) or not any("facts" in sheetName for sheetName in sheetNames)): raise OIMException("xbrlwe:missingWorkbookWorksheets", _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts")) currentAction = "loading worksheet: dtsReferences" dtsReferences = [] for i, row in enumerate(oimWb["dtsReferences"]): if i == 0: header = [col.value for col in row] else: dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row))) currentAction = "loading worksheet: prefixes" prefixesList = [] for i, row in enumerate(oimWb["prefixes"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) else: prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value)) defaults = {} if "defaults" in sheetNames: currentAction = "loading worksheet: defaults" for i, row in enumerate(oimWb["defaults"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) fileCol = header["file"] else: defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol) facts = [] for sheetName in sheetNames: if sheetName == "facts" or "-facts" in sheetName: currentAction = "loading worksheet: {}".format(sheetName) tableDefaults = defaults.get(sheetName, {}) for i, row in enumerate(oimWb[sheetName]): if i == 0: header = [col.value for col in row] else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col.value is not None: if header[j]: # skip cols with no header if header[j].endswith("Value"): fact["value"] = str(col.value) else: fact[header[j]] = str(col.value) facts.append(fact) footnotes = [] if "footnotes" in sheetNames: currentAction = "loading worksheet: footnotes" for i, row in enumerate(oimWb["footnotes"]): if i == 0: header = [col.value for j,col in enumerate(row) if col.value] else: footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value)) currentAction = "identifying default dimensions" ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults currentAction = "validating OIM" prefixes = {} prefixedUris = {} for _prefix, _uri in prefixesList: if not _prefix: modelXbrl.error("{}:emptyPrefix".format(errPrefix), _("The empty string must not be used as a prefix, uri %(uri)s"), modelObject=modelXbrl, uri=_uri) elif not NCNamePattern.match(_prefix): modelXbrl.error("oime:prefixPattern", _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"), modelObject=modelXbrl, prefix=_prefix, uri=_uri) elif _prefix in prefixes: modelXbrl.error("{}:duplicatedPrefix".format(errPrefix), _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"), modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri) elif _uri in prefixedUris: modelXbrl.error("{}:duplicatedUri".format(errPrefix), _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"), modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix) else: prefixes[_prefix] = _uri prefixedUris[_uri] = _prefix for _nsOim in nsOim: if _nsOim in prefixedUris: _oimPrefix = prefixedUris[_nsOim] if not _oimPrefix: raise OIMException("oime:noOimPrefix", _("The oim namespace must have a declared prefix")) # create the instance document currentAction = "creating instance document" modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = _return = createModelDocument( modelXbrl, Type.INSTANCE, instanceFileName, schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef.get("type") == "schema" and dtsRef.get("href")], isEntry=True, initialComment="extracted from OIM {}".format(mappedUri), documentEncoding="utf-8") modelXbrl.modelDocument.inDTS = True # add linkbase, role and arcrole refs for refType in ("linkbase", "role", "arcrole"): for dtsRef in dtsReferences: if dtsRef.get("type") == refType and dtsRef.get("href"): elt = addChild(modelXbrl.modelDocument.xmlRootElement, qname(link, refType+"Ref"), attributes=(("{http://www.w3.org/1999/xlink}href", dtsRef["href"]), ("{http://www.w3.org/1999/xlink}type", "simple"))) href = modelXbrl.modelDocument.discoverHref(elt) if href: _elt, hrefDoc, hrefId = href _defElt = hrefDoc.idObjects.get(hrefId) _uriAttrName = refType + "URI" _uriAttrValue = _defElt.get(_uriAttrName) if _defElt is not None and _uriAttrValue: elt.set(_uriAttrName, _uriAttrValue) cntxTbl = {} unitTbl = {} # determine tuple dependency order idFacts = {} parentedFacts = defaultdict(list) for i, fact in enumerate(facts): aspects = fact["aspects"] id = fact.get("id") tupleParent = fact.get("aspects", EMPTYDICT).get(oimTupleParent) if id is not None: idFacts[id] = i parentedFacts[tupleParent].append(i) # root facts have tupleParent None currentAction = "creating facts" def createModelFact(fact, parentModelFact, topTupleFact): aspects = fact.get("aspects", EMPTYDICT) if oimConcept not in aspects: modelXbrl.error("{}:conceptQName".format(errPrefix), _("The concept QName could not be determined"), modelObject=modelXbrl) return conceptQn = qname(aspects[oimConcept], prefixes) concept = modelXbrl.qnameConcepts.get(conceptQn) attrs = {} if concept.isItem: missingAspects = [] if oimEntity not in aspects: missingAspects.append(oimEntity) if oimPeriod not in aspects and (oimPeriodStart not in aspects or oimPeriodEnd not in aspects): missingAspects.append(oimPeriod) if missingAspects: modelXbrl.error("{}:missingAspects".format(errPrefix), _("The concept %(element)s is missing aspects %(missingAspects)s"), modelObject=modelXbrl, element=conceptQn, missingAspects=", ".join(missingAspects)) return entityAsQn = qname(aspects[oimEntity], prefixes) or qname("error",fact[oimEntity]) if oimPeriod in aspects: periodStart = periodEnd = aspects[oimPeriod] if oimPeriodStart in aspects and oimPeriodEnd in aspects: periodStart = aspects[oimPeriodStart] periodEnd = aspects[oimPeriodEnd] cntxKey = ( # hashable context key ("periodType", concept.periodType), ("entity", entityAsQn), ("periodStart", periodStart), ("periodEnd", periodEnd)) + tuple(sorted( (dimName, dimVal["value"] if isinstance(dimVal,dict) else dimVal) for dimName, dimVal in aspects.items() if ":" in dimName and not dimName.startswith(oimPrefix))) if cntxKey in cntxTbl: _cntx = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) qnameDims = {} for dimName, dimVal in aspects.items(): if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal: dimQname = qname(dimName, prefixes) dimConcept = modelXbrl.qnameConcepts.get(dimQname) if dimConcept is None: modelXbrl.error("{}:taxonomyDefinedAspectQName".format(errPrefix), _("The taxonomy defined aspect concept QName %(qname)s could not be determined"), modelObject=modelXbrl, qname=dimQname) continue if isinstance(dimVal, dict): dimVal = dimVal["value"] else: dimVal = str(dimVal) # may be int or boolean if isinstance(dimVal,str) and ":" in dimVal and dimVal.partition(':')[0] in prefixes: mem = qname(dimVal, prefixes) # explicit dim elif dimConcept.isTypedDimension: # a modelObject xml element is needed for all of the instance functions to manage the typed dim mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment") _cntx = modelXbrl.createContext( entityAsQn.namespaceURI, entityAsQn.localName, concept.periodType, None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME), dateTime(periodEnd, type=DATETIME), None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId, beforeSibling=topTupleFact) cntxTbl[cntxKey] = _cntx if oimUnit in aspects and concept.isNumeric: unitKey = aspects[oimUnit] if unitKey in unitTbl: _unit = unitTbl[unitKey] else: _unit = None # validate unit unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey) if not UnitPattern.match(unitKeySub): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation is lexically invalid, %(unit)s"), modelObject=modelXbrl, unit=unitKey) else: _mul, _sep, _div = unitKey.partition('/') if _mul.startswith('('): _mul = _mul[1:-1] _muls = [u for u in _mul.split('*') if u] if _div.startswith('('): _div = _div[1:-1] _divs = [u for u in _div.split('*') if u] if _muls != sorted(_muls) or _divs != sorted(_divs): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation measures are not in alphabetical order, %(unit)s"), modelObject=modelXbrl, unit=unitKey) try: mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _muls] divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _divs] unitId = 'u-{:02}'.format(len(unitTbl) + 1) for _measures in mulQns, divQns: for _measure in _measures: addQnameValue(modelXbrl.modelDocument, _measure) _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId, beforeSibling=topTupleFact) except OIMException as ex: modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) unitTbl[unitKey] = _unit else: _unit = None attrs["contextRef"] = _cntx.id if fact.get("value") is None: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact["value"] if concept.isNumeric: if _unit is None: return # skip creating fact because unit was invalid attrs["unitRef"] = _unit.id if "accuracy" in attrs or attrs.get(XbrlConst.qnXsiNil, "false") != "true": attrs["decimals"] = fact.get("accuracy", "INF") else: text = None #tuple id = fact.get("id") if id is not None: attrs["id"] = fact["id"] # is value a QName? if concept.baseXbrliType == "QName": addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes)) f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text, parent=parentModelFact, validate=False) if id is not None and id in parentedFacts: # create child facts for i in sorted(parentedFacts[id], key=lambda j: facts[j].get("aspects", EMPTYDICT).get(oimTupleOrder,0)): createModelFact(facts[i], f, f if topTupleFact is None else topTupleFact) # validate after creating tuple contents xmlValidate(modelXbrl, f) for i in parentedFacts[None]: createModelFact(facts[i], None, None) currentAction = "creating footnotes" footnoteLinks = OrderedDict() # ELR elements factLocs = {} # index by (linkrole, factId) footnoteNbr = 0 locNbr = 0 for factOrFootnote in footnotes: if "factId" in factOrFootnote: factId = factOrFootnote["factId"] factFootnotes = (factOrFootnote,) # CSV or XL elif "id" in factOrFootnote and "footnotes" in factOrFootnote: factId = factOrFootnote["id"] factFootnotes = factOrFootnote["footnotes"] else: factFootnotes = () for footnote in factFootnotes: linkrole = footnote.get("group") arcrole = footnote.get("footnoteType") if not factId or not linkrole or not arcrole or not ( footnote.get("factRef") or footnote.get("footnote")): # invalid footnote continue if linkrole not in footnoteLinks: footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, XbrlConst.qnLinkFootnoteLink, attributes={"{http://www.w3.org/1999/xlink}type": "extended", "{http://www.w3.org/1999/xlink}role": linkrole}) footnoteLink = footnoteLinks[linkrole] if (linkrole, factId) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factId)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factId, XLINKLABEL: locLabel}) locFromLabel = factLocs[(linkrole, factId)] if footnote.get("footnote"): footnoteNbr += 1 footnoteToLabel = "f_{:02}".format(footnoteNbr) attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteToLabel} if footnote.get("language"): attrs[XMLLANG] = footnote["language"] # note, for HTML will need to build an element structure addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"]) elif footnote.get("factRef"): factRef = footnote.get("factRef") if (linkrole, factRef) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factRef)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factRef, XLINKLABEL: locLabel}) footnoteToLabel = factLocs[(linkrole, factRef)] footnoteArc = addChild(footnoteLink, XbrlConst.qnLinkFootnoteArc, attributes={XLINKTYPE: "arc", XLINKARCROLE: arcrole, XLINKFROM: locFromLabel, XLINKTO: footnoteToLabel}) if footnoteLinks: modelXbrl.modelDocument.linkbaseDiscover(footnoteLinks.values(), inInstance=True) currentAction = "done loading facts and footnotes" #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") except NotOIMException as ex: _return = ex # not an OIM document except Exception as ex: _return = ex if isinstance(ex, OIMException): modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) else: modelXbrl.error("arelleOIMloader:error", "Error while %(action)s, error %(error)s\ntraceback %(traceback)s", modelObject=modelXbrl, action=currentAction, error=ex, traceback=traceback.format_tb(sys.exc_info()[2])) if priorCWD: os.chdir(priorCWD) # restore prior current working directory startingErrorCount = len(modelXbrl.errors) #if startingErrorCount < len(modelXbrl.errors): # # had errors, don't allow ModelDocument.load to continue # return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors") return _return
def loadXbrlFromDB(self, loadDBsaveToFile): # load from database modelXbrl = self.modelXbrl # find instance in DB instanceURI = os.path.basename(loadDBsaveToFile) results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant" " FROM dInstance WHERE FileName = '{}'" .format(instanceURI)) instanceId = moduleId = None for instanceId, moduleId, entScheme, entId, datePerEnd in results: break # find module in DB results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId)) xbrlSchemaRef = None for result in results: xbrlSchemaRef = result[0] break if not instanceId or not xbrlSchemaRef: raise XPDBException("sqlDB:MissingDTS", _("The instance and module were not found for %(instanceURI)"), instanceURI = instanceURI) if modelXbrl.skipDTS: # find prefixes and namespaces in DB results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]") dpmPrefixedNamespaces = dict((prefix, namespace) for owner, prefix, namespace in results) # create the instance document and resulting filing modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument(modelXbrl, Type.INSTANCE, loadDBsaveToFile, schemaRefs=[xbrlSchemaRef], isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement, 'xbrl-streamable-instance', 'version="1.0" contextBuffer="1"') # add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue) # filing indicator code IDs # get filing indicators results = self.execute("SELECT mToT.TemplateOrTableCode " " FROM dFilingIndicator dFI, mTemplateOrTable mToT " " WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID" .format(instanceId)) filingIndicatorCodes = [code[0] for code in results] if filingIndicatorCodes: modelXbrl.createContext(entScheme, entId, 'instant', None, datePerEnd, None, # no dimensional validity checking (like formula does) {}, [], [], id='c') filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators) for filingIndicatorCode in filingIndicatorCodes: modelXbrl.createFact(qnFindFilingIndicator, parent=filingIndicatorsTuple, attributes={"contextRef": "c"}, text=filingIndicatorCode) # facts in this instance factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards," " Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue " "FROM dFact WHERE InstanceID = {} " "ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL " " THEN DataPointSignature" " ELSE DataPointSignatureWithValuesForWildcards" " END, instr(DataPointSignature,'|') + 1)" .format(instanceId)) # results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal # get typed dimension values prefixedNamespaces = modelXbrl.prefixedNamespaces prefixedNamespaces["iso4217"] = XbrlConst.iso4217 if modelXbrl.skipDTS: prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed cntxTbl = {} # index by d unitTbl = {} def typedDimElt(s): # add xmlns into s for known qnames tag, angleBrkt, rest = s[1:].partition('>') text, angleBrkt, rest = rest.partition("<") qn = qname(tag, prefixedNamespaces) # a modelObject xml element is needed for all of the instance functions to manage the typed dim return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False) # contexts and facts for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl: metric, sep, dims = (dpSigTypedDims or dpSig).partition('|') conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces) concept = modelXbrl.qnameConcepts.get(conceptQn) isNumeric = isBool = isDateTime = isQName = isText = False if concept is not None: if concept.isNumeric: isNumeric = True else: baseXbrliType = concept.baseXbrliType if baseXbrliType == "booleanItemType": isBool = True elif baseXbrliType == "dateTimeItemType": # also is dateItemType? isDateTime = True elif baseXbrliType == "QNameItemType": isQName = True else: c = conceptQn.localName[0] if c == 'm': isNumeric = True elif c == 'd': isDateTime = True elif c == 'b': isBool = True elif c == 'e': isQName = True isText = not (isNumeric or isBool or isDateTime or isQName) if isinstance(datePerEnd, _STR_BASE): datePerEnd = datetimeValue(datePerEnd, addOneDay=True) cntxKey = (dims, entId, datePerEnd) if cntxKey in cntxTbl: cntxId = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) cntxTbl[cntxKey] = cntxId qnameDims = {} if dims: for dim in dims.split('|'): dQn, sep, dVal = dim[:-1].partition('(') dimQname = qname(dQn, prefixedNamespaces) if dVal.startswith('<'): # typed dim mem = typedDimElt(dVal) else: mem = qname(dVal, prefixedNamespaces) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario") modelXbrl.createContext(entScheme, entId, 'instant', None, datePerEnd, None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) if unit: if unit in unitTbl: unitId = unitTbl[unit] else: unitQn = qname(unit, prefixedNamespaces) unitId = 'u{}'.format(unitQn.localName) unitTbl[unit] = unitId modelXbrl.createUnit([unitQn], [], id=unitId) else: unitId = None attrs = {"contextRef": cntxId} if unitId: attrs["unitRef"] = unitId if dec is not None: if isinstance(dec, float): # must be an integer dec = int(dec) elif isinstance(dec, _STR_BASE) and '.' in dec: dec = dec.partition('.')[0] # drop .0 from any SQLite string attrs["decimals"] = str(dec) # somehow it is float from the database if False: # fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None elif numVal is not None: num = roundValue(numVal, None, dec) # round using reported decimals if dec is None or dec == "INF": # show using decimals or reported format dec = len(numVal.partition(".")[2]) else: # max decimals at 28 dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num)) elif dateVal is not None: text = dateVal elif boolVal is not None: text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false' else: if isQName: # declare namespace addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces)) text = textVal modelXbrl.createFact(conceptQn, attributes=attrs, text=text) # add footnotes if any # save to file modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000) return modelXbrl.modelDocument
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath): # check if big instance and has header with an initial incomplete tree walk (just 2 elements def logSyntaxErrors(parsercontext): for error in parsercontext.error_log: modelXbrl.error("xmlSchema:syntax", _("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"), modelObject=modelDocument, fileName=os.path.basename(filepath), error=error.message, line=error.line, column=error.column, sourceAction="streaming") #### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ### #### note: iterparse wants a binary file, but file is text mode _file, = modelXbrl.fileSource.file(filepath, binary=True) startedAt = time.time() modelXbrl.profileActivity() parsercontext = etree.iterparse(_file, events=("start","end"), huge_tree=True) foundInstance = False foundErrors = False streamingAspects = None numRootFacts1 = 0 numElts = 0 elt = None for event, elt in parsercontext: if event == "start": if elt.getparent() is not None: if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl": if not foundInstance: foundInstance = True pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance") if pi is None: break else: streamingAspects = dict(pi.attrib.copy()) if not elt.tag.startswith("{http://www.xbrl.org/"): numRootFacts1 += 1 if numRootFacts1 % 1000 == 0: modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0) elif not foundInstance: break elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl" and precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None: modelXbrl.error("streamingExtensions:headerMisplaced", _("Header is misplaced: %(error)s, must follow xbrli:xbrl element"), modelObject=elt) elif event == "end": elt.clear() numElts += 1 if numElts % 1000 == 0 and elt.getparent() is not None: while elt.getprevious() is not None and elt.getparent() is not None: del elt.getparent()[0] if elt is not None: elt.clear() _file.seek(0,io.SEEK_SET) # allow reparsing if not foundInstance or streamingAspects is None: del elt, parsercontext _file.close() return None modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt) startedAt = time.time() try: version = Decimal(streamingAspects.get("version")) if int(version) != 1: modelXbrl.error("streamingExtensions:unsupportedVersion", _("Streaming version %(version)s, major version number must be 1"), modelObject=elt, version=version) foundErrors = True except (InvalidOperation, OverflowError): modelXbrl.error("streamingExtensions:versionError", _("Version %(version)s, number must be 1.n"), modelObject=elt, version=streamingAspects.get("version", "(none)")) foundErrors = True for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"): try: bufLimit = Decimal(streamingAspects.get(bufAspect, "INF")) if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0): raise InvalidOperation elif bufAspect == "contextBuffer": contextBufferLimit = bufLimit elif bufAspect == "unitBuffer": unitBufferLimit = bufLimit elif bufAspect == "footnoteBuffer": footnoteBufferLimit = bufLimit except InvalidOperation: modelXbrl.error("streamingExtensions:valueError", _("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"), modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect)) foundErrors = True if parsercontext.error_log: foundErrors = True logSyntaxErrors(parsercontext) if foundErrors: _file.close() return None parsercontext = etree.iterparse(_file, events=("start","end"), huge_tree=True) _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl,filepath) eltMdlObjs = {} beforeInstanceStream = True validator = None contextBuffer = [] unitBuffer = [] footnoteBuffer = [] factBuffer = [] numFacts = numRootFacts2 = 1 for event, elt in parsercontext: if event == "start": mdlObj = _parser.makeelement(elt.tag, attrib=elt.attrib, nsmap=elt.nsmap) mdlObj.sourceline = elt.sourceline eltMdlObjs[elt] = mdlObj if elt.getparent() is None: modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, etree.ElementTree(mdlObj)) modelDocument.xmlRootElement = mdlObj modelXbrl.modelDocument = modelDocument # needed for incremental validation mdlObj.init(modelDocument) modelXbrl.info("streamingExtensions:streaming", _("Stream processing this instance."), modelObject = modelDocument) else: eltMdlObjs[elt.getparent()].append(mdlObj) mdlObj._init() ns = mdlObj.namespaceURI ln = mdlObj.localName if (beforeInstanceStream and ( (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or (ns == XbrlConst.xbrli and ln in ("context", "unit")) or (ns not in (XbrlConst.link, XbrlConst.xbrli)))): beforeInstanceStream = False if _streamingExtensionsValidate: validator = Validate(modelXbrl) validator.instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters()) else: # need default dimensions ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) mdlObj = None # deref elif event == "end": mdlObj = eltMdlObjs.pop(elt) if elt.text: # text available after child nodes processed mdlObj.text = elt.text ns = mdlObj.namespaceURI ln = mdlObj.localName parentMdlObj = mdlObj.getparent() if ns == XbrlConst.xbrli: if ln == "context": if mdlObj.get("sticky"): del mdlObj.attrib["sticky"] modelDocument.contextDiscover(mdlObj) else: if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit: # drop before adding as dropped may have same id as added cntx = contextBuffer.pop(0) dropContext(modelXbrl, cntx) del parentMdlObj[parentMdlObj.index(cntx)] cntx = None modelDocument.contextDiscover(mdlObj) if contextBufferLimit.is_finite(): contextBuffer.append(mdlObj) if _streamingExtensionsValidate: contextsToCheck = (mdlObj,) validator.instValidator.checkContexts(contextsToCheck) if modelXbrl.hasXDT: validator.instValidator.checkContextsDimensions(contextsToCheck) del contextsToCheck # dereference elif ln == "unit": if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit: # drop before additing as dropped may have same id as added unit = unitBuffer.pop(0) dropUnit(modelXbrl, unit) del parentMdlObj[parentMdlObj.index(unit)] unit = None modelDocument.unitDiscover(mdlObj) if unitBufferLimit.is_finite(): unitBuffer.append(mdlObj) if _streamingExtensionsValidate: validator.instValidator.checkUnits( (mdlObj,) ) elif ln == "xbrl": # end of document # check remaining footnote refs for footnoteLink in footnoteBuffer: checkFootnoteHrefs(modelXbrl, footnoteLink) elt.clear() elif ns == XbrlConst.link: if ln in ("schemaRef", "linkbaseRef"): modelDocument.discoverHref(mdlObj) elif ln in ("roleRef", "arcroleRef"): modelDocument.linkbaseDiscover((mdlObj,), inInstance=True) elif ln == "footnoteLink": footnoteLinks = (mdlObj,) modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True) if footnoteBufferLimit.is_finite(): footnoteBuffer.append(mdlObj) if _streamingExtensionsValidate: validator.instValidator.checkLinks(footnoteLinks) if len(footnoteBuffer) > footnoteBufferLimit: # check that hrefObjects for locators were all satisfied # drop before addition as dropped may have same id as added footnoteLink = footnoteBuffer.pop(0) checkFootnoteHrefs(modelXbrl, footnoteLink) dropFootnoteLink(modelXbrl, footnoteLink) del parentMdlObj[parentMdlObj.index(footnoteLink)] footnoteLink = None footnoteLinks = None elt.clear() elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl: numRootFacts2 += 1 modelDocument.factDiscover(mdlObj, modelXbrl.facts) XmlValidate.validate(modelXbrl, mdlObj) if _streamingExtensionsValidate: factsToCheck = (mdlObj,) validator.instValidator.checkFacts(factsToCheck) if modelXbrl.hasXDT: validator.instValidator.checkFactsDimensions(factsToCheck) del factsToCheck dropFact(modelXbrl, mdlObj, modelXbrl.facts) del parentMdlObj[parentMdlObj.index(mdlObj)] if numRootFacts2 % 1000 == 0: modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(numRootFacts2, numRootFacts1, 100.0 * numRootFacts2 / numRootFacts1), minTimeToShow=20.0) # get rid of root element from iterparse's tree elt.clear() while elt.getprevious() is not None: # cleans up any prior siblings del elt.getparent()[0] mdlObj = None # deref logSyntaxErrors(parsercontext) del parsercontext if validator is not None: validator.close() _file.close() modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt) return modelDocument
def evaluateTableIndex(modelXbrl): disclosureSystem = modelXbrl.modelManager.disclosureSystem if disclosureSystem.EFM: COVER = "1Cover" STMTS = "2Financial Statements" NOTES = "3Notes to Financial Statements" POLICIES = "4Accounting Policies" TABLES = "5Notes Tables" DETAILS = "6Notes Details" UNCATEG = "7Uncategorized" roleDefinitionPattern = re.compile( r"([0-9]+) - (Statement|Disclosure|Schedule|Document) - (.+)") # build EFM rendering-compatible index definitionElrs = dict( (roleType.definition, roleType) for roleURI in modelXbrl.relationshipSet( XbrlConst.parentChild).linkRoleUris for roleType in modelXbrl.roleTypes.get(roleURI, ())) isRR = any( ns.startswith("http://xbrl.sec.gov/rr/") for ns in modelXbrl.namespaceDocs.keys()) tableGroup = None firstTableLinkroleURI = None firstDocumentLinkroleURI = None sortedRoleTypes = sorted(definitionElrs.items(), key=lambda item: item[0]) for roleDefinition, roleType in sortedRoleTypes: roleType._tableChildren = [] match = roleDefinitionPattern.match( roleDefinition) if roleDefinition else None if not match: roleType._tableIndex = (UNCATEG, "", roleType.roleURI) continue seq, tblType, tblName = match.groups() if isRR: tableGroup = COVER elif not tableGroup: tableGroup = ("Paren" in tblName and COVER or tblType == "Statement" and STMTS or "(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or COVER) elif tableGroup == COVER: tableGroup = (tblType == "Statement" and STMTS or "Paren" in tblName and COVER or "(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or NOTES) elif tableGroup == STMTS: tableGroup = ((tblType == "Statement" or "Paren" in tblName) and STMTS or "(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or NOTES) elif tableGroup == NOTES: tableGroup = ("(Polic" in tblName and POLICIES or "(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or tblType == "Disclosure" and NOTES or UNCATEG) elif tableGroup == POLICIES: tableGroup = ("(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or ("Paren" in tblName or "(Polic" in tblName) and POLICIES or UNCATEG) elif tableGroup == TABLES: tableGroup = ("(Detail" in tblName and DETAILS or ("Paren" in tblName or "(Table" in tblName) and TABLES or UNCATEG) elif tableGroup == DETAILS: tableGroup = (("Paren" in tblName or "(Detail" in tblName) and DETAILS or UNCATEG) else: tableGroup = UNCATEG if firstTableLinkroleURI is None and tableGroup == COVER: firstTableLinkroleURI = roleType.roleURI if tblType == "Document" and not firstDocumentLinkroleURI: firstDocumentLinkroleURI = roleType.roleURI roleType._tableIndex = (tableGroup, seq, tblName) # flow allocate facts to roles (SEC presentation groups) if not modelXbrl.qnameDimensionDefaults: # may not have run validatino yet from arelle import ValidateXbrlDimensions ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) reportedFacts = set( ) # facts which were shown in a higher-numbered ELR table reportingPeriods = set() nextEnd = None deiFact = {} for conceptName in ("DocumentPeriodEndDate", "DocumentType", "CurrentFiscalPeriodEndDate"): for concept in modelXbrl.nameConcepts[conceptName]: for fact in modelXbrl.factsByQname(concept.qname): deiFact[conceptName] = fact if fact.context is not None: reportingPeriods.add( (None, fact.context.endDatetime)) # for instant reportingPeriods.add( (fact.context.startDatetime, fact.context.endDatetime)) # for startEnd nextEnd = fact.context.startDatetime duration = (fact.context.endDatetime - fact.context.startDatetime).days + 1 break if "DocumentType" in deiFact: fact = deiFact["DocumentType"] if "-Q" in fact.xValue: # need quarterly and yr to date durations endDatetime = fact.context.endDatetime # if within 2 days of end of month use last day of month endDatetimeMonth = endDatetime.month if (endDatetime + timedelta(2)).month != endDatetimeMonth: # near end of month endOfMonth = True while endDatetime.month == endDatetimeMonth: endDatetime += timedelta(1) # go forward to next month else: endOfMonth = False startYr = endDatetime.year startMo = endDatetime.month - 3 if startMo <= 0: startMo += 12 startYr -= 1 startDatetime = datetime(startYr, startMo, endDatetime.day, endDatetime.hour, endDatetime.minute, endDatetime.second) if endOfMonth: startDatetime -= timedelta(1) endDatetime -= timedelta(1) reportingPeriods.add((startDatetime, endDatetime)) duration = 91 # find preceding compatible default context periods while (nextEnd is not None): thisEnd = nextEnd prevMaxStart = thisEnd - timedelta(duration * .9) prevMinStart = thisEnd - timedelta(duration * 1.1) nextEnd = None for cntx in modelXbrl.contexts.values(): if cntx is not None: if (cntx.isStartEndPeriod and not cntx.qnameDims and thisEnd == cntx.endDatetime and prevMinStart <= cntx.startDatetime <= prevMaxStart): reportingPeriods.add((None, cntx.endDatetime)) reportingPeriods.add( (cntx.startDatetime, cntx.endDatetime)) nextEnd = cntx.startDatetime break elif (cntx.isInstantPeriod and not cntx.qnameDims and thisEnd == cntx.endDatetime): reportingPeriods.add((None, cntx.endDatetime)) stmtReportingPeriods = set(reportingPeriods) sortedRoleTypes.reverse() # now in descending order for i, roleTypes in enumerate(sortedRoleTypes): roleDefinition, roleType = roleTypes # find defined non-default axes in pre hierarchy for table tableFacts = set() tableGroup, tableSeq, tableName = roleType._tableIndex roleURIdims, priItemQNames = EFMlinkRoleURIstructure( modelXbrl, roleType.roleURI) for priItemQName in priItemQNames: for fact in modelXbrl.factsByQname(priItemQName): cntx = fact.context # non-explicit dims must be default if (cntx is not None and all(dimQn in modelXbrl.qnameDimensionDefaults for dimQn in (roleURIdims.keys() - cntx.qnameDims.keys())) and all(mdlDim.memberQname in roleURIdims[dimQn] for dimQn, mdlDim in cntx.qnameDims.items() if dimQn in roleURIdims)): # the flow-up part, drop cntxStartDatetime = cntx.startDatetime cntxEndDatetime = cntx.endDatetime if (tableGroup != STMTS or (cntxStartDatetime, cntxEndDatetime) in stmtReportingPeriods and (fact not in reportedFacts or all( dimQn not in cntx. qnameDims # unspecified dims are all defaulted if reported elsewhere for dimQn in (cntx.qnameDims.keys() - roleURIdims.keys())))): tableFacts.add(fact) reportedFacts.add(fact) roleType._tableFacts = tableFacts # find parent if any closestParentType = None closestParentMatchLength = 0 for _parentRoleDefinition, parentRoleType in sortedRoleTypes[i + 1:]: matchLen = parentNameMatchLen(tableName, parentRoleType) if matchLen > closestParentMatchLength: closestParentMatchLength = matchLen closestParentType = parentRoleType if closestParentType is not None: closestParentType._tableChildren.insert(0, roleType) # remove lesser-matched children if there was a parent match unmatchedChildRoles = set() longestChildMatchLen = 0 numChildren = 0 for childRoleType in roleType._tableChildren: matchLen = parentNameMatchLen(tableName, childRoleType) if matchLen < closestParentMatchLength: unmatchedChildRoles.add(childRoleType) elif matchLen > longestChildMatchLen: longestChildMatchLen = matchLen numChildren += 1 if numChildren > 1: # remove children that don't have the full match pattern length to parent for childRoleType in roleType._tableChildren: if (childRoleType not in unmatchedChildRoles and parentNameMatchLen(tableName, childRoleType) < longestChildMatchLen): unmatchedChildRoles.add(childRoleType) for unmatchedChildRole in unmatchedChildRoles: roleType._tableChildren.remove(unmatchedChildRole) for childRoleType in roleType._tableChildren: childRoleType._tableParent = roleType unmatchedChildRoles = None # dereference global UGT_TOPICS if UGT_TOPICS is None: try: from arelle import FileSource fh = FileSource.openFileStream( modelXbrl.modelManager.cntlr, os.path.join(modelXbrl.modelManager.cntlr.configDir, "ugt-topics.zip/ugt-topics.json"), 'r', 'utf-8') UGT_TOPICS = json.load(fh) fh.close() for topic in UGT_TOPICS: topic[6] = set( topic[6] ) # change concept abstracts list into concept abstracts set topic[7] = set( topic[7] ) # change concept text blocks list into concept text blocks set topic[8] = set( topic[8] ) # change concept names list into concept names set except Exception as ex: UGT_TOPICS = None if UGT_TOPICS is not None: def roleUgtConcepts(roleType): roleConcepts = set() for rel in modelXbrl.relationshipSet( XbrlConst.parentChild, roleType.roleURI).modelRelationships: if rel.toModelObject is not None: roleConcepts.add(rel.toModelObject.name) if rel.fromModelObject is not None: roleConcepts.add(rel.fromModelObject.name) if hasattr(roleType, "_tableChildren"): for _tableChild in roleType._tableChildren: roleConcepts |= roleUgtConcepts(_tableChild) return roleConcepts topicMatches = {} # topicNum: (best score, roleType) for roleDefinition, roleType in sortedRoleTypes: roleTopicType = 'S' if roleDefinition.startswith('S') else 'D' if getattr(roleType, "_tableParent", None) is None: # rooted tables in reverse order concepts = roleUgtConcepts(roleType) for i, ugtTopic in enumerate(UGT_TOPICS): if ugtTopic[0] == roleTopicType: countAbstracts = len(concepts & ugtTopic[6]) countTextBlocks = len(concepts & ugtTopic[7]) countLineItems = len(concepts & ugtTopic[8]) if countAbstracts or countTextBlocks or countLineItems: _score = (10 * countAbstracts + 1000 * countTextBlocks + countLineItems / len(concepts)) if i not in topicMatches or _score > topicMatches[ i][0]: topicMatches[i] = (_score, roleType) for topicNum, scoredRoleType in topicMatches.items(): _score, roleType = scoredRoleType if _score > getattr(roleType, "_tableTopicScore", 0): ugtTopic = UGT_TOPICS[topicNum] roleType._tableTopicScore = _score roleType._tableTopicType = ugtTopic[0] roleType._tableTopicName = ugtTopic[3] roleType._tableTopicCode = ugtTopic[4] # print ("Match score {:.2f} topic {} preGrp {}".format(_score, ugtTopic[3], roleType.definition)) return firstTableLinkroleURI or firstDocumentLinkroleURI # did build _tableIndex attributes return None
def evaluateTableIndex(modelXbrl): disclosureSystem = modelXbrl.modelManager.disclosureSystem if disclosureSystem.EFM: COVER = "1Cover" STMTS = "2Financial Statements" NOTES = "3Notes to Financial Statements" POLICIES = "4Accounting Policies" TABLES = "5Notes Tables" DETAILS = "6Notes Details" UNCATEG = "7Uncategorized" roleDefinitionPattern = re.compile(r"([0-9]+) - (Statement|Disclosure|Schedule|Document) - (.+)") # build EFM rendering-compatible index definitionElrs = dict((roleType.definition, roleType) for roleURI in modelXbrl.relationshipSet(XbrlConst.parentChild).linkRoleUris for roleType in modelXbrl.roleTypes.get(roleURI,())) isRR = any(ns.startswith("http://xbrl.sec.gov/rr/") for ns in modelXbrl.namespaceDocs.keys()) tableGroup = None firstTableLinkroleURI = None firstDocumentLinkroleURI = None sortedRoleTypes = sorted(definitionElrs.items(), key=lambda item: item[0]) for roleDefinition, roleType in sortedRoleTypes: match = roleDefinitionPattern.match(roleDefinition) if roleDefinition else None if not match: roleType._tableIndex = (UNCATEG, roleType.roleURI) continue seq, tblType, tblName = match.groups() if isRR: tableGroup = COVER elif not tableGroup: tableGroup = ("Paren" in tblName and COVER or tblType == "Statement" and STMTS or "(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or COVER) elif tableGroup == COVER: tableGroup = (tblType == "Statement" and STMTS or "Paren" in tblName and COVER or "(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or NOTES) elif tableGroup == STMTS: tableGroup = ((tblType == "Statement" or "Paren" in tblName) and STMTS or "(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or NOTES) elif tableGroup == NOTES: tableGroup = ("(Polic" in tblName and POLICIES or "(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or tblType == "Disclosure" and NOTES or UNCATEG) elif tableGroup == POLICIES: tableGroup = ("(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or ("Paren" in tblName or "(Polic" in tblName) and POLICIES or UNCATEG) elif tableGroup == TABLES: tableGroup = ("(Detail" in tblName and DETAILS or ("Paren" in tblName or "(Table" in tblName) and TABLES or UNCATEG) elif tableGroup == DETAILS: tableGroup = (("Paren" in tblName or "(Detail" in tblName) and DETAILS or UNCATEG) else: tableGroup = UNCATEG if firstTableLinkroleURI is None and tableGroup == COVER: firstTableLinkroleURI = roleType.roleURI if tblType == "Document" and not firstDocumentLinkroleURI: firstDocumentLinkroleURI = roleType.roleURI roleType._tableIndex = (tableGroup, seq, tblName) # flow allocate facts to roles (SEC presentation groups) if not modelXbrl.qnameDimensionDefaults: # may not have run validatino yet from arelle import ValidateXbrlDimensions ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) reportedFacts = set() # facts which were shown in a higher-numbered ELR table factsByQname = modelXbrl.factsByQname reportingPeriods = set() nextEnd = None deiFact = {} for conceptName in ("DocumentPeriodEndDate", "DocumentType", "CurrentFiscalPeriodEndDate"): for concept in modelXbrl.nameConcepts[conceptName]: for fact in factsByQname[concept.qname]: deiFact[conceptName] = fact if fact.context is not None: reportingPeriods.add((None, fact.context.endDatetime)) # for instant reportingPeriods.add((fact.context.startDatetime, fact.context.endDatetime)) # for startEnd nextEnd = fact.context.startDatetime duration = (fact.context.endDatetime - fact.context.startDatetime).days + 1 break if "DocumentType" in deiFact: fact = deiFact["DocumentType"] if "-Q" in fact.xValue: # need quarterly and yr to date durations endDatetime = fact.context.endDatetime # if within 2 days of end of month use last day of month endDatetimeMonth = endDatetime.month if (endDatetime + timedelta(2)).month != endDatetimeMonth: # near end of month endOfMonth = True while endDatetime.month == endDatetimeMonth: endDatetime += timedelta(1) # go forward to next month else: endOfMonth = False startYr = endDatetime.year startMo = endDatetime.month - 3 if startMo < 0: startMo += 12 startYr -= 1 startDatetime = datetime(startYr, startMo, endDatetime.day, endDatetime.hour, endDatetime.minute, endDatetime.second) if endOfMonth: startDatetime -= timedelta(1) endDatetime -= timedelta(1) reportingPeriods.add((startDatetime, endDatetime)) duration = 91 # find preceding compatible default context periods while (nextEnd is not None): thisEnd = nextEnd prevMaxStart = thisEnd - timedelta(duration * .9) prevMinStart = thisEnd - timedelta(duration * 1.1) nextEnd = None for cntx in modelXbrl.contexts.values(): if (cntx.isStartEndPeriod and not cntx.qnameDims and thisEnd == cntx.endDatetime and prevMinStart <= cntx.startDatetime <= prevMaxStart): reportingPeriods.add((None, cntx.endDatetime)) reportingPeriods.add((cntx.startDatetime, cntx.endDatetime)) nextEnd = cntx.startDatetime break elif (cntx.isInstantPeriod and not cntx.qnameDims and thisEnd == cntx.endDatetime): reportingPeriods.add((None, cntx.endDatetime)) stmtReportingPeriods = set(reportingPeriods) for roleDefinition, roleType in reversed(sortedRoleTypes): # find defined non-default axes in pre hierarchy for table tableFacts = set() tableGroup = roleType._tableIndex[0] roleURIdims, priItemQNames = EFMlinkRoleURIstructure(modelXbrl, roleType.roleURI) for priItemQName in priItemQNames: for fact in factsByQname[priItemQName]: cntx = fact.context # non-explicit dims must be default if (cntx is not None and all(dimQn in modelXbrl.qnameDimensionDefaults for dimQn in (roleURIdims.keys() - cntx.qnameDims.keys())) and all(mdlDim.memberQname in roleURIdims[dimQn] for dimQn, mdlDim in cntx.qnameDims.items() if dimQn in roleURIdims)): # the flow-up part, drop cntxStartDatetime = cntx.startDatetime cntxEndDatetime = cntx.endDatetime if (tableGroup != STMTS or (cntxStartDatetime, cntxEndDatetime) in stmtReportingPeriods and (fact not in reportedFacts or all(dimQn not in cntx.qnameDims # unspecified dims are all defaulted if reported elsewhere for dimQn in (cntx.qnameDims.keys() - roleURIdims.keys())))): tableFacts.add(fact) reportedFacts.add(fact) roleType._tableFacts = tableFacts return firstTableLinkroleURI or firstDocumentLinkroleURI # did build _tableIndex attributes return None
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri): from openpyxl import load_workbook from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname try: currentAction = "initializing" startedAt = time.time() if os.path.isabs(oimFile): # allow relative filenames to loading directory priorCWD = os.getcwd() os.chdir(os.path.dirname(oimFile)) else: priorCWD = None currentAction = "determining file type" isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json") isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json") isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls") isCSVorXL = isCSV or isXL instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl" if isJSON: currentAction = "loading and parsing JSON OIM file" with io.open(oimFile, 'rt', encoding='utf-8') as f: oimObject = json.load(f) missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject] if missing: raise OIMException("oime:missingJSONelements", _("The required elements %(missing)s {} in JSON input"), missing = ", ".join(missing)) currentAction = "identifying JSON objects" dtsReferences = oimObject["dtsReferences"] prefixes = oimObject["prefixes"] facts = oimObject["facts"] footnotes = oimOject["facts"] # shares this object elif isCSV: currentAction = "identifying CSV input tables" if sys.version[0] >= '3': csvOpenMode = 'w' csvOpenNewline = '' else: csvOpenMode = 'wb' # for 2.7 csvOpenNewline = None oimFileBase = None if "-facts" in oimFile: oimFileBase = oimFile.partition("-facts")[0] else: for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"): if oimFile.endswith(suffix): oimFileBase = oimFile[:-length(suffix)] break if oimFileBase is None: raise OIMException("oime:missingCSVtables", _("Unable to identify CSV tables file name pattern")) if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or not os.path.exists(oimFileBase + "-prefixes.csv")): raise OIMException("oime:missingCSVtables", _("Unable to identify CSV tables for dtsReferences or prefixes")) instanceFileName = oimFileBase + ".xbrl" currentAction = "loading CSV dtsReferences table" dtsReferences = [] with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: dtsReferences.append(dict((header[j], col) for j, col in enumerate(row))) currentAction = "loading CSV prefixes table" prefixes = {} with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = dict((col,i) for i,col in enumerate(row)) else: prefixes[row[header["prefix"]]] = row[header["URI"]] currentAction = "loading CSV defaults table" defaults = {} if os.path.exists(oimFileBase + "-defaults.csv"): with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row fileCol = row.index("file") else: defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol) currentAction = "loading CSV facts tables" facts = [] _dir = os.path.dirname(oimFileBase) for filename in os.listdir(_dir): filepath = os.path.join(_dir, filename) if "-facts" in filename and filepath.startswith(oimFileBase): tableDefaults = defaults.get(filename, {}) with io.open(filepath, 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col is not None: if header[j].endswith("Value"): if col: # ignore empty columns (= null CSV value) fact["value"] = col else: fact[header[j]] = col facts.append(fact) footnotes = [] if os.path.exists(oimFileBase + "-footnotes.csv"): with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col)) elif isXL: oimWb = load_workbook(oimFile, read_only=True, data_only=True) sheetNames = oimWb.get_sheet_names() if (not any(sheetName == "prefixes" for sheetName in sheetNames) or not any(sheetName == "dtsReferences" for sheetName in sheetNames) or not any("facts" in sheetName for sheetName in sheetNames)): if priorCWD: os.chdir(priorCWD) return None try: dtsReferences = [] for i, row in oimWb["dtsReferences"]: if i == 0: header = [col.value for col in row] else: dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row))) prefixes = {} for i, row in oimWb["dtsReferences"]: if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) else: prefixes[row[header["prefix"]].value] = row[header["URI"].value] defaults = {} for i, row in oimW.get("defaults", ()): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) fileCol = header["file"] else: defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol) facts = [] _dir = os.path.dirpart(oimFileBase) for sheetName in sheetNames: if sheetName == "facts" or "-facts" in sheetName: for i, row in oimWb[sheetName]: if i == 0: header = [col.value for col in row] else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col.value is not None: if header[j].endswith("Value"): fact["value"] = str(col.value) else: fact[header[j]] = str(col.value) facts.append(fact) footnotes = [] for i, row in oimWb.get("footnotes", ()): if i == 0: header = dict((col.value,i) for i,col in enumerate(row) if col.value) else: footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value)) except Exception as ex: if priorCWD: os.chdir(priorCWD) return None ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults # create the instance document modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument( modelXbrl, Type.INSTANCE, instanceFileName, schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"], isEntry=True, initialComment="extracted from OIM {}".format(mappedUri), documentEncoding="utf-8") cntxTbl = {} unitTbl = {} for fact in facts: conceptQn = qname(fact["oim:concept"], prefixes) concept = modelXbrl.qnameConcepts.get(conceptQn) entityAsQn = qname(fact["oim:entity"], prefixes) if "oim:period" in fact: periodStart = fact["oim:period"]["start"] periodEnd = fact["oim:period"]["end"] else: periodStart = fact["oim:periodStart"] periodEnd = fact["oim:periodEnd"] cntxKey = ( # hashable context key ("periodType", concept.periodType), ("entity", entityAsQn), ("periodStart", periodStart), ("periodEnd", periodEnd)) + tuple(sorted( (dimName, dimVal) for dimName, dimVal in fact.items() if ":" in dimName and not dimName.startswith("oim:"))) if cntxKey in cntxTbl: _cntx = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) qnameDims = {} for dimName, dimVal in fact.items(): if ":" in dimName and not dimName.startswith("oim:"): dimQname = qname(dimName, prefixes) dimConcept = modelXbrl.qnameConcepts.get(dimQname) if ":" in dimVal and dimVal.partition[':'][0] in prefixes: mem = qname(dimVal, prefixes) # explicit dim elif dimConcept.isTypedDimension: # a modelObject xml element is needed for all of the instance functions to manage the typed dim mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment") _cntx = modelXbrl.createContext( entityAsQn.namespaceURI, entityAsQn.localName, concept.periodType, None if concept.periodType == "instnat" else dateTime(periodStart, type=DATETIME), dateTime(periodEnd, type=DATETIME), None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) cntxTbl[cntxKey] = _cntx if "oim:unit" in fact: unitKey = fact["oim:unit"] if unitKey in unitTbl: _unit = unitTbl[unitKey] else: _mul, _sep, _div = unitKey.partition('/') if _mul.startswith('('): _mul = _mul[1:-1] mulQns = [qname(u, prefixes) for u in _mul.split('*') if u] if _div.startswith('('): _div = _div[1:-1] divQns = [qname(u, prefixes) for u in _div.split('*') if u] unitId = 'u-{:02}'.format(len(unitTbl) + 1) for _measures in mulQns, divQns: for _measure in _measures: addQnameValue(modelXbrl.modelDocument, _measure) _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId) unitTbl[unitKey] = _unit else: _unit = None attrs = {"contextRef": _cntx.id} if fact.get("value") is None: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact["value"] if fact.get("id"): attrs["id"] = fact["id"] if concept.isNumeric: if _unit is not None: attrs["unitRef"] = _unit.id if "accuracy" in fact: attrs["decimals"] = fact["accuracy"] # is value a QName? if concept.baseXbrliType == "QName": addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes)) f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text) footnoteLinks = {} # ELR elements factLocs = {} # index by (linkrole, factId) footnoteNbr = 0 locNbr = 0 for factOrFootnote in footnotes: if "factId" in factOrFootnote: factId = factOrFootnote["factId"] factFootnotes = (factOrFootnote,) # CSV or XL elif "id" in factOrFootnote and "footnotes" in factOrFootnote: factId = factOrFootnote["id"] factFootnotes = factOrFootnote["footnotes"] else: factFootnotes = () for footnote in factFootnotes: linkrole = footnote.get("group") arcrole = footnote.get("footnoteType") if not factId or not linkrole or not arcrole or not ( footnote.get("factRef") or footnote.get("footnote")): # invalid footnote continue if linkrole not in footnoteLinks: footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, XbrlConst.qnLinkFootnoteLink, attributes={"{http://www.w3.org/1999/xlink}type": "extended", "{http://www.w3.org/1999/xlink}role": linkrole}) footnoteLink = footnoteLinks[linkrole] if (linkrole, factId) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factId)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factId, XLINKLABEL: locLabel}) locLabel = factLocs[(linkrole, factId)] if footnote.get("footnote"): footnoteNbr += 1 footnoteLabel = "f_{:02}".format(footnoteNbr) attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel} if footnote.get("language"): attrs[XMLLANG] = footnote["language"] # note, for HTML will need to build an element structure addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"]) elif footnote.get("factRef"): factRef = footnote.get("factRef") if (linkrole, factRef) not in factLocs: locNbr += 1 locLabel = "f_{:02}".format(footnoteNbr) factLoc[(linkrole, factRef)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factRef, XLINKLABEL: locLabel}) footnoteLabel = factLoc[(linkrole, factId)] footnoteArc = addChild(footnoteLink, XbrlConst.qnLinkFootnoteArc, attributes={XLINKTYPE: "arc", XLINKARCROLE: arcrole, XLINKFROM: locLabel, XLINKTO: footnoteLabel}) #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") except Exception as ex: if ex is OIMException: modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) else: modelXbrl.error("Error while %(action)s, error %(error)s", modelObject=modelXbrl, action=currentAction, error=ex) if priorCWD: os.chdir(priorCWD) # restore prior current working directory return modelXbrl.modelDocument
def init(modelXbrl): # setup modelXbrl for rendering evaluation # dimension defaults required in advance of validation from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) hasXbrlTables = False # validate table linkbase dimensions for baseSetKey in modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(arcrole): ValidateFormula.checkBaseSet(modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname)) if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableAxis2011): hasXbrlTables = True # provide context for view if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE: instance = None # use instance of the entry pont else: # need dummy instance instance = ModelDocument.create(modelXbrl, ModelDocument.Type.INSTANCE, "dummy.xml", # fake URI and fake schemaRef ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd",)) if hasXbrlTables: # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance) modelXbrl.profileStat(None) # setup fresh parameters from formula optoins modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters() # validate parameters and custom function signatures ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables")) # check and extract message expressions into compilable programs for msgArcrole in (XbrlConst.tableDefinitionNodeMessage, XbrlConst.tableDefinitionNodeSelectionMessage, XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011): for msgRel in modelXbrl.relationshipSet(msgArcrole).modelRelationships: ValidateFormula.checkMessageExpressions(modelXbrl, msgRel.toModelObject) # compile and validate tables for modelTable in modelXbrl.modelRenderingTables: modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining modelTable.compile() # check aspectModel if modelTable.aspectModel not in ("non-dimensional", "dimensional"): modelXbrl.error("xbrlte:unknownAspectModel", _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel) else: modelTable.priorAspectAxisDisposition = {} # check ordinate aspects against aspectModel oppositeAspectModel = (_DICT_SET({'dimensional','non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop() uncoverableAspects = aspectModels[oppositeAspectModel] - aspectModels[modelTable.aspectModel] for tblAxisRel in modelXbrl.relationshipSet((XbrlConst.tableBreakdown,XbrlConst.tableAxis2011)).fromModelObject(modelTable): checkDefinitionNodeAspectModel(modelXbrl, modelTable, tblAxisRel, uncoverableAspects) del modelTable.priorAspectAxisDisposition modelXbrl.profileStat(_("compileTables"))
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in modelXbrl.contexts.values(): newCntx = targetInstance.createContext(context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in modelXbrl.facts: if fact.isItem: attrs = {"contextRef": fact.contextID} if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) newFactForOldObjId[fact.objectIndex] = newFact elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) # footnote links modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships")) for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: newLink = XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, linkqname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: if isinstance(linkChild, LocPrototype) and "{http://www.w3.org/1999/xlink}href" not in linkChild.attributes: linkChild.attributes["{http://www.w3.org/1999/xlink}href"] = \ "#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) XmlUtil.addChild(newLink, linkChild.qname, attributes=linkChild.attributes, text=linkChild.textValue) targetInstance.saveInstance(overrideFilepath=targetUrl) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def loadXbrlFromDB(self, loadDBsaveToFile): # load from database modelXbrl = self.modelXbrl # find instance in DB instanceURI = os.path.basename(loadDBsaveToFile) results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant" " FROM dInstance WHERE FileName = '{}'" .format(instanceURI)) instanceId = moduleId = None for instanceId, moduleId, entScheme, entId, datePerEnd in results: break # find module in DB results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId)) xbrlSchemaRef = None for result in results: xbrlSchemaRef = result[0] break if not instanceId or not xbrlSchemaRef: raise XPDBException("sqlDB:MissingDTS", _("The instance and module were not found for %(instanceURI)"), instanceURI = instanceURI) if modelXbrl.skipDTS: # find prefixes and namespaces in DB results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]") dpmPrefixedNamespaces = dict((prefix, namespace) for owner, prefix, namespace in results) # create the instance document and resulting filing modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument(modelXbrl, Type.INSTANCE, loadDBsaveToFile, schemaRefs=[xbrlSchemaRef], isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement, 'xbrl-streamable-instance', 'version="1.0" contextBuffer="1"') # add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue) # filing indicator code IDs # get filing indicators results = self.execute("SELECT mToT.TemplateOrTableCode " " FROM dFilingIndicator dFI, mTemplateOrTable mToT " " WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID" .format(instanceId)) filingIndicatorCodes = [code[0] for code in results] if filingIndicatorCodes: modelXbrl.createContext(entScheme, entId, 'instant', None, datePerEnd, None, # no dimensional validity checking (like formula does) {}, [], [], id='c') filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators) for filingIndicatorCode in filingIndicatorCodes: modelXbrl.createFact(qnFindFilingIndicator, parent=filingIndicatorsTuple, attributes={"contextRef": "c"}, text=filingIndicatorCode) # facts in this instance factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards," " Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue " "FROM dFact WHERE InstanceID = {} " "ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL " " THEN DataPointSignature" " ELSE DataPointSignatureWithValuesForWildcards" " END, instr(DataPointSignature,'|') + 1)" .format(instanceId)) # results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal # get typed dimension values prefixedNamespaces = modelXbrl.prefixedNamespaces prefixedNamespaces["iso4217"] = XbrlConst.iso4217 if modelXbrl.skipDTS: prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed cntxTbl = {} # index by d unitTbl = {} def typedDimElt(s): # add xmlns into s for known qnames tag, angleBrkt, rest = s[1:].partition('>') text, angleBrkt, rest = rest.partition("<") qn = qname(tag, prefixedNamespaces) # a modelObject xml element is needed for all of the instance functions to manage the typed dim return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False) # contexts and facts for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl: metric, sep, dims = (dpSigTypedDims or dpSig).partition('|') conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces) concept = modelXbrl.qnameConcepts.get(conceptQn) isNumeric = isBool = isDateTime = isQName = isText = False if concept is not None: if concept.isNumeric: isNumeric = True else: baseXbrliType = concept.baseXbrliType if baseXbrliType == "booleanItemType": isBool = True elif baseXbrliType == "dateTimeItemType": # also is dateItemType? isDateTime = True elif baseXbrliType == "QNameItemType": isQName = True else: c = conceptQn.localName[0] if c == 'm': isNumeric = True elif c == 'd': isDateTime = True elif c == 'b': isBool = True elif c == 'e': isQName = True isText = not (isNumeric or isBool or isDateTime or isQName) if isinstance(datePerEnd, _STR_BASE): datePerEnd = datetimeValue(datePerEnd, addOneDay=True) cntxKey = (dims, entId, datePerEnd) if cntxKey in cntxTbl: cntxId = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) cntxTbl[cntxKey] = cntxId qnameDims = {} if dims: for dim in dims.split('|'): dQn, sep, dVal = dim[:-1].partition('(') dimQname = qname(dQn, prefixedNamespaces) if dVal.startswith('<'): mem = typedDimElt(dVal) # typed dim else: mem = qname(dVal, prefixedNamespaces) # explicit dim (even if treat-as-typed) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario") modelXbrl.createContext(entScheme, entId, 'instant', None, datePerEnd, None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) if unit: if unit in unitTbl: unitId = unitTbl[unit] else: unitQn = qname(unit, prefixedNamespaces) unitId = 'u{}'.format(unitQn.localName) unitTbl[unit] = unitId modelXbrl.createUnit([unitQn], [], id=unitId) else: unitId = None attrs = {"contextRef": cntxId} if unitId: attrs["unitRef"] = unitId if dec is not None: if isinstance(dec, float): # must be an integer dec = int(dec) elif isinstance(dec, _STR_BASE) and '.' in dec: dec = dec.partition('.')[0] # drop .0 from any SQLite string attrs["decimals"] = str(dec) # somehow it is float from the database if False: # fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None elif numVal is not None: num = roundValue(numVal, None, dec) # round using reported decimals if dec is None or dec == "INF": # show using decimals or reported format dec = len(numVal.partition(".")[2]) else: # max decimals at 28 dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num)) elif dateVal is not None: text = dateVal elif boolVal is not None: text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false' else: if isQName: # declare namespace addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces)) text = textVal modelXbrl.createFact(conceptQn, attributes=attrs, text=text) # add footnotes if any # save to file modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000) return modelXbrl.modelDocument
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, *args, **kwargs): # check if big instance and has header with an initial incomplete tree walk (just 2 elements if not _streamingExtensionsCheck: return None # track whether modelXbrl has been validated by this streaming extension modelXbrl._streamingExtensionValidated = False def logSyntaxErrors(parsercontext): for error in parsercontext.error_log: modelXbrl.error("xmlSchema:syntax", _("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"), modelObject=modelXbrl, fileName=os.path.basename(filepath), error=error.message, line=error.line, column=error.column, sourceAction="streaming") #### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ### #### note: iterparse wants a binary file, but file is text mode _file, = modelXbrl.fileSource.file(filepath, binary=True) startedAt = time.time() modelXbrl.profileActivity() ''' this seems twice as slow as iterparse class instInfoTarget(): def __init__(self, element_factory=None, parser=None): self.newTree = True self.streamingAspects = None self.foundInstance = False self.creationSoftwareComment = '' self.currentEltTag = "(before xbrli:xbrl)" self.numRootFacts = 0 def start(self, tag, attrib, nsmap=None): if self.newTree: if tag == "{http://www.xbrl.org/2003/instance}xbrl": self.foundInstance = True self.newTree = False else: # break raise NotInstanceDocumentException() elif not tag.startswith("{http://www.xbrl.org/"): self.numRootFacts += 1 if self.numRootFacts % 1000 == 0: modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0) self.currentEltTag = tag def end(self, tag): pass def data(self, data): pass def comment(self, text): if not self.foundInstance: # accumulate comments before xbrli:xbrl self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text elif not self.creationSoftwareComment: self.creationSoftwareComment = text # or first comment after xbrli:xbrl def pi(self, target, data): if target == "xbrl-streamable-instance": if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl": self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results else: modelXbrl.error("streamingExtensions:headerMisplaced", _("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"), modelObject=modelXbrl, target=target, element=self.currentEltTag) def close(self): if not self.creationSoftwareComment: self.creationSoftwareComment = None return True instInfo = instInfoTarget() infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo) try: etree.parse(_file, parser=infoParser, base_url=filepath) except NotInstanceDocumentException: pass ''' foundErrors = False foundInstance = False streamingAspects = None creationSoftwareComment = None instInfoNumRootFacts = 0 numElts = 0 elt = None instInfoContext = etree.iterparse(_file, events=("start","end"), huge_tree=True) try: for event, elt in instInfoContext: if event == "start": if elt.getparent() is not None: if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl": if not foundInstance: foundInstance = True pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance") if pi is None: break else: streamingAspects = dict(pi.attrib.copy()) if creationSoftwareComment is None: creationSoftwareComment = precedingComment(elt) if not elt.tag.startswith("{http://www.xbrl.org/"): instInfoNumRootFacts += 1 if instInfoNumRootFacts % 1000 == 0: modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0) elif not foundInstance: break elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl": creationSoftwareComment = precedingComment(elt) if precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None: modelXbrl.error("streamingExtensions:headerMisplaced", _("Header is misplaced: %(error)s, must follow xbrli:xbrl element"), modelObject=elt) elif event == "end": elt.clear() numElts += 1 if numElts % 1000 == 0 and elt.getparent() is not None: while elt.getprevious() is not None and elt.getparent() is not None: del elt.getparent()[0] except etree.XMLSyntaxError as err: modelXbrl.error("xmlSchema:syntax", _("Unrecoverable error: %(error)s"), error=err) _file.close() return err _file.seek(0,io.SEEK_SET) # allow reparsing if not foundInstance or streamingAspects is None: del elt _file.close() return None modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt) startedAt = time.time() try: version = Decimal(streamingAspects.get("version")) if int(version) != 1: modelXbrl.error("streamingExtensions:unsupportedVersion", _("Streaming version %(version)s, major version number must be 1"), modelObject=elt, version=version) foundErrors = True except (InvalidOperation, OverflowError): modelXbrl.error("streamingExtensions:versionError", _("Version %(version)s, number must be 1.n"), modelObject=elt, version=streamingAspects.get("version", "(none)")) foundErrors = True for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"): try: bufLimit = Decimal(streamingAspects.get(bufAspect, "INF")) if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0): raise InvalidOperation elif bufAspect == "contextBuffer": contextBufferLimit = bufLimit elif bufAspect == "unitBuffer": unitBufferLimit = bufLimit elif bufAspect == "footnoteBuffer": footnoteBufferLimit = bufLimit except InvalidOperation: modelXbrl.error("streamingExtensions:valueError", _("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"), modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect)) foundErrors = True if _streamingExtensionsValidate: incompatibleValidations = [] _validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem _disclosureSystem = modelXbrl.modelManager.disclosureSystem if _validateDisclosureSystem and _disclosureSystem.validationType == "EFM": incompatibleValidations.append("EFM") if _validateDisclosureSystem and _disclosureSystem.validationType == "GFM": incompatibleValidations.append("GFM") if _validateDisclosureSystem and _disclosureSystem.validationType == "HMRC": incompatibleValidations.append("HMRC") if modelXbrl.modelManager.validateCalcLB: incompatibleValidations.append("calculation LB") if incompatibleValidations: modelXbrl.error("streamingExtensions:incompatibleValidation", _("Streaming instance validation does not support %(incompatibleValidations)s validation"), modelObject=modelXbrl, incompatibleValidations=', '.join(incompatibleValidations)) foundErrors = True if instInfoContext.error_log: foundErrors = True logSyntaxErrors(instInfoContext) del instInfoContext # dereference for pluginMethod in pluginClassMethods("Streaming.BlockStreaming"): _blockingPluginName = pluginMethod(modelXbrl) if _blockingPluginName: # name of blocking plugin is returned modelXbrl.error("streamingExtensions:incompatiblePlugIn", _("Streaming instance not supported by plugin %(blockingPlugin)s"), modelObject=modelXbrl, blockingPlugin=_blockingPluginName) foundErrors = True if foundErrors: _file.close() return None _encoding = XmlUtil.encoding(_file.read(512)) _file.seek(0,io.SEEK_SET) # allow reparsing if _streamingExtensionsValidate: validator = Validate(modelXbrl) instValidator = validator.instValidator contextBuffer = [] contextsToDrop = [] unitBuffer = [] unitsToDrop = [] footnoteBuffer = [] footnoteLinksToDrop = [] _streamingFactsPlugin = any(True for pluginMethod in pluginClassMethods("Streaming.Facts")) _streamingValidateFactsPlugin = (_streamingExtensionsValidate and any(True for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"))) ''' this is very much slower than iterparse class modelLoaderTarget(): def __init__(self, element_factory=None, parser=None): self.newTree = True self.currentMdlObj = None self.beforeInstanceStream = True self.beforeStartStreamingPlugin = True self.numRootFacts = 1 modelXbrl.makeelementParentModelObject = None modelXbrl.isStreamingMode = True self.factsCheckVersion = None self.factsCheckMd5s = Md5Sum() def start(self, tag, attrib, nsmap=None): modelXbrl.makeelementParentModelObject = self.currentMdlObj # pass parent to makeelement for ModelObjectFactory mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap) mdlObj.sourceline = 1 if self.newTree: self.newTree = False self.currentMdlObj = mdlObj modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree()) modelXbrl.modelDocument = modelDocument # needed for incremental validation mdlObj.init(modelDocument) modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement modelDocument.parserLookupName = _parserLookupName modelDocument.parserLookupClass = _parserLookupClass modelDocument.xmlRootElement = mdlObj modelDocument.schemaLocationElements.add(mdlObj) modelDocument.documentEncoding = _encoding modelDocument._creationSoftwareComment = creationSoftwareComment modelXbrl.info("streamingExtensions:streaming", _("Stream processing this instance."), modelObject = modelDocument) else: self.currentMdlObj.append(mdlObj) self.currentMdlObj = mdlObj mdlObj._init() ns = mdlObj.namespaceURI ln = mdlObj.localName if (self.beforeInstanceStream and ( (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or (ns == XbrlConst.xbrli and ln in ("context", "unit")) or (ns not in (XbrlConst.link, XbrlConst.xbrli)))): self.beforeInstanceStream = False if _streamingExtensionsValidate: instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters(modelXbrl.prefixedNamespaces)) else: # need default dimensions ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) elif not self.beforeInstanceStream and self.beforeStartStreamingPlugin: for pluginMethod in pluginClassMethods("Streaming.Start"): pluginMethod(modelXbrl) self.beforeStartStreamingPlugin = False return mdlObj def end(self, tag): modelDocument = modelXbrl.modelDocument mdlObj = self.currentMdlObj parentMdlObj = mdlObj.getparent() self.currentMdlObj = parentMdlObj ns = mdlObj.namespaceURI ln = mdlObj.localName if ns == XbrlConst.xbrli: if ln == "context": if mdlObj.get("sticky"): del mdlObj.attrib["sticky"] XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) else: if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit: # drop before adding as dropped may have same id as added cntx = contextBuffer.pop(0) if _streamingValidateFactsPlugin: contextsToDrop.append(cntx) else: dropContext(modelXbrl, cntx) del parentMdlObj[parentMdlObj.index(cntx)] cntx = None #>>XmlValidate.validate(modelXbrl, mdlObj) #>>modelDocument.contextDiscover(mdlObj) if contextBufferLimit.is_finite(): contextBuffer.append(mdlObj) if _streamingExtensionsValidate: contextsToCheck = (mdlObj,) instValidator.checkContexts(contextsToCheck) if modelXbrl.hasXDT: instValidator.checkContextsDimensions(contextsToCheck) del contextsToCheck # dereference elif ln == "unit": if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit: # drop before adding as dropped may have same id as added unit = unitBuffer.pop(0) if _streamingValidateFactsPlugin: unitsToDrop.append(unit) else: dropUnit(modelXbrl, unit) del parentMdlObj[parentMdlObj.index(unit)] unit = None #>>XmlValidate.validate(modelXbrl, mdlObj) #>>modelDocument.unitDiscover(mdlObj) if unitBufferLimit.is_finite(): unitBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkUnits( (mdlObj,) ) elif ln == "xbrl": # end of document # check remaining batched facts if any if _streamingValidateFactsPlugin: # plugin attempts to process batch of all root facts not yet processed (not just current one) # finish any final batch of facts if len(modelXbrl.facts) > 0: factsToCheck = modelXbrl.facts.copy() factsHaveBeenProcessed = True # can block facts deletion if required data not yet available, such as numeric unit for DpmDB for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"): if not pluginMethod(modelXbrl, factsToCheck): factsHaveBeenProcessed = False if factsHaveBeenProcessed: for fact in factsToCheck: dropFact(modelXbrl, fact, modelXbrl.facts) del parentMdlObj[parentMdlObj.index(fact)] for cntx in contextsToDrop: dropContext(modelXbrl, cntx) del parentMdlObj[parentMdlObj.index(cntx)] for unit in unitsToDrop: dropUnit(modelXbrl, unit) del parentMdlObj[parentMdlObj.index(unit)] for footnoteLink in footnoteLinksToDrop: dropFootnoteLink(modelXbrl, footnoteLink) del parentMdlObj[parentMdlObj.index(footnoteLink)] fact = cntx = unit = footnoteLink = None del contextsToDrop[:] del unitsToDrop[:] del footnoteLinksToDrop[:] del factsToCheck # check remaining footnote refs for footnoteLink in footnoteBuffer: checkFootnoteHrefs(modelXbrl, footnoteLink) for pluginMethod in pluginClassMethods("Streaming.Finish"): pluginMethod(modelXbrl) elif ns == XbrlConst.link: if ln == "footnoteLink": XmlValidate.validate(modelXbrl, mdlObj) footnoteLinks = (mdlObj,) modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True) if footnoteBufferLimit.is_finite(): footnoteBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkLinks(footnoteLinks) if len(footnoteBuffer) > footnoteBufferLimit: # check that hrefObjects for locators were all satisfied # drop before addition as dropped may have same id as added footnoteLink = footnoteBuffer.pop(0) checkFootnoteHrefs(modelXbrl, footnoteLink) if _streamingValidateFactsPlugin: footnoteLinksToDrop.append(footnoteLink) else: dropFootnoteLink(modelXbrl, footnoteLink) del parentMdlObj[parentMdlObj.index(footnoteLink)] footnoteLink = None footnoteLinks = None elif ln in ("schemaRef", "linkbaseRef"): modelDocument.discoverHref(mdlObj) elif not modelXbrl.skipDTS: if ln in ("roleRef", "arcroleRef"): modelDocument.linkbaseDiscover((mdlObj,), inInstance=True) elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl: self.numRootFacts += 1 #>>XmlValidate.validate(modelXbrl, mdlObj) #>>modelDocument.factDiscover(mdlObj, modelXbrl.facts) if self.factsCheckVersion: self.factCheckFact(mdlObj) if _streamingExtensionsValidate or _streamingValidateFactsPlugin: factsToCheck = (mdlObj,) # validate current fact by itself if _streamingExtensionsValidate: instValidator.checkFacts(factsToCheck) if modelXbrl.hasXDT: instValidator.checkFactsDimensions(factsToCheck) if _streamingValidateFactsPlugin: # plugin attempts to process batch of all root facts not yet processed (not just current one) # use batches of 1000 facts if len(modelXbrl.facts) > 1000: factsToCheck = modelXbrl.facts.copy() factsHaveBeenProcessed = True # can block facts deletion if required data not yet available, such as numeric unit for DpmDB for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"): if not pluginMethod(modelXbrl, factsToCheck): factsHaveBeenProcessed = False if factsHaveBeenProcessed: for fact in factsToCheck: dropFact(modelXbrl, fact, modelXbrl.facts) del parentMdlObj[parentMdlObj.index(fact)] for cntx in contextsToDrop: dropContext(modelXbrl, cntx) del parentMdlObj[parentMdlObj.index(cntx)] for unit in unitsToDrop: dropUnit(modelXbrl, unit) del parentMdlObj[parentMdlObj.index(unit)] for footnoteLink in footnoteLinksToDrop: dropFootnoteLink(modelXbrl, footnoteLink) del parentMdlObj[parentMdlObj.index(footnoteLink)] fact = cntx = unit = footnoteLink = None del contextsToDrop[:] del unitsToDrop[:] del footnoteLinksToDrop[:] del factsToCheck # dereference fact or batch of facts else: dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed del parentMdlObj[parentMdlObj.index(mdlObj)] if self.numRootFacts % 1000 == 0: pass #modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, # 100.0 * self.numRootFacts / instInfoNumRootFacts), # minTimeToShow=20.0) gc.collect() sys.stdout.write ("\rAt fact {} of {} mem {}".format(self.numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed)) return mdlObj def data(self, data): self.currentMdlObj.text = data def comment(self, text): pass def pi(self, target, data): if target == "xbrl-facts-check": _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", data) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "version": self.factsCheckVersion = _matchGroups[1] elif _matchGroups[0] == "sum-of-fact-md5s": try: expectedMd5 = Md5Sum(_matchGroups[1]) if self.factsCheckMd5s != expectedMd5: modelXbrl.warning("streamingExtensions:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"), modelObject=modelXbrl, expectedMd5=expectedMd5, actualMd5Sum=self.factsCheckMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) except ValueError: modelXbrl.error("streamingExtensions:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) def close(self): del modelXbrl.makeelementParentModelObject return None def factCheckFact(self, fact): self.factsCheckMd5s += fact.md5sum for _tupleFact in fact.modelTupleFacts: self.factCheckFact(_tupleFact) _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget()) etree.parse(_file, parser=_parser, base_url=filepath) logSyntaxErrors(_parser) ''' # replace modelLoaderTarget with iterparse (as it now supports CustomElementClassLookup) streamingParserContext = etree.iterparse(_file, events=("start","end"), huge_tree=True) from arelle.ModelObjectFactory import setParserElementClassLookup modelXbrl.isStreamingMode = True # must be set before setting element class lookup (_parser, _parserLookupName, _parserLookupClass) = setParserElementClassLookup(streamingParserContext, modelXbrl) foundInstance = False beforeInstanceStream = beforeStartStreamingPlugin = True numRootFacts = 0 factsCheckVersion = None def factCheckFact(fact): modelDocument._factsCheckMd5s += fact.md5sum for _tupleFact in fact.modelTupleFacts: factCheckFact(_tupleFact) for event, mdlObj in streamingParserContext: if event == "start": if mdlObj.tag == "{http://www.xbrl.org/2003/instance}xbrl": modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree()) modelXbrl.modelDocument = modelDocument # needed for incremental validation mdlObj.init(modelDocument) modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement modelDocument.parserLookupName = _parserLookupName modelDocument.parserLookupClass = _parserLookupClass modelDocument.xmlRootElement = mdlObj modelDocument.schemaLocationElements.add(mdlObj) modelDocument.documentEncoding = _encoding modelDocument._creationSoftwareComment = precedingComment(mdlObj) modelDocument._factsCheckMd5s = Md5Sum() modelXbrl.info("streamingExtensions:streaming", _("Stream processing this instance."), modelObject = modelDocument) elif mdlObj.getparent() is not None: mdlObj._init() # requires discovery as part of start elements if mdlObj.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl": if not foundInstance: foundInstance = True pi = precedingProcessingInstruction(mdlObj, "xbrl-facts-check") if pi is not None: factsCheckVersion = pi.attrib.get("version", None) elif not foundInstance: break ns = mdlObj.qname.namespaceURI ln = mdlObj.qname.localName if beforeInstanceStream: if ((ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or (ns == XbrlConst.xbrli and ln in ("context", "unit")) or (ns not in (XbrlConst.link, XbrlConst.xbrli))): beforeInstanceStream = False if _streamingExtensionsValidate: instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters(modelXbrl.prefixedNamespaces)) else: # need default dimensions ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) elif not beforeInstanceStream and beforeStartStreamingPlugin: for pluginMethod in pluginClassMethods("Streaming.Start"): pluginMethod(modelXbrl) beforeStartStreamingPlugin = False elif event == "end": parentMdlObj = mdlObj.getparent() ns = mdlObj.namespaceURI ln = mdlObj.localName if ns == XbrlConst.xbrli: if ln == "context": if mdlObj.get("sticky"): del mdlObj.attrib["sticky"] XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) else: if len(contextBuffer) >= contextBufferLimit: # drop before adding as dropped may have same id as added cntx = contextBuffer.pop(0) if _streamingFactsPlugin or _streamingValidateFactsPlugin: contextsToDrop.append(cntx) else: dropContext(modelXbrl, cntx) #>>del parentMdlObj[parentMdlObj.index(cntx)] cntx = None XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) if contextBufferLimit.is_finite(): contextBuffer.append(mdlObj) if _streamingExtensionsValidate: contextsToCheck = (mdlObj,) instValidator.checkContexts(contextsToCheck) if modelXbrl.hasXDT: instValidator.checkContextsDimensions(contextsToCheck) del contextsToCheck # dereference elif ln == "unit": if len(unitBuffer) >= unitBufferLimit: # drop before additing as dropped may have same id as added unit = unitBuffer.pop(0) if _streamingFactsPlugin or _streamingValidateFactsPlugin: unitsToDrop.append(unit) else: dropUnit(modelXbrl, unit) #>>del parentMdlObj[parentMdlObj.index(unit)] unit = None XmlValidate.validate(modelXbrl, mdlObj) modelDocument.unitDiscover(mdlObj) if unitBufferLimit.is_finite(): unitBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkUnits( (mdlObj,) ) elif ln == "xbrl": # end of document # check remaining batched facts if any if _streamingFactsPlugin or _streamingValidateFactsPlugin: # plugin attempts to process batch of all root facts not yet processed (not just current one) # finish any final batch of facts if len(modelXbrl.facts) > 0: factsToCheck = modelXbrl.facts.copy() # can block facts deletion if required data not yet available, such as numeric unit for DpmDB if _streamingValidateFactsPlugin: for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"): pluginMethod(instValidator, factsToCheck) if _streamingFactsPlugin: for pluginMethod in pluginClassMethods("Streaming.Facts"): pluginMethod(modelXbrl, factsToCheck) for fact in factsToCheck: dropFact(modelXbrl, fact, modelXbrl.facts) #>>del parentMdlObj[parentMdlObj.index(fact)] for cntx in contextsToDrop: dropContext(modelXbrl, cntx) #>>del parentMdlObj[parentMdlObj.index(cntx)] for unit in unitsToDrop: dropUnit(modelXbrl, unit) #>>del parentMdlObj[parentMdlObj.index(unit)] for footnoteLink in footnoteLinksToDrop: dropFootnoteLink(modelXbrl, footnoteLink) #>>del parentMdlObj[parentMdlObj.index(footnoteLink)] fact = cntx = unit = footnoteLink = None del contextsToDrop[:] del unitsToDrop[:] del footnoteLinksToDrop[:] del factsToCheck # check remaining footnote refs for footnoteLink in footnoteBuffer: checkFootnoteHrefs(modelXbrl, footnoteLink) pi = childProcessingInstruction(mdlObj, "xbrl-facts-check", reversed=True) if pi is not None: # attrib is in .text, not attrib, no idea why!!! _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "sum-of-fact-md5s": try: expectedMd5 = Md5Sum(_matchGroups[1]) if modelDocument._factsCheckMd5s != expectedMd5: modelXbrl.warning("streamingExtensions:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"), modelObject=modelXbrl, expectedMd5=expectedMd5, actualMd5Sum=modelDocument._factsCheckMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) except ValueError: modelXbrl.error("streamingExtensions:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) if _streamingValidateFactsPlugin: for pluginMethod in pluginClassMethods("Streaming.ValidateFinish"): pluginMethod(instValidator) if _streamingFactsPlugin: for pluginMethod in pluginClassMethods("Streaming.Finish"): pluginMethod(modelXbrl) elif ns == XbrlConst.link: if ln in ("schemaRef", "linkbaseRef"): modelDocument.discoverHref(mdlObj, urlRewritePluginClass="ModelDocument.InstanceSchemaRefRewriter") elif ln in ("roleRef", "arcroleRef"): modelDocument.linkbaseDiscover((mdlObj,), inInstance=True) elif ln == "footnoteLink": XmlValidate.validate(modelXbrl, mdlObj) footnoteLinks = (mdlObj,) modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True) if footnoteBufferLimit.is_finite(): footnoteBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkLinks(footnoteLinks) if len(footnoteBuffer) > footnoteBufferLimit: # check that hrefObjects for locators were all satisfied # drop before addition as dropped may have same id as added footnoteLink = footnoteBuffer.pop(0) checkFootnoteHrefs(modelXbrl, footnoteLink) if _streamingValidateFactsPlugin: footnoteLinksToDrop.append(footnoteLink) else: dropFootnoteLink(modelXbrl, footnoteLink) #>>del parentMdlObj[parentMdlObj.index(footnoteLink)] footnoteLink = None footnoteLinks = None elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl and isinstance(mdlObj, ModelFact): numRootFacts += 1 XmlValidate.validate(modelXbrl, mdlObj) modelDocument.factDiscover(mdlObj, modelXbrl.facts) if factsCheckVersion: factCheckFact(mdlObj) if _streamingExtensionsValidate or _streamingFactsPlugin or _streamingValidateFactsPlugin: factsToCheck = (mdlObj,) # validate current fact by itself if _streamingExtensionsValidate: instValidator.checkFacts(factsToCheck) if modelXbrl.hasXDT: instValidator.checkFactsDimensions(factsToCheck) if _streamingFactsPlugin or _streamingValidateFactsPlugin: # plugin attempts to process batch of all root facts not yet processed (not just current one) # use batches of 1000 facts if len(modelXbrl.facts) > 1000: factsToCheck = modelXbrl.facts.copy() # can block facts deletion if required data not yet available, such as numeric unit for DpmDB if _streamingValidateFactsPlugin: for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"): pluginMethod(instValidator, factsToCheck) if _streamingFactsPlugin: for pluginMethod in pluginClassMethods("Streaming.Facts"): pluginMethod(modelXbrl, factsToCheck) for fact in factsToCheck: dropFact(modelXbrl, fact, modelXbrl.facts) #>>del parentMdlObj[parentMdlObj.index(fact)] for cntx in contextsToDrop: dropContext(modelXbrl, cntx) #>>del parentMdlObj[parentMdlObj.index(cntx)] for unit in unitsToDrop: dropUnit(modelXbrl, unit) #>>del parentMdlObj[parentMdlObj.index(unit)] for footnoteLink in footnoteLinksToDrop: dropFootnoteLink(modelXbrl, footnoteLink) #>>del parentMdlObj[parentMdlObj.index(footnoteLink)] fact = cntx = unit = footnoteLink = None del contextsToDrop[:] del unitsToDrop[:] del footnoteLinksToDrop[:] del factsToCheck # dereference fact or batch of facts else: dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed #>>del parentMdlObj[parentMdlObj.index(mdlObj)] if numRootFacts % 1000 == 0: pass #modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, # 100.0 * self.numRootFacts / instInfoNumRootFacts), # minTimeToShow=20.0) #gc.collect() #sys.stdout.write ("\rAt fact {} of {} mem {}".format(numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed)) if mdlObj is not None: mdlObj.clear() del _parser, _parserLookupName, _parserLookupClass if _streamingExtensionsValidate and validator is not None: _file.close() del instValidator validator.close() # track that modelXbrl has been validated by this streaming extension modelXbrl._streamingExtensionValidated = True modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt) return modelXbrl.modelDocument
def createTargetInstance(modelXbrl, targetUrl, targetDocumentSchemaRefs, filingFiles, baseXmlLang=None, defaultXmlLang=None): targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True, discover=False) # don't attempt to load DTS if baseXmlLang: targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang) if defaultXmlLang is None: defaultXmlLang = baseXmlLang # allows facts/footnotes to override baseXmlLang ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in sorted(modelXbrl.contexts.values(), key=lambda c: c.objectIndex): # contexts may come from multiple IXDS files ignore = targetInstance.createContext(context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in sorted(modelXbrl.units.values(), key=lambda u: u.objectIndex): # units may come from multiple IXDS files measures = unit.measures ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default xmlLang = fact.xmlLang if xmlLang is not None and xmlLang != defaultXmlLang: attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) # if fact.isFraction, create numerator and denominator newFactForOldObjId[fact.objectIndex] = newFact if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references so that referenced files are included in the zip. for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter(): addLocallyReferencedFile(elt, filingFiles) except (XMLSyntaxError, UnicodeDecodeError): pass # TODO: Why ignore UnicodeDecodeError? elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) footnoteIdCount = {} for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format(attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) xmlLang = linkChild.xmlLang if xmlLang is not None and xmlLang != defaultXmlLang: # default newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang) copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): addLocallyReferencedFile(elt,filingFiles) return targetInstance
def validate(self, modelXbrl, parameters=None): self.parameters = parameters self.precisionPattern = re.compile("^([0-9]+|INF)$") self.decimalsPattern = re.compile("^(-?[0-9]+|INF)$") self.isoCurrencyPattern = re.compile(r"^[A-Z]{3}$") self.modelXbrl = modelXbrl self.validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem self.disclosureSystem = modelXbrl.modelManager.disclosureSystem self.validateEFM = self.validateDisclosureSystem and self.disclosureSystem.EFM self.validateGFM = self.validateDisclosureSystem and self.disclosureSystem.GFM self.validateEFMorGFM = self.validateDisclosureSystem and self.disclosureSystem.EFMorGFM self.validateHMRC = self.validateDisclosureSystem and self.disclosureSystem.HMRC self.validateSBRNL = self.validateDisclosureSystem and self.disclosureSystem.SBRNL self.validateXmlLang = self.validateDisclosureSystem and self.disclosureSystem.xmlLangPattern self.validateCalcLB = modelXbrl.modelManager.validateCalcLB self.validateInferDecimals = modelXbrl.modelManager.validateInferDecimals # xlink validation modelXbrl.profileStat(None) modelXbrl.modelManager.showStatus(_("validating links")) modelLinks = set() self.remoteResourceLocElements = set() self.genericArcArcroles = set() for baseSetExtLinks in modelXbrl.baseSets.values(): for baseSetExtLink in baseSetExtLinks: modelLinks.add(baseSetExtLink) # ext links are unique (no dups) for modelLink in modelLinks: fromToArcs = {} locLabels = {} resourceLabels = {} resourceArcTos = [] for arcElt in modelLink.iterchildren(): if isinstance(arcElt,ModelObject): xlinkType = arcElt.get("{http://www.w3.org/1999/xlink}type") # locator must have an href if xlinkType == "locator": if arcElt.get("{http://www.w3.org/1999/xlink}href") is None: modelXbrl.error("xlink:locatorHref", _("Xlink locator %(xlinkLabel)s missing href in extended link %(linkrole)s"), modelObject=arcElt, linkrole=modelLink.role, xlinkLabel=arcElt.get("{http://www.w3.org/1999/xlink}label")) locLabels[arcElt.get("{http://www.w3.org/1999/xlink}label")] = arcElt elif xlinkType == "resource": resourceLabels[arcElt.get("{http://www.w3.org/1999/xlink}label")] = arcElt # can be no duplicated arcs between same from and to elif xlinkType == "arc": fromLabel = arcElt.get("{http://www.w3.org/1999/xlink}from") toLabel = arcElt.get("{http://www.w3.org/1999/xlink}to") fromTo = (fromLabel,toLabel) if fromTo in fromToArcs: modelXbrl.error("xlink:dupArcs", _("Duplicate xlink arcs in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s"), modelObject=arcElt, linkrole=modelLink.role, xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel) else: fromToArcs[fromTo] = arcElt if arcElt.namespaceURI == XbrlConst.link: if arcElt.localName in arcNamesTo21Resource: #("labelArc","referenceArc"): resourceArcTos.append((toLabel, arcElt.get("use"), arcElt)) elif self.isGenericArc(arcElt): arcrole = arcElt.get("{http://www.w3.org/1999/xlink}arcrole") self.genericArcArcroles.add(arcrole) if arcrole in (XbrlConst.elementLabel, XbrlConst.elementReference): resourceArcTos.append((toLabel, arcrole, arcElt)) # values of type (not needed for validating parsers) if xlinkType not in xlinkTypeValues: # ("", "simple", "extended", "locator", "arc", "resource", "title", "none"): modelXbrl.error("xlink:type", _("Xlink type %(xlinkType)s invalid in extended link %(linkrole)s"), modelObject=arcElt, linkrole=modelLink.role, xlinkType=xlinkType) # values of actuate (not needed for validating parsers) xlinkActuate = arcElt.get("{http://www.w3.org/1999/xlink}actuate") if xlinkActuate not in xlinkActuateValues: # ("", "onLoad", "onRequest", "other", "none"): modelXbrl.error("xlink:actuate", _("Actuate %(xlinkActuate)s invalid in extended link %(linkrole)s"), modelObject=arcElt, linkrole=modelLink.role, xlinkActuate=xlinkActuate) # values of show (not needed for validating parsers) xlinkShow = arcElt.get("{http://www.w3.org/1999/xlink}show") if xlinkShow not in xlinkShowValues: # ("", "new", "replace", "embed", "other", "none"): modelXbrl.error("xlink:show", _("Show %(xlinkShow)s invalid in extended link %(linkrole)s"), modelObject=arcElt, linkrole=modelLink.role, xlinkShow=xlinkShow) # check from, to of arcs have a resource or loc for fromTo, arcElt in fromToArcs.items(): fromLabel, toLabel = fromTo for name, value, sect in (("from", fromLabel, "3.5.3.9.2"),("to",toLabel, "3.5.3.9.3")): if value not in locLabels and value not in resourceLabels: modelXbrl.error("xbrl.{0}:arcResource".format(sect), _("Arc in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s attribute '%(attribute)s' has no matching loc or resource label"), modelObject=arcElt, linkrole=modelLink.role, xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel, attribute=name) if arcElt.localName == "footnoteArc" and arcElt.namespaceURI == XbrlConst.link and \ arcElt.get("{http://www.w3.org/1999/xlink}arcrole") == XbrlConst.factFootnote: if fromLabel not in locLabels: modelXbrl.error("xbrl.4.11.1.3.1:factFootnoteArcFrom", _("Footnote arc in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s \"from\" is not a loc"), modelObject=arcElt, linkrole=modelLink.role, xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel) if toLabel not in resourceLabels or qname(resourceLabels[toLabel]) != XbrlConst.qnLinkFootnote: modelXbrl.error("xbrl.4.11.1.3.1:factFootnoteArcTo", _("Footnote arc in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s \"to\" is not a footnote resource"), modelObject=arcElt, linkrole=modelLink.role, xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel) # check unprohibited label arcs to remote locs for resourceArcTo in resourceArcTos: resourceArcToLabel, resourceArcUse, arcElt = resourceArcTo if resourceArcToLabel in locLabels: toLabel = locLabels[resourceArcToLabel] if resourceArcUse == "prohibited": self.remoteResourceLocElements.add(toLabel) else: modelXbrl.error("xbrl.5.2.2.3:labelArcRemoteResource", _("Unprohibited labelArc in extended link %(linkrole)s has illegal remote resource loc labeled %(xlinkLabel)s href %(xlinkHref)s"), modelObject=arcElt, linkrole=modelLink.role, xlinkLabel=resourceArcToLabel, xlinkHref=toLabel.get("{http://www.w3.org/1999/xlink}href")) elif resourceArcToLabel in resourceLabels: toResource = resourceLabels[resourceArcToLabel] if resourceArcUse == XbrlConst.elementLabel: if not self.isGenericLabel(toResource): modelXbrl.error("xbrlle.2.1.1:genericLabelTarget", _("Generic label arc in extended link %(linkrole)s to %(xlinkLabel)s must target a generic label"), modelObject=arcElt, linkrole=modelLink.role, xlinkLabel=resourceArcToLabel) elif resourceArcUse == XbrlConst.elementReference: if not self.isGenericReference(toResource): modelXbrl.error("xbrlre.2.1.1:genericReferenceTarget", _("Generic reference arc in extended link %(linkrole)s to %(xlinkLabel)s must target a generic reference"), modelObject=arcElt, linkrole=modelLink.role, xlinkLabel=resourceArcToLabel) resourceArcTos = None # dereference arcs modelXbrl.profileStat(_("validateLinks")) modelXbrl.dimensionDefaultConcepts = {} modelXbrl.qnameDimensionDefaults = {} modelXbrl.qnameDimensionContextElement = {} # check base set cycles, dimensions modelXbrl.modelManager.showStatus(_("validating relationship sets")) for baseSetKey in modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if arcrole.startswith("XBRL-") or ELR is None or \ linkqname is None or arcqname is None: continue elif arcrole in XbrlConst.standardArcroleCyclesAllowed: # TODO: table should be in this module, where it is used cyclesAllowed, specSect = XbrlConst.standardArcroleCyclesAllowed[arcrole] elif arcrole in self.modelXbrl.arcroleTypes and len(self.modelXbrl.arcroleTypes[arcrole]) > 0: cyclesAllowed = self.modelXbrl.arcroleTypes[arcrole][0].cyclesAllowed if arcrole in self.genericArcArcroles: specSect = "xbrlgene:violatedCyclesConstraint" else: specSect = "xbrl.5.1.4.3:cycles" else: cyclesAllowed = "any" specSect = None if cyclesAllowed != "any" or arcrole in (XbrlConst.summationItem,) \ or arcrole in self.genericArcArcroles \ or arcrole.startswith(XbrlConst.formulaStartsWith): relsSet = modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname) if cyclesAllowed != "any" and \ (XbrlConst.isStandardExtLinkQname(linkqname) and XbrlConst.isStandardArcQname(arcqname)) \ or arcrole in self.genericArcArcroles: noUndirected = cyclesAllowed == "none" fromRelationships = relsSet.fromModelObjects() for relFrom, rels in fromRelationships.items(): cycleFound = self.fwdCycle(relsSet, rels, noUndirected, {relFrom}) if cycleFound is not None: path = str(relFrom.qname) + " " + " - ".join( "{0}:{1} {2}".format(rel.modelDocument.basename, rel.sourceline, rel.toModelObject.qname) for rel in reversed(cycleFound[1:])) modelXbrl.error(specSect, _("Relationships have a %(cycle)s cycle in arcrole %(arcrole)s \nlink role %(linkrole)s \nlink %(linkname)s, \narc %(arcname)s, \npath %(path)s"), modelObject=cycleFound[1:], cycle=cycleFound[0], path=path, arcrole=arcrole, linkrole=ELR, linkname=linkqname, arcname=arcqname), break # check calculation arcs for weight issues (note calc arc is an "any" cycles) if arcrole == XbrlConst.summationItem: for modelRel in relsSet.modelRelationships: weight = modelRel.weight fromConcept = modelRel.fromModelObject toConcept = modelRel.toModelObject if fromConcept is not None and toConcept is not None: if weight == 0: modelXbrl.error("xbrl.5.2.5.2.1:zeroWeight", _("Calculation relationship has zero weight from %(source)s to %(target)s in link role %(linkrole)s"), modelObject=modelRel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR), fromBalance = fromConcept.balance toBalance = toConcept.balance if fromBalance and toBalance: if (fromBalance == toBalance and weight < 0) or \ (fromBalance != toBalance and weight > 0): modelXbrl.error("xbrl.5.1.1.2:balanceCalcWeightIllegal" + ("Negative" if weight < 0 else "Positive"), _("Calculation relationship has illegal weight %(weight)s from %(source)s, %(sourceBalance)s, to %(target)s, %(targetBalance)s, in link role %(linkrole)s (per 5.1.1.2 Table 6)"), modelObject=modelRel, weight=weight, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR, sourceBalance=fromBalance, targetBalance=toBalance) if not fromConcept.isNumeric or not toConcept.isNumeric: modelXbrl.error("xbrl.5.2.5.2:nonNumericCalc", _("Calculation relationship has illegal concept from %(source)s%(sourceNumericDecorator)s to %(target)s%(targetNumericDecorator)s in link role %(linkrole)s"), modelObject=modelRel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR, sourceNumericDecorator="" if fromConcept.isNumeric else _(" (non-numeric)"), targetNumericDecorator="" if toConcept.isNumeric else _(" (non-numeric)")) # check presentation relationships for preferredLabel issues elif arcrole == XbrlConst.parentChild: for modelRel in relsSet.modelRelationships: preferredLabel = modelRel.preferredLabel toConcept = modelRel.toModelObject if preferredLabel is not None and toConcept is not None and \ toConcept.label(preferredLabel=preferredLabel,fallbackToQname=False) is None: modelXbrl.error("xbrl.5.2.4.2.1:preferredLabelMissing", _("Presentation relationship from %(source)s to %(target)s in link role %(linkrole)s missing preferredLabel %(preferredLabel)s"), modelObject=modelRel, source=modelRel.fromModelObject.qname, target=toConcept.qname, linkrole=ELR, preferredLabel=preferredLabel) # check essence-alias relationships elif arcrole == XbrlConst.essenceAlias: for modelRel in relsSet.modelRelationships: fromConcept = modelRel.fromModelObject toConcept = modelRel.toModelObject if fromConcept is not None and toConcept is not None: if fromConcept.type != toConcept.type or fromConcept.periodType != toConcept.periodType: modelXbrl.error("xbrl.5.2.6.2.2:essenceAliasTypes", _("Essence-alias relationship from %(source)s to %(target)s in link role %(linkrole)s has different types or periodTypes"), modelObject=modelRel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR) fromBalance = fromConcept.balance toBalance = toConcept.balance if fromBalance and toBalance: if fromBalance and toBalance and fromBalance != toBalance: modelXbrl.error("xbrl.5.2.6.2.2:essenceAliasBalance", _("Essence-alias relationship from %(source)s to %(target)s in link role %(linkrole)s has different balances")).format( modelObject=modelRel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR) elif modelXbrl.hasXDT and arcrole.startswith(XbrlConst.dimStartsWith): ValidateXbrlDimensions.checkBaseSet(self, arcrole, ELR, relsSet) elif (modelXbrl.hasFormulae or modelXbrl.hasTableRendering) and arcrole.startswith(XbrlConst.formulaStartsWith): ValidateFormula.checkBaseSet(self, arcrole, ELR, relsSet) modelXbrl.isDimensionsValidated = True modelXbrl.profileStat(_("validateRelationships")) # instance checks modelXbrl.modelManager.showStatus(_("validating instance")) self.footnoteRefs = set() if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE or \ modelXbrl.modelDocument.type == ModelDocument.Type.INLINEXBRL: self.factsWithDeprecatedIxNamespace = [] self.checkFacts(modelXbrl.facts) if self.factsWithDeprecatedIxNamespace: self.modelXbrl.info("arelle:info", _("%(count)s facts have deprecated transformation namespace %(namespace)s"), modelObject=self.factsWithDeprecatedIxNamespace, count=len(self.factsWithDeprecatedIxNamespace), namespace=FunctionIxt.deprecatedNamespaceURI) del self.factsWithDeprecatedIxNamespace #instance checks for cntx in modelXbrl.contexts.values(): if cntx.isStartEndPeriod: try: if cntx.endDatetime <= cntx.startDatetime: self.modelXbrl.error("xbrl.4.7.2:periodStartBeforeEnd", _("Context %(contextID)s must have startDate less than endDate"), modelObject=cntx, contextID=cntx.id) except (TypeError, ValueError) as err: self.modelXbrl.error("xbrl.4.7.2:contextDateError", _("Context %(contextID) startDate or endDate: %(error)s"), modelObject=cntx, contextID=cntx.id, error=err) elif cntx.isInstantPeriod: try: cntx.instantDatetime #parse field except ValueError as err: self.modelXbrl.error("xbrl.4.7.2:contextDateError", _("Context %(contextID)s instant date: %(error)s"), modelObject=cntx, contextID=cntx.id, error=err) self.segmentScenario(cntx.segment, cntx.id, "segment", "4.7.3.2") self.segmentScenario(cntx.scenario, cntx.id, "scenario", "4.7.4") for unit in modelXbrl.units.values(): mulDivMeasures = unit.measures if mulDivMeasures: for measures in mulDivMeasures: for measure in measures: if measure.namespaceURI == XbrlConst.xbrli and not \ measure in (XbrlConst.qnXbrliPure, XbrlConst.qnXbrliShares): self.modelXbrl.error("xbrl.4.8.2:measureElement", _("Unit %(unitID)s illegal measure: %(measure)s"), modelObject=unit, unitID=unit.id, measure=measure) for numeratorMeasure in mulDivMeasures[0]: if numeratorMeasure in mulDivMeasures[1]: self.modelXbrl.error("xbrl.4.8.4:measureBothNumDenom", _("Unit %(unitID)s numerator measure: %(measure)s also appears as denominator measure"), modelObject=unit, unitID=unit.id, measure=numeratorMeasure) modelXbrl.profileStat(_("validateInstance")) if modelXbrl.hasXDT: modelXbrl.modelManager.showStatus(_("validating dimensions")) ''' uncomment if using otherFacts in checkFact dimCheckableFacts = set(f for f in modelXbrl.factsInInstance if f.concept.isItem and f.context is not None) while (dimCheckableFacts): # check one and all of its compatible family members f = dimCheckableFacts.pop() ValidateXbrlDimensions.checkFact(self, f, dimCheckableFacts) del dimCheckableFacts ''' self.checkFactDimensions(modelXbrl.facts) # check fact dimensions in document order for cntx in modelXbrl.contexts.values(): ValidateXbrlDimensions.checkContext(self,cntx) modelXbrl.profileStat(_("validateDimensions")) # dimensional validity #concepts checks modelXbrl.modelManager.showStatus(_("validating concepts")) for concept in modelXbrl.qnameConcepts.values(): conceptType = concept.type if XbrlConst.isStandardNamespace(concept.qname.namespaceURI) or \ not concept.modelDocument.inDTS: continue if concept.isTuple: # must be global if not concept.getparent().localName == "schema": self.modelXbrl.error("xbrl.4.9:tupleGloballyDeclared", _("Tuple %(concept)s must be declared globally"), modelObject=concept, concept=concept.qname) if concept.periodType: self.modelXbrl.error("xbrl.4.9:tuplePeriodType", _("Tuple %(concept)s must not have periodType"), modelObject=concept, concept=concept.qname) if concept.balance: self.modelXbrl.error("xbrl.4.9:tupleBalance", _("Tuple %(concept)s must not have balance"), modelObject=concept, concept=concept.qname) if conceptType is not None: # check attribute declarations for attribute in conceptType.attributes.values(): if attribute.qname.namespaceURI in (XbrlConst.xbrli, XbrlConst.link, XbrlConst.xlink, XbrlConst.xl): self.modelXbrl.error("xbrl.4.9:tupleAttribute", _("Tuple %(concept)s must not have attribute in this namespace %(attribute)s"), modelObject=concept, concept=concept.qname, attribute=attribute.qname) # check for mixed="true" or simple content if XmlUtil.descendantAttr(conceptType, XbrlConst.xsd, ("complexType", "complexContent"), "mixed") == "true": self.modelXbrl.error("xbrl.4.9:tupleMixedContent", _("Tuple %(concept)s must not have mixed content"), modelObject=concept, concept=concept.qname) if XmlUtil.descendant(conceptType, XbrlConst.xsd, "simpleContent"): self.modelXbrl.error("xbrl.4.9:tupleSimpleContent", _("Tuple %(concept)s must not have simple content"), modelObject=concept, concept=concept.qname) # child elements must be item or tuple for elementQname in conceptType.elements: childConcept = self.modelXbrl.qnameConcepts.get(elementQname) if childConcept is None: self.modelXbrl.error("xbrl.4.9:tupleElementUndefined", _("Tuple %(concept)s element %(tupleElement)s not defined"), modelObject=concept, concept=str(concept.qname), tupleElement=elementQname) elif not (childConcept.isItem or childConcept.isTuple or # isItem/isTuple do not include item or tuple itself childConcept.qname == XbrlConst.qnXbrliItem or # subs group includes item as member childConcept.qname == XbrlConst.qnXbrliTuple): self.modelXbrl.error("xbrl.4.9:tupleElementItemOrTuple", _("Tuple %(concept)s must not have element %(tupleElement)s not an item or tuple"), modelObject=concept, concept=concept.qname, tupleElement=elementQname) elif concept.isItem: if concept.periodType not in periodTypeValues: #("instant","duration"): self.modelXbrl.error("xbrl.5.1.1.1:itemPeriodType", _("Item %(concept)s must have a valid periodType"), modelObject=concept, concept=concept.qname) if concept.isMonetary: if concept.balance not in balanceValues: #(None, "credit","debit"): self.modelXbrl.error("xbrl.5.1.1.2:itemBalance", _("Item %(concept)s must have a valid balance %(balance)s"), modelObject=concept, concept=concept.qname, balance=concept.balance) else: if concept.balance: self.modelXbrl.error("xbrl.5.1.1.2:itemBalance", _("Item %(concept)s may not have a balance"), modelObject=concept, concept=concept.qname) if concept.baseXbrliType not in baseXbrliTypes: self.modelXbrl.error("xbrl.5.1.1.3:itemType", _("Item %(concept)s type %(itemType)s invalid"), modelObject=concept, concept=concept.qname, itemType=concept.baseXbrliType) if modelXbrl.hasXDT: if concept.isHypercubeItem and not concept.abstract == "true": self.modelXbrl.error("xbrldte:HypercubeElementIsNotAbstractError", _("Hypercube item %(concept)s must be abstract"), modelObject=concept, concept=concept.qname) elif concept.isDimensionItem and not concept.abstract == "true": self.modelXbrl.error("xbrldte:DimensionElementIsNotAbstractError", _("Dimension item %(concept)s must be abstract"), modelObject=concept, concept=concept.qname) if modelXbrl.hasXDT: ValidateXbrlDimensions.checkConcept(self, concept) modelXbrl.profileStat(_("validateConcepts")) modelXbrl.modelManager.showStatus(_("validating DTS")) self.DTSreferenceResourceIDs = {} ValidateXbrlDTS.checkDTS(self, modelXbrl.modelDocument, []) del self.DTSreferenceResourceIDs global validateUniqueParticleAttribution if validateUniqueParticleAttribution is None: from arelle.XmlValidateParticles import validateUniqueParticleAttribution for modelType in modelXbrl.qnameTypes.values(): validateUniqueParticleAttribution(modelXbrl, modelType.particlesList, modelType) modelXbrl.profileStat(_("validateDTS")) if self.validateCalcLB: modelXbrl.modelManager.showStatus(_("Validating instance calculations")) ValidateXbrlCalcs.validate(modelXbrl, inferDecimals=self.validateInferDecimals) modelXbrl.profileStat(_("validateCalculations")) if (modelXbrl.modelManager.validateUtr or (self.parameters and self.parameters.get(qname("forceUtrValidation",noPrefixIsNoNamespace=True),(None,"false"))[1] == "true") or #(self.validateEFM and #any((concept.namespaceURI in self.disclosureSystem.standardTaxonomiesDict) # for concept in self.modelXbrl.nameConcepts.get("UTR",())))): (self.validateEFM and any(modelDoc.definesUTR for modelDoc in self.modelXbrl.urlDocs.values()))): ValidateUtr.validate(modelXbrl) modelXbrl.profileStat(_("validateUTR")) if modelXbrl.hasFormulae or modelXbrl.modelRenderingTables: ValidateFormula.validate(self, statusMsg=_("compiling formulae and rendering tables") if (modelXbrl.hasFormulae and modelXbrl.modelRenderingTables) else (_("compiling formulae") if modelXbrl.hasFormulae else _("compiling rendering tables"))) modelXbrl.modelManager.showStatus(_("ready"), 2000)
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, *args, **kwargs): # check if big instance and has header with an initial incomplete tree walk (just 2 elements if not _streamingExtensionsCheck: return None # track whether modelXbrl has been validated by this streaming extension modelXbrl._streamingExtensionValidated = False def logSyntaxErrors(parsercontext): for error in parsercontext.error_log: modelXbrl.error( "xmlSchema:syntax", _("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element" ), modelObject=modelXbrl, fileName=os.path.basename(filepath), error=error.message, line=error.line, column=error.column, sourceAction="streaming") #### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ### #### note: iterparse wants a binary file, but file is text mode _file, = modelXbrl.fileSource.file(filepath, binary=True) startedAt = time.time() modelXbrl.profileActivity() ''' this seems twice as slow as iterparse class instInfoTarget(): def __init__(self, element_factory=None, parser=None): self.newTree = True self.streamingAspects = None self.foundInstance = False self.creationSoftwareComment = '' self.currentEltTag = "(before xbrli:xbrl)" self.numRootFacts = 0 def start(self, tag, attrib, nsmap=None): if self.newTree: if tag == "{http://www.xbrl.org/2003/instance}xbrl": self.foundInstance = True self.newTree = False else: # break raise NotInstanceDocumentException() elif not tag.startswith("{http://www.xbrl.org/"): self.numRootFacts += 1 if self.numRootFacts % 1000 == 0: modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0) self.currentEltTag = tag def end(self, tag): pass def data(self, data): pass def comment(self, text): if not self.foundInstance: # accumulate comments before xbrli:xbrl self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text elif not self.creationSoftwareComment: self.creationSoftwareComment = text # or first comment after xbrli:xbrl def pi(self, target, data): if target == "xbrl-streamable-instance": if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl": self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results else: modelXbrl.error("streamingExtensions:headerMisplaced", _("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"), modelObject=modelXbrl, target=target, element=self.currentEltTag) def close(self): if not self.creationSoftwareComment: self.creationSoftwareComment = None return True instInfo = instInfoTarget() infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo) try: etree.parse(_file, parser=infoParser, base_url=filepath) except NotInstanceDocumentException: pass ''' foundErrors = False foundInstance = False streamingAspects = None creationSoftwareComment = None instInfoNumRootFacts = 0 numElts = 0 elt = None instInfoContext = etree.iterparse(_file, events=("start", "end"), huge_tree=True) try: for event, elt in instInfoContext: if event == "start": if elt.getparent() is not None: if elt.getparent( ).tag == "{http://www.xbrl.org/2003/instance}xbrl": if not foundInstance: foundInstance = True pi = precedingProcessingInstruction( elt, "xbrl-streamable-instance") if pi is None: break else: streamingAspects = dict(pi.attrib.copy()) if creationSoftwareComment is None: creationSoftwareComment = precedingComment( elt) if not elt.tag.startswith("{http://www.xbrl.org/"): instInfoNumRootFacts += 1 if instInfoNumRootFacts % 1000 == 0: modelXbrl.profileActivity( "... streaming tree check", minTimeToShow=20.0) elif not foundInstance: break elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl": creationSoftwareComment = precedingComment(elt) if precedingProcessingInstruction( elt, "xbrl-streamable-instance") is not None: modelXbrl.error( "streamingExtensions:headerMisplaced", _("Header is misplaced: %(error)s, must follow xbrli:xbrl element" ), modelObject=elt) elif event == "end": elt.clear() numElts += 1 if numElts % 1000 == 0 and elt.getparent() is not None: while elt.getprevious() is not None and elt.getparent( ) is not None: del elt.getparent()[0] except etree.XMLSyntaxError as err: modelXbrl.error("xmlSchema:syntax", _("Unrecoverable error: %(error)s"), error=err) _file.close() return err _file.seek(0, io.SEEK_SET) # allow reparsing if not foundInstance or streamingAspects is None: del elt _file.close() return None modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt) startedAt = time.time() try: version = Decimal(streamingAspects.get("version")) if int(version) != 1: modelXbrl.error( "streamingExtensions:unsupportedVersion", _("Streaming version %(version)s, major version number must be 1" ), modelObject=elt, version=version) foundErrors = True except (InvalidOperation, OverflowError): modelXbrl.error("streamingExtensions:versionError", _("Version %(version)s, number must be 1.n"), modelObject=elt, version=streamingAspects.get("version", "(none)")) foundErrors = True for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"): try: bufLimit = Decimal(streamingAspects.get(bufAspect, "INF")) if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0): raise InvalidOperation elif bufAspect == "contextBuffer": contextBufferLimit = bufLimit elif bufAspect == "unitBuffer": unitBufferLimit = bufLimit elif bufAspect == "footnoteBuffer": footnoteBufferLimit = bufLimit except InvalidOperation: modelXbrl.error( "streamingExtensions:valueError", _("Streaming %(attrib)s %(value)s, number must be a positive integer or INF" ), modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect)) foundErrors = True if _streamingExtensionsValidate: incompatibleValidations = [] _validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem _disclosureSystem = modelXbrl.modelManager.disclosureSystem if _validateDisclosureSystem and _disclosureSystem.validationType == "EFM": incompatibleValidations.append("EFM") if _validateDisclosureSystem and _disclosureSystem.validationType == "GFM": incompatibleValidations.append("GFM") if _validateDisclosureSystem and _disclosureSystem.validationType == "HMRC": incompatibleValidations.append("HMRC") if modelXbrl.modelManager.validateCalcLB: incompatibleValidations.append("calculation LB") if incompatibleValidations: modelXbrl.error( "streamingExtensions:incompatibleValidation", _("Streaming instance validation does not support %(incompatibleValidations)s validation" ), modelObject=modelXbrl, incompatibleValidations=', '.join(incompatibleValidations)) foundErrors = True if instInfoContext.error_log: foundErrors = True logSyntaxErrors(instInfoContext) del instInfoContext # dereference for pluginMethod in pluginClassMethods("Streaming.BlockStreaming"): _blockingPluginName = pluginMethod(modelXbrl) if _blockingPluginName: # name of blocking plugin is returned modelXbrl.error( "streamingExtensions:incompatiblePlugIn", _("Streaming instance not supported by plugin %(blockingPlugin)s" ), modelObject=modelXbrl, blockingPlugin=_blockingPluginName) foundErrors = True if foundErrors: _file.close() return None _encoding = XmlUtil.encoding(_file.read(512)) _file.seek(0, io.SEEK_SET) # allow reparsing if _streamingExtensionsValidate: validator = Validate(modelXbrl) instValidator = validator.instValidator contextBuffer = [] contextsToDrop = [] unitBuffer = [] unitsToDrop = [] footnoteBuffer = [] footnoteLinksToDrop = [] _streamingFactsPlugin = any( True for pluginMethod in pluginClassMethods("Streaming.Facts")) _streamingValidateFactsPlugin = (_streamingExtensionsValidate and any( True for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"))) ''' this is very much slower than iterparse class modelLoaderTarget(): def __init__(self, element_factory=None, parser=None): self.newTree = True self.currentMdlObj = None self.beforeInstanceStream = True self.beforeStartStreamingPlugin = True self.numRootFacts = 1 modelXbrl.makeelementParentModelObject = None modelXbrl.isStreamingMode = True self.factsCheckVersion = None self.factsCheckMd5s = Md5Sum() def start(self, tag, attrib, nsmap=None): modelXbrl.makeelementParentModelObject = self.currentMdlObj # pass parent to makeelement for ModelObjectFactory mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap) mdlObj.sourceline = 1 if self.newTree: self.newTree = False self.currentMdlObj = mdlObj modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree()) modelXbrl.modelDocument = modelDocument # needed for incremental validation mdlObj.init(modelDocument) modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement modelDocument.parserLookupName = _parserLookupName modelDocument.parserLookupClass = _parserLookupClass modelDocument.xmlRootElement = mdlObj modelDocument.schemaLocationElements.add(mdlObj) modelDocument.documentEncoding = _encoding modelDocument._creationSoftwareComment = creationSoftwareComment modelXbrl.info("streamingExtensions:streaming", _("Stream processing this instance."), modelObject = modelDocument) else: self.currentMdlObj.append(mdlObj) self.currentMdlObj = mdlObj mdlObj._init() ns = mdlObj.namespaceURI ln = mdlObj.localName if (self.beforeInstanceStream and ( (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or (ns == XbrlConst.xbrli and ln in ("context", "unit")) or (ns not in (XbrlConst.link, XbrlConst.xbrli)))): self.beforeInstanceStream = False if _streamingExtensionsValidate: instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters(modelXbrl.prefixedNamespaces)) else: # need default dimensions ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) elif not self.beforeInstanceStream and self.beforeStartStreamingPlugin: for pluginMethod in pluginClassMethods("Streaming.Start"): pluginMethod(modelXbrl) self.beforeStartStreamingPlugin = False return mdlObj def end(self, tag): modelDocument = modelXbrl.modelDocument mdlObj = self.currentMdlObj parentMdlObj = mdlObj.getparent() self.currentMdlObj = parentMdlObj ns = mdlObj.namespaceURI ln = mdlObj.localName if ns == XbrlConst.xbrli: if ln == "context": if mdlObj.get("sticky"): del mdlObj.attrib["sticky"] XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) else: if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit: # drop before adding as dropped may have same id as added cntx = contextBuffer.pop(0) if _streamingValidateFactsPlugin: contextsToDrop.append(cntx) else: dropContext(modelXbrl, cntx) del parentMdlObj[parentMdlObj.index(cntx)] cntx = None #>>XmlValidate.validate(modelXbrl, mdlObj) #>>modelDocument.contextDiscover(mdlObj) if contextBufferLimit.is_finite(): contextBuffer.append(mdlObj) if _streamingExtensionsValidate: contextsToCheck = (mdlObj,) instValidator.checkContexts(contextsToCheck) if modelXbrl.hasXDT: instValidator.checkContextsDimensions(contextsToCheck) del contextsToCheck # dereference elif ln == "unit": if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit: # drop before adding as dropped may have same id as added unit = unitBuffer.pop(0) if _streamingValidateFactsPlugin: unitsToDrop.append(unit) else: dropUnit(modelXbrl, unit) del parentMdlObj[parentMdlObj.index(unit)] unit = None #>>XmlValidate.validate(modelXbrl, mdlObj) #>>modelDocument.unitDiscover(mdlObj) if unitBufferLimit.is_finite(): unitBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkUnits( (mdlObj,) ) elif ln == "xbrl": # end of document # check remaining batched facts if any if _streamingValidateFactsPlugin: # plugin attempts to process batch of all root facts not yet processed (not just current one) # finish any final batch of facts if len(modelXbrl.facts) > 0: factsToCheck = modelXbrl.facts.copy() factsHaveBeenProcessed = True # can block facts deletion if required data not yet available, such as numeric unit for DpmDB for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"): if not pluginMethod(modelXbrl, factsToCheck): factsHaveBeenProcessed = False if factsHaveBeenProcessed: for fact in factsToCheck: dropFact(modelXbrl, fact, modelXbrl.facts) del parentMdlObj[parentMdlObj.index(fact)] for cntx in contextsToDrop: dropContext(modelXbrl, cntx) del parentMdlObj[parentMdlObj.index(cntx)] for unit in unitsToDrop: dropUnit(modelXbrl, unit) del parentMdlObj[parentMdlObj.index(unit)] for footnoteLink in footnoteLinksToDrop: dropFootnoteLink(modelXbrl, footnoteLink) del parentMdlObj[parentMdlObj.index(footnoteLink)] fact = cntx = unit = footnoteLink = None del contextsToDrop[:] del unitsToDrop[:] del footnoteLinksToDrop[:] del factsToCheck # check remaining footnote refs for footnoteLink in footnoteBuffer: checkFootnoteHrefs(modelXbrl, footnoteLink) for pluginMethod in pluginClassMethods("Streaming.Finish"): pluginMethod(modelXbrl) elif ns == XbrlConst.link: if ln == "footnoteLink": XmlValidate.validate(modelXbrl, mdlObj) footnoteLinks = (mdlObj,) modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True) if footnoteBufferLimit.is_finite(): footnoteBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkLinks(footnoteLinks) if len(footnoteBuffer) > footnoteBufferLimit: # check that hrefObjects for locators were all satisfied # drop before addition as dropped may have same id as added footnoteLink = footnoteBuffer.pop(0) checkFootnoteHrefs(modelXbrl, footnoteLink) if _streamingValidateFactsPlugin: footnoteLinksToDrop.append(footnoteLink) else: dropFootnoteLink(modelXbrl, footnoteLink) del parentMdlObj[parentMdlObj.index(footnoteLink)] footnoteLink = None footnoteLinks = None elif ln in ("schemaRef", "linkbaseRef"): modelDocument.discoverHref(mdlObj) elif not modelXbrl.skipDTS: if ln in ("roleRef", "arcroleRef"): modelDocument.linkbaseDiscover((mdlObj,), inInstance=True) elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl: self.numRootFacts += 1 #>>XmlValidate.validate(modelXbrl, mdlObj) #>>modelDocument.factDiscover(mdlObj, modelXbrl.facts) if self.factsCheckVersion: self.factCheckFact(mdlObj) if _streamingExtensionsValidate or _streamingValidateFactsPlugin: factsToCheck = (mdlObj,) # validate current fact by itself if _streamingExtensionsValidate: instValidator.checkFacts(factsToCheck) if modelXbrl.hasXDT: instValidator.checkFactsDimensions(factsToCheck) if _streamingValidateFactsPlugin: # plugin attempts to process batch of all root facts not yet processed (not just current one) # use batches of 1000 facts if len(modelXbrl.facts) > 1000: factsToCheck = modelXbrl.facts.copy() factsHaveBeenProcessed = True # can block facts deletion if required data not yet available, such as numeric unit for DpmDB for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"): if not pluginMethod(modelXbrl, factsToCheck): factsHaveBeenProcessed = False if factsHaveBeenProcessed: for fact in factsToCheck: dropFact(modelXbrl, fact, modelXbrl.facts) del parentMdlObj[parentMdlObj.index(fact)] for cntx in contextsToDrop: dropContext(modelXbrl, cntx) del parentMdlObj[parentMdlObj.index(cntx)] for unit in unitsToDrop: dropUnit(modelXbrl, unit) del parentMdlObj[parentMdlObj.index(unit)] for footnoteLink in footnoteLinksToDrop: dropFootnoteLink(modelXbrl, footnoteLink) del parentMdlObj[parentMdlObj.index(footnoteLink)] fact = cntx = unit = footnoteLink = None del contextsToDrop[:] del unitsToDrop[:] del footnoteLinksToDrop[:] del factsToCheck # dereference fact or batch of facts else: dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed del parentMdlObj[parentMdlObj.index(mdlObj)] if self.numRootFacts % 1000 == 0: pass #modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, # 100.0 * self.numRootFacts / instInfoNumRootFacts), # minTimeToShow=20.0) gc.collect() sys.stdout.write ("\rAt fact {} of {} mem {}".format(self.numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed)) return mdlObj def data(self, data): self.currentMdlObj.text = data def comment(self, text): pass def pi(self, target, data): if target == "xbrl-facts-check": _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", data) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "version": self.factsCheckVersion = _matchGroups[1] elif _matchGroups[0] == "sum-of-fact-md5s": try: expectedMd5 = Md5Sum(_matchGroups[1]) if self.factsCheckMd5s != expectedMd5: modelXbrl.warning("streamingExtensions:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"), modelObject=modelXbrl, expectedMd5=expectedMd5, actualMd5Sum=self.factsCheckMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) except ValueError: modelXbrl.error("streamingExtensions:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) def close(self): del modelXbrl.makeelementParentModelObject return None def factCheckFact(self, fact): self.factsCheckMd5s += fact.md5sum for _tupleFact in fact.modelTupleFacts: self.factCheckFact(_tupleFact) _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget()) etree.parse(_file, parser=_parser, base_url=filepath) logSyntaxErrors(_parser) ''' # replace modelLoaderTarget with iterparse (as it now supports CustomElementClassLookup) streamingParserContext = etree.iterparse(_file, events=("start", "end"), huge_tree=True) from arelle.ModelObjectFactory import setParserElementClassLookup modelXbrl.isStreamingMode = True # must be set before setting element class lookup (_parser, _parserLookupName, _parserLookupClass) = setParserElementClassLookup(streamingParserContext, modelXbrl) foundInstance = False beforeInstanceStream = beforeStartStreamingPlugin = True numRootFacts = 0 factsCheckVersion = None def factCheckFact(fact): modelDocument._factsCheckMd5s += fact.md5sum for _tupleFact in fact.modelTupleFacts: factCheckFact(_tupleFact) for event, mdlObj in streamingParserContext: if event == "start": if mdlObj.tag == "{http://www.xbrl.org/2003/instance}xbrl": modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree()) modelXbrl.modelDocument = modelDocument # needed for incremental validation mdlObj.init(modelDocument) modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement modelDocument.parserLookupName = _parserLookupName modelDocument.parserLookupClass = _parserLookupClass modelDocument.xmlRootElement = mdlObj modelDocument.schemaLocationElements.add(mdlObj) modelDocument.documentEncoding = _encoding modelDocument._creationSoftwareComment = precedingComment( mdlObj) modelDocument._factsCheckMd5s = Md5Sum() modelXbrl.info("streamingExtensions:streaming", _("Stream processing this instance."), modelObject=modelDocument) elif mdlObj.getparent() is not None: mdlObj._init() # requires discovery as part of start elements if mdlObj.getparent( ).tag == "{http://www.xbrl.org/2003/instance}xbrl": if not foundInstance: foundInstance = True pi = precedingProcessingInstruction( mdlObj, "xbrl-facts-check") if pi is not None: factsCheckVersion = pi.attrib.get("version", None) elif not foundInstance: break ns = mdlObj.qname.namespaceURI ln = mdlObj.qname.localName if beforeInstanceStream: if ((ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or (ns == XbrlConst.xbrli and ln in ("context", "unit")) or (ns not in (XbrlConst.link, XbrlConst.xbrli))): beforeInstanceStream = False if _streamingExtensionsValidate: instValidator.validate( modelXbrl, modelXbrl.modelManager.formulaOptions. typedParameters(modelXbrl.prefixedNamespaces)) else: # need default dimensions ValidateXbrlDimensions.loadDimensionDefaults( modelXbrl) elif not beforeInstanceStream and beforeStartStreamingPlugin: for pluginMethod in pluginClassMethods("Streaming.Start"): pluginMethod(modelXbrl) beforeStartStreamingPlugin = False elif event == "end": parentMdlObj = mdlObj.getparent() ns = mdlObj.namespaceURI ln = mdlObj.localName if ns == XbrlConst.xbrli: if ln == "context": if mdlObj.get("sticky"): del mdlObj.attrib["sticky"] XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) else: if len(contextBuffer) >= contextBufferLimit: # drop before adding as dropped may have same id as added cntx = contextBuffer.pop(0) if _streamingFactsPlugin or _streamingValidateFactsPlugin: contextsToDrop.append(cntx) else: dropContext(modelXbrl, cntx) #>>del parentMdlObj[parentMdlObj.index(cntx)] cntx = None XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) if contextBufferLimit.is_finite(): contextBuffer.append(mdlObj) if _streamingExtensionsValidate: contextsToCheck = (mdlObj, ) instValidator.checkContexts(contextsToCheck) if modelXbrl.hasXDT: instValidator.checkContextsDimensions( contextsToCheck) del contextsToCheck # dereference elif ln == "unit": if len(unitBuffer) >= unitBufferLimit: # drop before additing as dropped may have same id as added unit = unitBuffer.pop(0) if _streamingFactsPlugin or _streamingValidateFactsPlugin: unitsToDrop.append(unit) else: dropUnit(modelXbrl, unit) #>>del parentMdlObj[parentMdlObj.index(unit)] unit = None XmlValidate.validate(modelXbrl, mdlObj) modelDocument.unitDiscover(mdlObj) if unitBufferLimit.is_finite(): unitBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkUnits((mdlObj, )) elif ln == "xbrl": # end of document # check remaining batched facts if any if _streamingFactsPlugin or _streamingValidateFactsPlugin: # plugin attempts to process batch of all root facts not yet processed (not just current one) # finish any final batch of facts if len(modelXbrl.facts) > 0: factsToCheck = modelXbrl.facts.copy() # can block facts deletion if required data not yet available, such as numeric unit for DpmDB if _streamingValidateFactsPlugin: for pluginMethod in pluginClassMethods( "Streaming.ValidateFacts"): pluginMethod(instValidator, factsToCheck) if _streamingFactsPlugin: for pluginMethod in pluginClassMethods( "Streaming.Facts"): pluginMethod(modelXbrl, factsToCheck) for fact in factsToCheck: dropFact(modelXbrl, fact, modelXbrl.facts) #>>del parentMdlObj[parentMdlObj.index(fact)] for cntx in contextsToDrop: dropContext(modelXbrl, cntx) #>>del parentMdlObj[parentMdlObj.index(cntx)] for unit in unitsToDrop: dropUnit(modelXbrl, unit) #>>del parentMdlObj[parentMdlObj.index(unit)] for footnoteLink in footnoteLinksToDrop: dropFootnoteLink(modelXbrl, footnoteLink) #>>del parentMdlObj[parentMdlObj.index(footnoteLink)] fact = cntx = unit = footnoteLink = None del contextsToDrop[:] del unitsToDrop[:] del footnoteLinksToDrop[:] del factsToCheck # check remaining footnote refs for footnoteLink in footnoteBuffer: checkFootnoteHrefs(modelXbrl, footnoteLink) pi = childProcessingInstruction(mdlObj, "xbrl-facts-check", reversed=True) if pi is not None: # attrib is in .text, not attrib, no idea why!!! _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "sum-of-fact-md5s": try: expectedMd5 = Md5Sum(_matchGroups[1]) if modelDocument._factsCheckMd5s != expectedMd5: modelXbrl.warning( "streamingExtensions:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s" ), modelObject=modelXbrl, expectedMd5=expectedMd5, actualMd5Sum=modelDocument. _factsCheckMd5s) else: modelXbrl.info( "info", _("Successful XBRL facts sum of md5s." ), modelObject=modelXbrl) except ValueError: modelXbrl.error( "streamingExtensions:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s" ), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) if _streamingValidateFactsPlugin: for pluginMethod in pluginClassMethods( "Streaming.ValidateFinish"): pluginMethod(instValidator) if _streamingFactsPlugin: for pluginMethod in pluginClassMethods( "Streaming.Finish"): pluginMethod(modelXbrl) elif ns == XbrlConst.link: if ln in ("schemaRef", "linkbaseRef"): modelDocument.discoverHref( mdlObj, urlRewritePluginClass= "ModelDocument.InstanceSchemaRefRewriter") elif ln in ("roleRef", "arcroleRef"): modelDocument.linkbaseDiscover((mdlObj, ), inInstance=True) elif ln == "footnoteLink": XmlValidate.validate(modelXbrl, mdlObj) footnoteLinks = (mdlObj, ) modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True) if footnoteBufferLimit.is_finite(): footnoteBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkLinks(footnoteLinks) if len(footnoteBuffer) > footnoteBufferLimit: # check that hrefObjects for locators were all satisfied # drop before addition as dropped may have same id as added footnoteLink = footnoteBuffer.pop(0) checkFootnoteHrefs(modelXbrl, footnoteLink) if _streamingValidateFactsPlugin: footnoteLinksToDrop.append(footnoteLink) else: dropFootnoteLink(modelXbrl, footnoteLink) #>>del parentMdlObj[parentMdlObj.index(footnoteLink)] footnoteLink = None footnoteLinks = None elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl and isinstance( mdlObj, ModelFact): numRootFacts += 1 XmlValidate.validate(modelXbrl, mdlObj) modelDocument.factDiscover(mdlObj, modelXbrl.facts) if factsCheckVersion: factCheckFact(mdlObj) if _streamingExtensionsValidate or _streamingFactsPlugin or _streamingValidateFactsPlugin: factsToCheck = (mdlObj, ) # validate current fact by itself if _streamingExtensionsValidate: instValidator.checkFacts(factsToCheck) if modelXbrl.hasXDT: instValidator.checkFactsDimensions(factsToCheck) if _streamingFactsPlugin or _streamingValidateFactsPlugin: # plugin attempts to process batch of all root facts not yet processed (not just current one) # use batches of 1000 facts if len(modelXbrl.facts) > 1000: factsToCheck = modelXbrl.facts.copy() # can block facts deletion if required data not yet available, such as numeric unit for DpmDB if _streamingValidateFactsPlugin: for pluginMethod in pluginClassMethods( "Streaming.ValidateFacts"): pluginMethod(instValidator, factsToCheck) if _streamingFactsPlugin: for pluginMethod in pluginClassMethods( "Streaming.Facts"): pluginMethod(modelXbrl, factsToCheck) for fact in factsToCheck: dropFact(modelXbrl, fact, modelXbrl.facts) #>>del parentMdlObj[parentMdlObj.index(fact)] for cntx in contextsToDrop: dropContext(modelXbrl, cntx) #>>del parentMdlObj[parentMdlObj.index(cntx)] for unit in unitsToDrop: dropUnit(modelXbrl, unit) #>>del parentMdlObj[parentMdlObj.index(unit)] for footnoteLink in footnoteLinksToDrop: dropFootnoteLink(modelXbrl, footnoteLink) #>>del parentMdlObj[parentMdlObj.index(footnoteLink)] fact = cntx = unit = footnoteLink = None del contextsToDrop[:] del unitsToDrop[:] del footnoteLinksToDrop[:] del factsToCheck # dereference fact or batch of facts else: dropFact( modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed #>>del parentMdlObj[parentMdlObj.index(mdlObj)] if numRootFacts % 1000 == 0: pass #modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, # 100.0 * self.numRootFacts / instInfoNumRootFacts), # minTimeToShow=20.0) #gc.collect() #sys.stdout.write ("\rAt fact {} of {} mem {}".format(numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed)) if mdlObj is not None: mdlObj.clear() del _parser, _parserLookupName, _parserLookupClass if _streamingExtensionsValidate and validator is not None: _file.close() del instValidator validator.close() # track that modelXbrl has been validated by this streaming extension modelXbrl._streamingExtensionValidated = True modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt) return modelXbrl.modelDocument
def checkFactDimensions(self, facts): # check fact dimensions in document order for f in facts: if f.concept.isItem and f.context is not None: ValidateXbrlDimensions.checkFact(self, f) elif f.modelTupleFacts: self.checkFactDimensions(f.modelTupleFacts)
def createTargetInstance(modelXbrl, targetUrl, targetDocumentSchemaRefs, filingFiles, baseXmlLang=None, defaultXmlLang=None): def addLocallyReferencedFile(elt, filingFiles): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src") and not isHttpUrl( attrValue) and not os.path.isabs(attrValue): attrValue = attrValue.partition('#')[0] # remove anchor if attrValue: # ignore anchor references to base document attrValue = os.path.normpath( attrValue ) # change url path separators to host separators file = os.path.join(sourceDir, attrValue) if modelXbrl.fileSource.isInArchive( file, checkExistence=True) or os.path.exists(file): filingFiles.add(file) targetInstance = ModelXbrl.create( modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True, discover=False) # don't attempt to load DTS ixTargetRootElt = modelXbrl.ixTargetRootElements[getattr( modelXbrl, "ixdsTarget", None)] langIsSet = False # copy ix resources target root attributes for attrName, attrValue in ixTargetRootElt.items(): if attrName != "target": # ix:references target is not mapped to xbrli:xbrl targetInstance.modelDocument.xmlRootElement.set( attrName, attrValue) if attrName == "{http://www.w3.org/XML/1998/namespace}lang": langIsSet = True defaultXmlLang = attrValue if attrName.startswith("{"): ns, _sep, ln = attrName[1:].rpartition("}") if ns: prefix = xmlnsprefix(ixTargetRootElt, ns) if prefix not in (None, "xml"): setXmlns(targetInstance.modelDocument, prefix, ns) if not langIsSet and baseXmlLang: targetInstance.modelDocument.xmlRootElement.set( "{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang) if defaultXmlLang is None: defaultXmlLang = baseXmlLang # allows facts/footnotes to override baseXmlLang ValidateXbrlDimensions.loadDimensionDefaults( targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in sorted(modelXbrl.contexts.values(), key=lambda c: c.objectIndex ): # contexts may come from multiple IXDS files ignore = targetInstance.createContext( context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in sorted(modelXbrl.units.values(), key=lambda u: u.objectIndex ): # units may come from multiple IXDS files measures = unit.measures ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue for attrName, attrValue in fact.items(): if attrName.startswith("{"): attrs[qname( attrName, fact.nsmap )] = attrValue # using qname allows setting prefix in extracted instance newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) # if fact.isFraction, create numerator and denominator newFactForOldObjId[fact.objectIndex] = newFact if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references so that referenced files are included in the zip. for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML("<body>\n{0}\n</body>\n".format( xmltext)).iter(): addLocallyReferencedFile(elt, filingFiles) except (XMLSyntaxError, UnicodeDecodeError): pass # TODO: Why ignore UnicodeDecodeError? elif fact.isTuple: attrs = {} if fact.id: attrs["id"] = fact.id if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" for attrName, attrValue in fact.items(): if attrName.startswith("{"): attrs[qname(attrName, fact.nsmap)] = attrValue newTuple = targetInstance.createFact(fact.qname, attributes=attrs, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) modelXbrl.modelManager.showStatus( _("Creating and validating footnotes and relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) footnoteIdCount = {} for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any( lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format( attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) xmlLang = linkChild.xmlLang if xmlLang is not None and xmlLang != defaultXmlLang: # default newChild.set( "{http://www.w3.org/XML/1998/namespace}lang", xmlLang) copyIxFootnoteHtml( linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): addLocallyReferencedFile(elt, filingFiles) return targetInstance
def validate(logMessage, sphinxContext): modelXbrl = sphinxContext.modelXbrl hasDTS = modelXbrl is not None if hasDTS: # if no formulas loaded, set if not hasattr(modelXbrl, "modelFormulaEqualityDefinitions"): modelXbrl.modelFormulaEqualityDefinitions = {} import logging initialErrorCount = modelXbrl.logCount.get(logging.getLevelName('ERROR'), 0) # must also have default dimensions loaded from arelle import ValidateXbrlDimensions ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) sphinxContext.ruleBasePreconditionNodes = [] sphinxContext.preconditionNodes = {} # accumulate definitions for prog in sphinxContext.sphinxProgs: for node in prog: if isinstance(node, astRuleBasePrecondition): sphinxContext.ruleBasePreconditionNodes.append(node) elif isinstance(node, astPreconditionDeclaration): sphinxContext.preconditionNodes[node.name] = node elif isinstance(node, astFunctionDeclaration): sphinxContext.functions[node.name] = node elif isinstance(node, astConstant): sphinxContext.constants[node.constantName] = node node.value = None # compute dynamically on first reference if node.tagName: sphinxContext.taggedConstants[node.tagName] = node # check references def checkNodes(nodes, inMacro=False): if not nodes: return for node in nodes: if node is None: continue elif isinstance(node, (list,set)): checkNodes(node, inMacro) elif isinstance(node, astPreconditionReference): for name in node.names: if name not in sphinxContext.preconditionNodes: logMessage("ERROR", "sphinxCompiler:preconditionReferenceI", _("Precondition reference is not defined %(name)s"), sourceFileLine=node.sourceFileLine, name=name) elif isinstance(node, (astFormulaRule, astReportRule, astValidationRule)): checkNodes((node.precondition, node.severity, node.variableAssignments, node.expr, node.message), inMacro) sphinxContext.rules.append(node) if node.severity: severity = node.severity if isinstance(severity, astFunctionReference): severity = severity.name if (severity not in ("error", "warning", "info") and (isinstance(node, astFormulaRule) and severity not in sphinxContext.functions)): logMessage("ERROR", "sphinxCompiler:ruleSeverity", _("Rule %(name)s severity is not recognized: %(severity)s"), sourceFileLine=node.sourceFileLine, name=node.name, severity=node.severity) if isinstance(node, astFormulaRule) and not hasFormulaOp(node): logMessage("ERROR", "sphinxCompiler:formulaSyntax", _("Formula %(name)s missing \":=\" operation"), sourceFileLine=node.sourceFileLine, name=node.name) elif isinstance(node, astHyperspaceExpression) and hasDTS: # check axes for axis in node.axes: if isinstance(axis.aspect, QName): concept = modelXbrl.qnameConcepts.get(axis) if concept is None or not concept.isDimensionItem: logMessage("ERROR", "sphinxCompiler:axisNotDimension", _("Axis is not a dimension in the DTS %(qname)s"), sourceFileLine=node.sourceFileLine, qname=axis) elif axis not in sphinxContext.dimensionIsExplicit: sphinxContext.dimensionIsExplicit[axis] = concept.isExplicitDimension elif isinstance(axis.aspect, astNode): if not inMacro: logMessage("ERROR", "sphinxCompiler:axisDisallowed", _("Hypercube axis aspect not static %(aspect)s"), sourceFileLine=node.sourceFileLine, aspect=axis.aspect) elif (axis.aspect not in {"unit", "segment", "scenario"} and isinstance(axis.restriction, (list, tuple))): for restrictionValue in axis.restriction: if isinstance(restrictionValue, QName) and not restrictionValue in modelXbrl.qnameConcepts: logMessage("ERROR", "sphinxCompiler:axisNotDimension", _("Hypercube value not in the DTS %(qname)s"), sourceFileLine=node.sourceFileLine, qname=restrictionValue) checkNodes((axis.whereExpr,), inMacro) elif isinstance(node, astWith): node.axes = {} def checkWithAxes(withNode): if isinstance(withNode, astHyperspaceExpression): checkNodes((withNode,), inMacro) node.axes.update(withNode.axes) else: logMessage("ERROR", "sphinxCompiler:withRestrictionError", _("With restriction is not a single hyperspace expression"), sourceFileLine=withNode.sourceFileLine) checkWithAxes(node.restrictionExpr) checkNodes((node.variableAssignments, node.bodyExpr,), inMacro) elif isinstance(node, astNode): nestedMacro = inMacro or (isinstance(node, astFunctionDeclaration) and node.functionType == "macro") checkNodes([expr for expr in node.__dict__.values() if isinstance(expr, (astNode, list, set))], nestedMacro) for prog in sphinxContext.sphinxProgs: checkNodes(prog) if len(sphinxContext.ruleBasePreconditionNodes) > 1: logMessage("ERROR", "sphinxCompiler:multipleRuleBaseDeclarations", _("Multiple rule-base declarations %(preconditions)s"), sourceFileLines=[node.sourceFileLine for node in sphinxContext.ruleBasePreconditionNodes], preconditions=", ".join(str(r) for r in sphinxContext.ruleBasePreconditionNodes)) if hasDTS: # if no errors in checking sphinx if initialErrorCount == modelXbrl.logCount.get(logging.getLevelName('ERROR'), 0): from .SphinxEvaluator import evaluateRuleBase evaluateRuleBase(sphinxContext)
def init(modelXbrl): # setup modelXbrl for rendering evaluation # dimension defaults required in advance of validation from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) hasXbrlTables = False # validate table linkbase dimensions for baseSetKey in modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(arcrole): ValidateFormula.checkBaseSet(modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname)) if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011): hasXbrlTables = True # provide context for view if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE: instance = None # use instance of the entry pont else: # need dummy instance instance = ModelDocument.create(modelXbrl, ModelDocument.Type.INSTANCE, "dummy.xml", # fake URI and fake schemaRef ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd",)) if hasXbrlTables: # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance) modelXbrl.profileStat(None) # setup fresh parameters from formula options modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters() # validate parameters and custom function signatures ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables")) # deprecated as of 2013-05-17 # check and extract message expressions into compilable programs for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301, XbrlConst.tableDefinitionNodeSelectionMessage201301, XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011): for msgRel in modelXbrl.relationshipSet(msgArcrole).modelRelationships: ValidateFormula.checkMessageExpressions(modelXbrl, msgRel.toModelObject) # compile and validate tables for modelTable in modelXbrl.modelRenderingTables: modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining modelTable.compile() hasNsWithAspectModel = modelTable.namespaceURI in (XbrlConst.euRend, XbrlConst.table2011, XbrlConst.table201301, XbrlConst.table201305) # check aspectModel (attribute removed 2013-06, now always dimensional) if modelTable.aspectModel not in ("non-dimensional", "dimensional") and hasNsWithAspectModel: modelXbrl.error("xbrlte:unknownAspectModel", _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel) else: modelTable.priorAspectAxisDisposition = {} # check ordinate aspects against aspectModel oppositeAspectModel = (_DICT_SET({'dimensional','non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop() if hasNsWithAspectModel: uncoverableAspects = aspectModels[oppositeAspectModel] - aspectModels[modelTable.aspectModel] else: uncoverableAspects = () aspectsCovered = set() for tblAxisRel in modelXbrl.relationshipSet((XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301,XbrlConst.tableAxis2011)).fromModelObject(modelTable): breakdownAspectsCovered = set() hasCoveredAspect = checkBreakdownDefinitionNode(modelXbrl, modelTable, tblAxisRel, tblAxisRel.axisDisposition, uncoverableAspects, breakdownAspectsCovered) ''' removed 2013-10 if not hasCoveredAspect: definitionNode = tblAxisRel.toModelObject modelXbrl.error("xbrlte:breakdownDefinesNoAspects", _("Breakdown %(xlinkLabel)s has no participating aspects"), modelObject=(modelTable,definitionNode), xlinkLabel=definitionNode.xlinkLabel, axis=definitionNode.localName) ''' aspectsCovered |= breakdownAspectsCovered checkBreakdownLeafNodeAspects(modelXbrl, modelTable, tblAxisRel, set(), breakdownAspectsCovered) if Aspect.CONCEPT not in aspectsCovered and not hasNsWithAspectModel: modelXbrl.error("xbrlte:tableMissingConceptAspect", _("Table %(xlinkLabel)s does not include the concept aspect as one of its participating aspects"), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel) del modelTable.priorAspectAxisDisposition # check for table-parameter name clash parameterNames = {} for tblParamRel in modelXbrl.relationshipSet((XbrlConst.tableParameter, XbrlConst.tableParameterMMDD)).fromModelObject(modelTable): parameterName = tblParamRel.variableQname if parameterName in parameterNames: modelXbrl.error("xbrlte:tableParameterNameClash ", _("Table %(xlinkLabel)s has parameter name clash for variable %(name)s"), modelObject=(modelTable,tblParamRel,parameterNames[parameterName]), xlinkLabel=modelTable.xlinkLabel, name=parameterName) else: parameterNames[parameterName] = tblParamRel modelXbrl.profileStat(_("compileTables"))
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in modelXbrl.contexts.values(): newCntx = targetInstance.createContext(context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) newFactForOldObjId[fact.objectIndex] = newFact if filingFiles and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)): if elt.tag in ("a", "img") and not isHttpUrl(attrValue) and not os.path.isabs(attrvalue): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src"): filingFiles.add(attrValue) except (XMLSyntaxError, UnicodeDecodeError): pass elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) # footnote links footnoteIdCount = {} modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format(attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) copyIxFootnoteHtml(linkChild, newChild, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrvalue): filingFiles.add(attrValue) targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def init(modelXbrl): # setup modelXbrl for rendering evaluation # dimension defaults required in advance of validation from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) hasXbrlTables = False # validate table linkbase dimensions for baseSetKey in modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole( arcrole): ValidateFormula.checkBaseSet( modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole, ELR, linkqname, arcqname)) if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011): hasXbrlTables = True # provide context for view if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE: instance = None # use instance of the entry pont else: # need dummy instance instance = ModelDocument.create( modelXbrl, ModelDocument.Type.INSTANCE, "dummy.xml", # fake URI and fake schemaRef ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd", )) if hasXbrlTables: # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance) modelXbrl.profileStat(None) # setup fresh parameters from formula options modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters( ) # validate parameters and custom function signatures ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables")) # deprecated as of 2013-05-17 # check and extract message expressions into compilable programs for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301, XbrlConst.tableDefinitionNodeSelectionMessage201301, XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011): for msgRel in modelXbrl.relationshipSet( msgArcrole).modelRelationships: ValidateFormula.checkMessageExpressions( modelXbrl, msgRel.toModelObject) # compile and validate tables for modelTable in modelXbrl.modelRenderingTables: modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining modelTable.compile() hasNsWithAspectModel = modelTable.namespaceURI in ( XbrlConst.euRend, XbrlConst.table2011, XbrlConst.table201301, XbrlConst.table201305) # check aspectModel (attribute removed 2013-06, now always dimensional) if modelTable.aspectModel not in ( "non-dimensional", "dimensional") and hasNsWithAspectModel: modelXbrl.error( "xbrlte:unknownAspectModel", _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized" ), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel) else: modelTable.priorAspectAxisDisposition = {} # check ordinate aspects against aspectModel oppositeAspectModel = ( _DICT_SET({'dimensional', 'non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop() if hasNsWithAspectModel: uncoverableAspects = aspectModels[ oppositeAspectModel] - aspectModels[ modelTable.aspectModel] else: uncoverableAspects = () aspectsCovered = set() for tblAxisRel in modelXbrl.relationshipSet( (XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011)).fromModelObject(modelTable): breakdownAspectsCovered = set() hasCoveredAspect = checkBreakdownDefinitionNode( modelXbrl, modelTable, tblAxisRel, tblAxisRel.axisDisposition, uncoverableAspects, breakdownAspectsCovered) ''' removed 2013-10 if not hasCoveredAspect: definitionNode = tblAxisRel.toModelObject modelXbrl.error("xbrlte:breakdownDefinesNoAspects", _("Breakdown %(xlinkLabel)s has no participating aspects"), modelObject=(modelTable,definitionNode), xlinkLabel=definitionNode.xlinkLabel, axis=definitionNode.localName) ''' aspectsCovered |= breakdownAspectsCovered checkBreakdownLeafNodeAspects(modelXbrl, modelTable, tblAxisRel, set(), breakdownAspectsCovered) if Aspect.CONCEPT not in aspectsCovered and not hasNsWithAspectModel: modelXbrl.error( "xbrlte:tableMissingConceptAspect", _("Table %(xlinkLabel)s does not include the concept aspect as one of its participating aspects" ), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel) del modelTable.priorAspectAxisDisposition # check for table-parameter name clash parameterNames = {} for tblParamRel in modelXbrl.relationshipSet( (XbrlConst.tableParameter, XbrlConst.tableParameterMMDD )).fromModelObject(modelTable): parameterName = tblParamRel.variableQname if parameterName in parameterNames: modelXbrl.error( "xbrlte:tableParameterNameClash ", _("Table %(xlinkLabel)s has parameter name clash for variable %(name)s" ), modelObject=(modelTable, tblParamRel, parameterNames[parameterName]), xlinkLabel=modelTable.xlinkLabel, name=parameterName) else: parameterNames[parameterName] = tblParamRel modelXbrl.profileStat(_("compileTables"))
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( targetDocumentFilename, modelXbrl.modelDocument.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus( _("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults( targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in modelXbrl.contexts.values(): newCntx = targetInstance.createContext( context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in modelXbrl.facts: if fact.isItem: attrs = {"contextRef": fact.contextID} if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) newFactForOldObjId[fact.objectIndex] = newFact elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) # footnote links modelXbrl.modelManager.showStatus( _("Creating and validating footnotes & relationships")) for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: newLink = XmlUtil.addChild( targetInstance.modelDocument.xmlRootElement, linkqname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: if isinstance( linkChild, LocPrototype ) and "{http://www.w3.org/1999/xlink}href" not in linkChild.attributes: linkChild.attributes["{http://www.w3.org/1999/xlink}href"] = \ "#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) XmlUtil.addChild(newLink, linkChild.qname, attributes=linkChild.attributes, text=linkChild.textValue) targetInstance.saveInstance(overrideFilepath=targetUrl) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def init(modelXbrl): # setup modelXbrl for rendering evaluation # dimension defaults required in advance of validation from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) hasXbrlTables = False # validate table linkbase dimensions for baseSetKey in modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole( arcrole): ValidateFormula.checkBaseSet( modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole, ELR, linkqname, arcqname)) if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011): hasXbrlTables = True # provide context for view if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE: instance = None # use instance of the entry pont else: # need dummy instance instance = ModelDocument.create( modelXbrl, ModelDocument.Type.INSTANCE, "dummy.xml", # fake URI and fake schemaRef ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd", )) if hasXbrlTables: # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance) modelXbrl.profileStat(None) # setup fresh parameters from formula optoins modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters( ) # validate parameters and custom function signatures ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables")) # deprecated as of 2013-05-17 # check and extract message expressions into compilable programs for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301, XbrlConst.tableDefinitionNodeSelectionMessage201301, XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011): for msgRel in modelXbrl.relationshipSet( msgArcrole).modelRelationships: ValidateFormula.checkMessageExpressions( modelXbrl, msgRel.toModelObject) # compile and validate tables for modelTable in modelXbrl.modelRenderingTables: modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining modelTable.compile() # check aspectModel if modelTable.aspectModel not in ("non-dimensional", "dimensional"): modelXbrl.error( "xbrlte:unknownAspectModel", _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized" ), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel) else: modelTable.priorAspectAxisDisposition = {} # check ordinate aspects against aspectModel oppositeAspectModel = ( _DICT_SET({'dimensional', 'non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop() uncoverableAspects = aspectModels[ oppositeAspectModel] - aspectModels[modelTable.aspectModel] for tblAxisRel in modelXbrl.relationshipSet( (XbrlConst.tableBreakdown, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011)).fromModelObject(modelTable): checkDefinitionNodeAspectModel(modelXbrl, modelTable, tblAxisRel, uncoverableAspects) del modelTable.priorAspectAxisDisposition modelXbrl.profileStat(_("compileTables"))
def validate(self, modelXbrl, parameters=None): self.parameters = parameters self.NCnamePattern = re.compile( "^[_A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]" r"[_\-\." "\xB7A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040]*$" ) self.precisionPattern = re.compile("^([0-9]+|INF)$") self.decimalsPattern = re.compile("^(-?[0-9]+|INF)$") self.isoCurrencyPattern = re.compile(r"^[A-Z]{3}$") self.modelXbrl = modelXbrl self.validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem self.disclosureSystem = modelXbrl.modelManager.disclosureSystem self.validateEFM = self.validateDisclosureSystem and self.disclosureSystem.EFM self.validateGFM = self.validateDisclosureSystem and self.disclosureSystem.GFM self.validateEFMorGFM = self.validateDisclosureSystem and self.disclosureSystem.EFMorGFM self.validateHMRC = self.validateDisclosureSystem and self.disclosureSystem.HMRC self.validateSBRNL = self.validateDisclosureSystem and self.disclosureSystem.SBRNL self.validateXmlLang = self.validateDisclosureSystem and self.disclosureSystem.xmlLangPattern self.validateCalcLB = modelXbrl.modelManager.validateCalcLB self.validateInferDecimals = modelXbrl.modelManager.validateInferDecimals # xlink validation modelXbrl.modelManager.showStatus(_("validating links")) modelLinks = set() self.remoteResourceLocElements = set() self.genericArcArcroles = set() for baseSetExtLinks in modelXbrl.baseSets.values(): for baseSetExtLink in baseSetExtLinks: modelLinks.add( baseSetExtLink) # ext links are unique (no dups) for modelLink in modelLinks: fromToArcs = {} locLabels = {} resourceLabels = {} resourceArcTos = [] for arcElt in modelLink.element.childNodes: if arcElt.nodeType == 1: xlinkType = arcElt.getAttributeNS(XbrlConst.xlink, "type") # locator must have an href if xlinkType == "locator": if not arcElt.hasAttributeNS(XbrlConst.xlink, "href"): modelXbrl.error( _("Linkbase {0} extended link {1} locator {2} missing href" ).format( modelLink.modelDocument.basename, modelLink.role, arcElt.hasAttributeNS( XbrlConst.xlink, "label")), "err", "xlink:locatorHref") locLabels[arcElt.getAttributeNS( XbrlConst.xlink, "label")] = arcElt elif xlinkType == "resource": resourceLabels[arcElt.getAttributeNS( XbrlConst.xlink, "label")] = arcElt # can be no duplicated arcs between same from and to elif xlinkType == "arc": fromLabel = arcElt.getAttributeNS( XbrlConst.xlink, "from") toLabel = arcElt.getAttributeNS(XbrlConst.xlink, "to") fromTo = (fromLabel, toLabel) if fromTo in fromToArcs: modelXbrl.error( _("Linkbase {0} extended link {1} duplicate arcs from {2} to {3}" ).format(modelLink.modelDocument.basename, modelLink.role, fromLabel, toLabel), "err", "xlink:dupArcs") else: fromToArcs[fromTo] = arcElt if arcElt.namespaceURI == XbrlConst.link: if arcElt.localName in arcNamesTo21Resource: #("labelArc","referenceArc"): resourceArcTos.append( (toLabel, arcElt.getAttribute("use"))) elif self.isGenericArc(arcElt): arcrole = arcElt.getAttributeNS( XbrlConst.xlink, "arcrole") self.genericArcArcroles.add(arcrole) if arcrole in (XbrlConst.elementLabel, XbrlConst.elementReference): resourceArcTos.append((toLabel, arcrole)) # values of type (not needed for validating parsers) if xlinkType not in xlinkTypeValues: # ("", "simple", "extended", "locator", "arc", "resource", "title", "none"): modelXbrl.error( _("Linkbase {0} extended link {1} type {2} invalid" ).format(modelLink.modelDocument.basename, modelLink.role, xlinkType), "err", "xlink:type") # values of actuate (not needed for validating parsers) xlinkActuate = arcElt.getAttributeNS( XbrlConst.xlink, "actuate") if xlinkActuate not in xlinkActuateValues: # ("", "onLoad", "onRequest", "other", "none"): modelXbrl.error( _("Linkbase {0} extended link {1} actuate {2} invalid" ).format(modelLink.modelDocument.basename, modelLink.role, xlinkActuate), "err", "xlink:actuate") # values of show (not needed for validating parsers) xlinkShow = arcElt.getAttributeNS(XbrlConst.xlink, "show") if xlinkShow not in xlinkShowValues: # ("", "new", "replace", "embed", "other", "none"): modelXbrl.error( _("Linkbase {0} extended link {1} show {2} invalid" ).format(modelLink.modelDocument.basename, modelLink.role, xlinkShow), "err", "xlink:show") # values of label, from, to (not needed for validating parsers) for name in xlinkLabelAttributes: # ("label", "from", "to"): value = arcElt.getAttributeNS(XbrlConst.xlink, name) if value != "" and not self.NCnamePattern.match(value): modelXbrl.error( _("Linkbase {0} extended link {1} element {2} {3} '{4}' not an NCname" ).format(modelLink.modelDocument.basename, modelLink.role, arcElt.tagName, name, value), "err", "xlink:{0}".format(name)) # check from, to of arcs have a resource or loc for fromTo, arcElt in fromToArcs.items(): fromLabel, toLabel in fromTo for name, value, sect in (("from", fromLabel, "3.5.3.9.2"), ("to", toLabel, "3.5.3.9.3")): if value not in locLabels and value not in resourceLabels: modelXbrl.error( _("Arc in linkbase {0} extended link {1} from {2} to {3} attribute \"{4}\" has no matching loc or resource label" ).format(modelLink.modelDocument.basename, modelLink.role, fromLabel, toLabel, name), "err", "xbrl.{0}:arcResource".format(sect)) if arcElt.localName == "footnoteArc" and arcElt.namespaceURI == XbrlConst.link and \ arcElt.getAttributeNS(XbrlConst.xlink,"arcrole") == XbrlConst.factFootnote: if fromLabel not in locLabels: modelXbrl.error( _("FootnoteArc in {0} extended link {1} from {2} to {3} \"from\" is not a loc" ).format(modelLink.modelDocument.basename, modelLink.role, fromLabel, toLabel), "err", "xbrl.4.11.1.3.1:factFootnoteArcFrom") if toLabel not in resourceLabels or qname( resourceLabels[toLabel] ) != XbrlConst.qnLinkFootnote: modelXbrl.error( _("FootnoteArc in {0} extended link {1} from {2} to {3} \"to\" is not a footnote resource" ).format(modelLink.modelDocument.basename, modelLink.role, fromLabel, toLabel), "err", "xbrl.4.11.1.3.1:factFootnoteArcTo") # check unprohibited label arcs to remote locs for resourceArcTo in resourceArcTos: resourceArcToLabel, resourceArcUse = resourceArcTo if resourceArcToLabel in locLabels: toLabel = locLabels[resourceArcToLabel] if resourceArcUse == "prohibited": self.remoteResourceLocElements.add(toLabel) else: modelXbrl.error( _("Unprohibited labelArc in linkbase {0} extended link {1} has illegal remote resource loc labeled {2} href {3}" ).format( modelLink.modelDocument.basename, modelLink.role, resourceArcToLabel, toLabel.getAttributeNS( XbrlConst.xlink, "href")), "err", "xbrl.5.2.2.3:labelArcRemoteResource") elif resourceArcToLabel in resourceLabels: toResource = resourceLabels[resourceArcToLabel] if resourceArcUse == XbrlConst.elementLabel: if not self.isGenericLabel(toResource): modelXbrl.error( _("Generic label arc in linkbase {0} extended link {1} to {2} must target a generic label" ).format(modelLink.modelDocument.basename, modelLink.role, resourceArcToLabel), "err", "xbrlle.2.1.1:genericLabelTarget") elif resourceArcUse == XbrlConst.elementReference: if not self.isGenericReference(toResource): modelXbrl.error( _("Generic reference arc in linkbase {0} extended link {1} to {2} must target a generic reference" ).format(modelLink.modelDocument.basename, modelLink.role, resourceArcToLabel), "err", "xbrlre.2.1.1:genericReferenceTarget") self.dimensionDefaults = {} modelXbrl.qnameDimensionDefaults = {} modelXbrl.qnameDimensionContextElement = {} # check base set cycles, dimensions modelXbrl.modelManager.showStatus(_("validating relationship sets")) for baseSetKey in modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if arcrole.startswith("XBRL-") or ELR is None or \ linkqname is None or arcqname is None: continue elif arcrole in XbrlConst.standardArcroleCyclesAllowed: cyclesAllowed, specSect = XbrlConst.standardArcroleCyclesAllowed[ arcrole] elif arcrole in self.modelXbrl.arcroleTypes and len( self.modelXbrl.arcroleTypes[arcrole]) > 0: cyclesAllowed = self.modelXbrl.arcroleTypes[arcrole][ 0].cyclesAllowed if arcrole in self.genericArcArcroles: specSect = "xbrlgene:violatedCyclesConstraint" else: specSect = "xbrl.5.1.4.3:cycles" else: cyclesAllowed = "any" specSect = None if cyclesAllowed != "any" or arcrole in (XbrlConst.summationItem,) \ or arcrole in self.genericArcArcroles \ or arcrole.startswith(XbrlConst.formulaStartsWith): relsSet = modelXbrl.relationshipSet(arcrole, ELR, linkqname, arcqname) if cyclesAllowed != "any" and \ (XbrlConst.isStandardExtLinkQname(linkqname) and XbrlConst.isStandardArcQname(arcqname)) \ or arcrole in self.genericArcArcroles: noUndirected = cyclesAllowed == "none" fromRelationships = relsSet.fromModelObjects() for relFrom, rels in fromRelationships.items(): cycleFound = self.fwdCycle(relsSet, rels, noUndirected, {relFrom}) if cycleFound: modelXbrl.error( _("Relationships have a {0} cycle in arcrole {1} link role {2} link {3}, arc {4} starting from {5}" ).format(cycleFound, arcrole, ELR, linkqname, arcqname, relFrom.qname), "err", "{0}".format(specSect)) break # check calculation arcs for weight issues (note calc arc is an "any" cycles) if arcrole == XbrlConst.summationItem: for modelRel in relsSet.modelRelationships: weight = modelRel.weight fromConcept = modelRel.fromModelObject toConcept = modelRel.toModelObject if fromConcept and toConcept: if weight == 0: modelXbrl.error( _("Calculation relationship has zero weight from {0} to {1} in link role {2}" ).format(fromConcept.qname, toConcept.qname, ELR), "err", "xbrl.5.2.5.2.1:zeroWeight") fromBalance = fromConcept.balance toBalance = toConcept.balance if fromBalance and toBalance: if (fromBalance == toBalance and weight < 0) or \ (fromBalance != toBalance and weight > 0): modelXbrl.error( _("Calculation relationship has illegal weight {0} from {1}, {2}, to {3}, {4} in link role {5} (per 5.1.1.2 Table 6)" ).format(weight, fromConcept.qname, fromBalance, toConcept.qname, toBalance, ELR), "err", "xbrl.5.1.1.2:balanceCalcWeight") if not fromConcept.isNumeric or not toConcept.isNumeric: modelXbrl.error( _("Calculation relationship has illegal concept from {0}{1} to {2}{3} in link role {4}" ).format( fromConcept.qname, "" if fromConcept.isNumeric else " (non-numeric)", toConcept.qname, "" if fromConcept.isNumeric else " (non-numeric)", ELR), "err", "xbrl.5.2.5.2:nonNumericCalc") # check presentation relationships for preferredLabel issues elif arcrole == XbrlConst.parentChild: for modelRel in relsSet.modelRelationships: preferredLabel = modelRel.preferredLabel toConcept = modelRel.toModelObject if preferredLabel and toConcept and \ toConcept.label(preferredLabel=preferredLabel,fallbackToQname=False) is None: modelXbrl.error( _("Presentation relationship from {0} to {1} in link role {2} missing preferredLabel {3}" ).format(modelRel.fromModelObject.qname, toConcept.qname, ELR, preferredLabel), "err", "xbrl.5.2.4.2.1:preferredLabelMissing") # check essence-alias relationships elif arcrole == XbrlConst.essenceAlias: for modelRel in relsSet.modelRelationships: fromConcept = modelRel.fromModelObject toConcept = modelRel.toModelObject if fromConcept and toConcept: if fromConcept.type != toConcept.type or fromConcept.periodType != toConcept.periodType: modelXbrl.error( _("Essence-alias relationship from {0} to {1} in link role {2} has different types or periodTypes" ).format(fromConcept.qname, toConcept.qname, ELR), "err", "xbrl.5.2.6.2.2:essenceAliasTypes") fromBalance = fromConcept.balance toBalance = toConcept.balance if fromBalance and toBalance: if fromBalance and toBalance and fromBalance != toBalance: modelXbrl.error( _("Essence-alias relationship from {0} to {1} in link role {2} has different balances" ).format(fromConcept.qname, toConcept.qname, ELR), "err", "xbrl.5.2.6.2.2:essenceAliasBalance") elif modelXbrl.hasXDT and arcrole.startswith( XbrlConst.dimStartsWith): ValidateXbrlDimensions.checkBaseSet(self, arcrole, ELR, relsSet) elif modelXbrl.hasFormulae and arcrole.startswith( XbrlConst.formulaStartsWith): ValidateFormula.checkBaseSet(self, arcrole, ELR, relsSet) # instance checks modelXbrl.modelManager.showStatus(_("validating instance")) self.footnoteRefs = set() if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE or \ modelXbrl.modelDocument.type == ModelDocument.Type.INLINEXBRL: for f in modelXbrl.facts: concept = f.concept if concept: if concept.isNumeric: unit = f.unit if f.unitID is None or unit is None: self.modelXbrl.error( _("Fact {0} context {1} is numeric and must have a unit" ).format(modelXbrl.modelDocument.basename, f.qname, f.contextID), "err", "xbrl.4.6.2:numericUnit") else: if concept.isMonetary: measures = unit.measures if not measures or len(measures[0]) != 1 or len(measures[1]) != 0 or \ measures[0][0].namespaceURI != XbrlConst.iso4217 or \ not self.isoCurrencyPattern.match(measures[0][0].localName): self.modelXbrl.error( _("Fact {0} context {1} must have a monetary unit {2}" ).format( modelXbrl.modelDocument.basename, f.qname, f.contextID, f.unitID), "err", "xbrl.4.8.2:monetaryFactUnit") elif concept.isShares: measures = unit.measures if not measures or len(measures[0]) != 1 or len(measures[1]) != 0 or \ measures[0][0] != XbrlConst.qnXbrliShares: self.modelXbrl.error( _("Fact {0} context {1} must have a xbrli:shares unit {2}" ).format(f.qname, f.contextID, f.unitID), "err", "xbrl.4.8.2:sharesFactUnit") precision = f.precision hasPrecision = precision is not None if hasPrecision and precision != "INF" and not precision.isdigit( ): self.modelXbrl.error( _("Fact {0} context {1} precision {2} is invalid" ).format(f.qname, f.contextID, precision), "err", "xbrl.4.6.4:precision") decimals = f.decimals hasDecimals = decimals is not None if hasPrecision and not self.precisionPattern.match( precision): self.modelXbrl.error( _("Fact {0} context {1} precision {2} is invalid" ).format(f.qname, f.contextID, precision), "err", "xbrl.4.6.4:precision") if hasPrecision and hasDecimals: self.modelXbrl.error( _("Fact {0} context {1} can not have both precision and decimals" ).format(f.qname, f.contextID), "err", "xbrl.4.6.3:bothPrecisionAndDecimals") if hasDecimals and not self.decimalsPattern.match( decimals): self.modelXbrl.error( _("Fact {0} context {1} decimals {2} is invalid" ).format(f.qname, f.contextID, decimals), "err", "xbrl.4.6.5:decimals") if concept.isItem: context = f.context if context is None: self.modelXbrl.error( _("Item {0} must have a context").format( f.qname), "err", "xbrl.4.6.1:itemContextRef") else: periodType = concept.periodType if (periodType == "instant" and not context.isInstantPeriod) or \ (periodType == "duration" and not (context.isStartEndPeriod or context.isForeverPeriod)): self.modelXbrl.error( _("Fact {0} context {1} has period type {2} conflict with context" ).format(f.qname, f.contextID, periodType), "err", "xbrl.4.7.2:contextPeriodType") if modelXbrl.hasXDT: ValidateXbrlDimensions.checkFact(self, f) # check precision and decimals if f.xsiNil == "true": if hasPrecision or hasDecimals: self.modelXbrl.error( _("Fact {0} context {1} can not be nil and have either precision or decimals" ).format(f.qname, f.contextID), "err", "xbrl.4.6.3:nilPrecisionDecimals") elif concept.isFraction: if hasPrecision or hasDecimals: self.modelXbrl.error( _("Fact {0} context {1} is a fraction concept and cannot have either precision or decimals" ).format(f.qname, f.contextID), "err", "xbrl.4.6.3:fractionPrecisionDecimals") numerator, denominator = f.fractionValue if not (numerator == "INF" or numerator.isnumeric()): self.modelXbrl.error( _("Fact {0} context {1} is a fraction with invalid numerator {2}" ).format(f.qname, f.contextID, numerator), "err", "xbrl.5.1.1:fractionPrecisionDecimals") if not denominator.isnumeric() or int( denominator) == 0: self.modelXbrl.error( _("Fact {0} context {1} is a fraction with invalid denominator {2}" ).format(f.qname, f.contextID, denominator), "err", "xbrl.5.1.1:fractionPrecisionDecimals") else: if modelXbrl.modelDocument.type != ModelDocument.Type.INLINEXBRL: for child in f.element.childNodes: if child.nodeType == 1: self.modelXbrl.error( _("Fact {0} context {1} may not have child elements {2}" ).format(f.qname, f.contextID, child.tagName), "err", "xbrl.5.1.1:itemMixedContent") break if concept.isNumeric and not hasPrecision and not hasDecimals: self.modelXbrl.error( _("Fact {0} context {1} is a numeric concept and must have either precision or decimals" ).format(f.qname, f.contextID), "err", "xbrl.4.6.3:missingPrecisionDecimals") elif concept.isTuple: if f.contextID: self.modelXbrl.error( _("Tuple {0} must not have a context").format( f.qname), "err", "xbrl.4.6.1:tupleContextRef") if hasPrecision or hasDecimals: self.modelXbrl.error( _("Fact {0} is a tuple and cannot have either precision or decimals" ).format(f.qname), "err", "xbrl.4.6.3:tuplePrecisionDecimals") # custom attributes may be allowed by anyAttribute but not by 2.1 for attrQname, attrValue in XbrlUtil.attributes( self.modelXbrl, concept, f.element): if attrQname.namespaceURI in (XbrlConst.xbrli, XbrlConst.link, XbrlConst.xlink, XbrlConst.xl): self.modelXbrl.error( _("Fact {0} is a tuple and must not have attribute in this namespace {1}" ).format(f.qname, attrQname), "err", "xbrl.4.9:tupleAttribute") else: self.modelXbrl.error( _("Fact {0} must be an item or tuple").format( f.qname), "err", "xbrl.4.6:notItemOrTuple") from arelle.ModelObject import ModelInlineFact if isinstance(f, ModelInlineFact): self.footnoteRefs.update(f.footnoteRefs) #instance checks for cntx in modelXbrl.contexts.values(): if cntx.isStartEndPeriod: try: if cntx.endDatetime <= cntx.startDatetime: self.modelXbrl.error( _("Context {0} must have startDate less than endDate" ).format(cntx.id), "err", "xbrl.4.7.2:periodStartBeforeEnd") except ValueError as err: self.modelXbrl.error( _("Context {0} startDate or endDate: {1}").format( cntx.id, err), "err", "xbrl.4.7.2:contextDateError") elif cntx.isInstantPeriod: try: cntx.instantDatetime #parse field except ValueError as err: self.modelXbrl.error( _("Context {0} instant date: {1}").format( cntx.id, err), "err", "xbrl.4.7.2:contextDateError") self.segmentScenario(cntx.segment, cntx.id, "segment", "4.7.3.2") self.segmentScenario(cntx.scenario, cntx.id, "scenario", "4.7.4") if modelXbrl.hasXDT: ValidateXbrlDimensions.checkContext(self, cntx) for unit in modelXbrl.units.values(): mulDivMeasures = unit.measures if mulDivMeasures: for measures in mulDivMeasures: for measure in measures: if measure.namespaceURI == XbrlConst.xbrli and not \ measure in (XbrlConst.qnXbrliPure, XbrlConst.qnXbrliShares): self.modelXbrl.error( _("Unit {0} illegal measure: {1}").format( unit.id, measure), "err", "xbrl.4.8.2:measureElement") for numeratorMeasure in mulDivMeasures[0]: if numeratorMeasure in mulDivMeasures[1]: self.modelXbrl.error( _("Unit {0} numerator measure: {1} also appears as denominator measure" ).format(unit.id, numeratorMeasure), "err", "xbrl.4.8.4:measureBothNumDenom") #concepts checks modelXbrl.modelManager.showStatus(_("validating concepts")) for concept in modelXbrl.qnameConcepts.values(): conceptType = concept.type if XbrlConst.isStandardNamespace(concept.namespaceURI) or \ not concept.modelDocument.inDTS: continue if concept.isTuple: # must be global if not concept.element.parentNode.localName == "schema": self.modelXbrl.error( _("Tuple {0} must be declared globally").format( concept.qname), "err", "xbrl.4.9:tupleGloballyDeclared") if concept.periodType: self.modelXbrl.error( _("Tuple {0} must not have periodType").format( concept.qname), "err", "xbrl.4.9:tuplePeriodType") if concept.balance: self.modelXbrl.error( _("Tuple {0} must not have balance").format( concept.qname), "err", "xbrl.4.9:tupleBalance") # check attribute declarations for attributeQname in conceptType.attributes: if attributeQname.namespaceURI in (XbrlConst.xbrli, XbrlConst.link, XbrlConst.xlink, XbrlConst.xl): self.modelXbrl.error( _("Tuple {0} must not have attribute in this namespace {1}" ).format(concept.qname, attributeQname), "err", "xbrl.4.9:tupleAttribute") # check for mixed="true" or simple content if XmlUtil.descendantAttr(conceptType.element, XbrlConst.xsd, ("complexType", "complexContent"), "mixed") == "true": self.modelXbrl.error( _("Tuple {0} must not have mixed content").format( concept.qname), "err", "xbrl.4.9:tupleMixedContent") if XmlUtil.descendant(conceptType.element, XbrlConst.xsd, "simpleContent"): self.modelXbrl.error( _("Tuple {0} must not have simple content").format( concept.qname), "err", "xbrl.4.9:tupleSimpleContent") # child elements must be item or tuple for elementQname in conceptType.elements: childConcept = self.modelXbrl.qnameConcepts.get( elementQname) if childConcept is None: self.modelXbrl.error( _("Tuple {0} element {1} not defined").format( concept.qname, elementQname), "err", "xbrl.4.9:tupleElementUndefined") elif not ( childConcept.isItem or childConcept.isTuple or # isItem/isTuple do not include item or tuple itself childConcept.qname == XbrlConst.qnXbrliItem or # subs group includes item as member childConcept.qname == XbrlConst.qnXbrliTuple): self.modelXbrl.error( _("Tuple {0} must not have element {1} not an item or tuple" ).format(concept.qname, elementQname), "err", "xbrl.4.9:tupleElementItemOrTuple") elif concept.isItem: if concept.periodType not in periodTypeValues: #("instant","duration"): self.modelXbrl.error( _("Item {0} must have a valid periodType").format( concept.qname), "err", "xbrl.5.1.1.1:itemPeriodType") if concept.isMonetary: if concept.balance not in balanceValues: #(None, "credit","debit"): self.modelXbrl.error( _("Item {0} must have a valid balance {1}").format( concept.qname, concept.balance), "err", "xbrl.5.1.1.2:itemBalance") else: if concept.balance: self.modelXbrl.error( _("Item {0} may not have a balance").format( concept.qname), "err", "xbrl.5.1.1.2:itemBalance") if concept.baseXbrliType not in baseXbrliTypes: self.modelXbrl.error( _("Item {0} type {1} invalid").format( concept.qname, concept.baseXbrliType), "err", "xbrl.5.1.1.3:itemType") if modelXbrl.hasXDT: if concept.isHypercubeItem and not concept.abstract == "true": self.modelXbrl.error( _("Hypercube item {0} must be abstract").format( concept.qname), "err", "xbrldte:HypercubeElementIsNotAbstractError") elif concept.isDimensionItem and not concept.abstract == "true": self.modelXbrl.error( _("Dimension item {0} must be abstract").format( concept.qname), "err", "xbrldte:DimensionElementIsNotAbstractError") if modelXbrl.hasXDT: ValidateXbrlDimensions.checkConcept(self, concept) modelXbrl.modelManager.showStatus(_("validating DTS")) self.DTSreferenceResourceIDs = {} ValidateXbrlDTS.checkDTS(self, modelXbrl.modelDocument, []) del self.DTSreferenceResourceIDs if self.validateCalcLB: modelXbrl.modelManager.showStatus( _("Validating instance calculations")) ValidateXbrlCalcs.validate( modelXbrl, inferPrecision=(not self.validateInferDecimals)) if modelXbrl.modelManager.validateUtr: ValidateUtr.validate(modelXbrl) if modelXbrl.hasFormulae: ValidateFormula.validate(self) modelXbrl.modelManager.showStatus(_("ready"), 2000)