def typedDimElt(s): # add xmlns into s for known qnames tag, angleBrkt, rest = s[1:].partition('>') text, angleBrkt, rest = rest.partition("<") qn = qname(tag, prefixedNamespaces) # a modelObject xml element is needed for all of the instance functions to manage the typed dim return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
def typedDimElt(s): # add xmlns into s for known qnames tag, angleBrkt, rest = s[1:].partition('>') text, angleBrkt, rest = rest.partition("<") qn = qname(tag, prefixedNamespaces) # a modelObject xml element is needed for all of the instance functions to manage the typed dim return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
def createTargetInstance(modelXbrl, targetUrl, targetDocumentSchemaRefs, filingFiles, baseXmlLang=None, defaultXmlLang=None): def addLocallyReferencedFile(elt, filingFiles): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src") and not isHttpUrl( attrValue) and not os.path.isabs(attrValue): attrValue = attrValue.partition('#')[0] # remove anchor if attrValue: # ignore anchor references to base document attrValue = os.path.normpath( attrValue ) # change url path separators to host separators file = os.path.join(sourceDir, attrValue) if modelXbrl.fileSource.isInArchive( file, checkExistence=True) or os.path.exists(file): filingFiles.add(file) targetInstance = ModelXbrl.create( modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True, discover=False) # don't attempt to load DTS ixTargetRootElt = modelXbrl.ixTargetRootElements[getattr( modelXbrl, "ixdsTarget", None)] langIsSet = False # copy ix resources target root attributes for attrName, attrValue in ixTargetRootElt.items(): if attrName != "target": # ix:references target is not mapped to xbrli:xbrl targetInstance.modelDocument.xmlRootElement.set( attrName, attrValue) if attrName == "{http://www.w3.org/XML/1998/namespace}lang": langIsSet = True defaultXmlLang = attrValue if attrName.startswith("{"): ns, _sep, ln = attrName[1:].rpartition("}") if ns: prefix = xmlnsprefix(ixTargetRootElt, ns) if prefix not in (None, "xml"): setXmlns(targetInstance.modelDocument, prefix, ns) if not langIsSet and baseXmlLang: targetInstance.modelDocument.xmlRootElement.set( "{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang) if defaultXmlLang is None: defaultXmlLang = baseXmlLang # allows facts/footnotes to override baseXmlLang ValidateXbrlDimensions.loadDimensionDefaults( targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in sorted(modelXbrl.contexts.values(), key=lambda c: c.objectIndex ): # contexts may come from multiple IXDS files ignore = targetInstance.createContext( context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in sorted(modelXbrl.units.values(), key=lambda u: u.objectIndex ): # units may come from multiple IXDS files measures = unit.measures ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue for attrName, attrValue in fact.items(): if attrName.startswith("{"): attrs[qname( attrName, fact.nsmap )] = attrValue # using qname allows setting prefix in extracted instance newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) # if fact.isFraction, create numerator and denominator newFactForOldObjId[fact.objectIndex] = newFact if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references so that referenced files are included in the zip. for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML("<body>\n{0}\n</body>\n".format( xmltext)).iter(): addLocallyReferencedFile(elt, filingFiles) except (XMLSyntaxError, UnicodeDecodeError): pass # TODO: Why ignore UnicodeDecodeError? elif fact.isTuple: attrs = {} if fact.id: attrs["id"] = fact.id if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" for attrName, attrValue in fact.items(): if attrName.startswith("{"): attrs[qname(attrName, fact.nsmap)] = attrValue newTuple = targetInstance.createFact(fact.qname, attributes=attrs, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) modelXbrl.modelManager.showStatus( _("Creating and validating footnotes and relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) footnoteIdCount = {} for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any( lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format( attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) xmlLang = linkChild.xmlLang if xmlLang is not None and xmlLang != defaultXmlLang: # default newChild.set( "{http://www.w3.org/XML/1998/namespace}lang", xmlLang) copyIxFootnoteHtml( linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): addLocallyReferencedFile(elt, filingFiles) return targetInstance
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None, *args, **kwargs): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( targetDocumentFilename, modelXbrl.modelDocument.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus( _("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults( targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in modelXbrl.contexts.values(): newCntx = targetInstance.createContext( context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) newFactForOldObjId[fact.objectIndex] = newFact if filingFiles and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML( "<body>\n{0}\n</body>\n".format(xmltext)): if elt.tag in ("a", "img") and not isHttpUrl( attrValue) and not os.path.isabs( attrvalue): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src"): filingFiles.add(attrValue) except (XMLSyntaxError, UnicodeDecodeError): pass elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) # footnote links footnoteIdCount = {} modelXbrl.modelManager.showStatus( _("Creating and validating footnotes & relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any( lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format( attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) copyIxFootnoteHtml( linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ( "href", "src") and not isHttpUrl( attrValue ) and not os.path.isabs(attrvalue): filingFiles.add(attrValue) targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip) if getattr(modelXbrl, "isTestcaseVariation", False): modelXbrl.extractedInlineInstance = True # for validation comparison modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None, *args, **kwargs): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( targetDocumentFilename, modelXbrl.modelDocument.filepath) def addLocallyReferencedFile(elt, filingFiles): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src") and not isHttpUrl( attrValue) and not os.path.isabs(attrValue): attrValue = attrValue.partition('#')[0] # remove anchor if attrValue: # ignore anchor references to base document attrValue = os.path.normpath( attrValue ) # change url path separators to host separators file = os.path.join(sourceDir, attrValue) if modelXbrl.fileSource.isInArchive( file, checkExistence=True) or os.path.exists(file): filingFiles.add(file) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus( _("Extracting instance ") + os.path.basename(targetUrl)) rootElt = modelXbrl.modelDocument.xmlRootElement # take baseXmlLang from <html> or <base> baseXmlLang = rootElt.get( "{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang") for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants( tag="{http://www.w3.org/1999/xhtml}body"): baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang" ) or rootElt.get("lang") or baseXmlLang targetInstance = ModelXbrl.create( modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True, discover=False) # don't attempt to load DTS if baseXmlLang: targetInstance.modelDocument.xmlRootElement.set( "{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang) ValidateXbrlDimensions.loadDimensionDefaults( targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in sorted(modelXbrl.contexts.values(), key=lambda c: elementChildSequence(c)): ignore = targetInstance.createContext( context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue if fact.concept is not None and fact.concept.baseXsdType in ( "string", "normalizedString"): # default xmlLang = fact.xmlLang if xmlLang is not None and xmlLang != baseXmlLang: attrs[ "{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) # if fact.isFraction, create numerator and denominator newFactForOldObjId[fact.objectIndex] = newFact if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references so that referenced files are included in the zip. for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML("<body>\n{0}\n</body>\n".format( xmltext)).iter(): addLocallyReferencedFile(elt, filingFiles) except (XMLSyntaxError, UnicodeDecodeError): pass # TODO: Why ignore UnicodeDecodeError? elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) modelXbrl.modelManager.showStatus( _("Creating and validating footnotes and relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) footnoteIdCount = {} for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any( lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format( attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) xmlLang = linkChild.xmlLang if xmlLang is not None and xmlLang != baseXmlLang: # default newChild.set( "{http://www.w3.org/XML/1998/namespace}lang", xmlLang) copyIxFootnoteHtml( linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): addLocallyReferencedFile(elt, filingFiles) targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip) if getattr(modelXbrl, "isTestcaseVariation", False): modelXbrl.extractedInlineInstance = True # for validation comparison modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def createTargetInstance(modelXbrl, targetUrl, targetDocumentSchemaRefs, filingFiles, baseXmlLang=None, defaultXmlLang=None): targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True, discover=False) # don't attempt to load DTS if baseXmlLang: targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang) if defaultXmlLang is None: defaultXmlLang = baseXmlLang # allows facts/footnotes to override baseXmlLang ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in sorted(modelXbrl.contexts.values(), key=lambda c: c.objectIndex): # contexts may come from multiple IXDS files ignore = targetInstance.createContext(context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in sorted(modelXbrl.units.values(), key=lambda u: u.objectIndex): # units may come from multiple IXDS files measures = unit.measures ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default xmlLang = fact.xmlLang if xmlLang is not None and xmlLang != defaultXmlLang: attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) # if fact.isFraction, create numerator and denominator newFactForOldObjId[fact.objectIndex] = newFact if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references so that referenced files are included in the zip. for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter(): addLocallyReferencedFile(elt, filingFiles) except (XMLSyntaxError, UnicodeDecodeError): pass # TODO: Why ignore UnicodeDecodeError? elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) footnoteIdCount = {} for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format(attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) xmlLang = linkChild.xmlLang if xmlLang is not None and xmlLang != defaultXmlLang: # default newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang) copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): addLocallyReferencedFile(elt,filingFiles) return targetInstance
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in modelXbrl.contexts.values(): newCntx = targetInstance.createContext(context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) newFactForOldObjId[fact.objectIndex] = newFact if filingFiles and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)): if elt.tag in ("a", "img") and not isHttpUrl(attrValue) and not os.path.isabs(attrvalue): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src"): filingFiles.add(attrValue) except (XMLSyntaxError, UnicodeDecodeError): pass elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) # footnote links footnoteIdCount = {} modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format(attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) copyIxFootnoteHtml(linkChild, newChild, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrvalue): filingFiles.add(attrValue) targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def createModelFact(fact, parentModelFact, topTupleFact): aspects = fact.get("aspects", EMPTYDICT) if oimConcept not in aspects: modelXbrl.error("{}:conceptQName".format(errPrefix), _("The concept QName could not be determined"), modelObject=modelXbrl) return conceptQn = qname(aspects[oimConcept], prefixes) concept = modelXbrl.qnameConcepts.get(conceptQn) attrs = {} if concept.isItem: missingAspects = [] if oimEntity not in aspects: missingAspects.append(oimEntity) if oimPeriod not in aspects and (oimPeriodStart not in aspects or oimPeriodEnd not in aspects): missingAspects.append(oimPeriod) if missingAspects: modelXbrl.error("{}:missingAspects".format(errPrefix), _("The concept %(element)s is missing aspects %(missingAspects)s"), modelObject=modelXbrl, element=conceptQn, missingAspects=", ".join(missingAspects)) return entityAsQn = qname(aspects[oimEntity], prefixes) or qname("error",fact[oimEntity]) if oimPeriod in aspects: periodStart = periodEnd = aspects[oimPeriod] if oimPeriodStart in aspects and oimPeriodEnd in aspects: periodStart = aspects[oimPeriodStart] periodEnd = aspects[oimPeriodEnd] cntxKey = ( # hashable context key ("periodType", concept.periodType), ("entity", entityAsQn), ("periodStart", periodStart), ("periodEnd", periodEnd)) + tuple(sorted( (dimName, dimVal["value"] if isinstance(dimVal,dict) else dimVal) for dimName, dimVal in aspects.items() if ":" in dimName and not dimName.startswith(oimPrefix))) if cntxKey in cntxTbl: _cntx = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) qnameDims = {} for dimName, dimVal in aspects.items(): if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal: dimQname = qname(dimName, prefixes) dimConcept = modelXbrl.qnameConcepts.get(dimQname) if dimConcept is None: modelXbrl.error("{}:taxonomyDefinedAspectQName".format(errPrefix), _("The taxonomy defined aspect concept QName %(qname)s could not be determined"), modelObject=modelXbrl, qname=dimQname) continue if isinstance(dimVal, dict): dimVal = dimVal["value"] else: dimVal = str(dimVal) # may be int or boolean if isinstance(dimVal,str) and ":" in dimVal and dimVal.partition(':')[0] in prefixes: mem = qname(dimVal, prefixes) # explicit dim elif dimConcept.isTypedDimension: # a modelObject xml element is needed for all of the instance functions to manage the typed dim mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment") _cntx = modelXbrl.createContext( entityAsQn.namespaceURI, entityAsQn.localName, concept.periodType, None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME), dateTime(periodEnd, type=DATETIME), None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId, beforeSibling=topTupleFact) cntxTbl[cntxKey] = _cntx if oimUnit in aspects and concept.isNumeric: unitKey = aspects[oimUnit] if unitKey in unitTbl: _unit = unitTbl[unitKey] else: _unit = None # validate unit unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey) if not UnitPattern.match(unitKeySub): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation is lexically invalid, %(unit)s"), modelObject=modelXbrl, unit=unitKey) else: _mul, _sep, _div = unitKey.partition('/') if _mul.startswith('('): _mul = _mul[1:-1] _muls = [u for u in _mul.split('*') if u] if _div.startswith('('): _div = _div[1:-1] _divs = [u for u in _div.split('*') if u] if _muls != sorted(_muls) or _divs != sorted(_divs): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation measures are not in alphabetical order, %(unit)s"), modelObject=modelXbrl, unit=unitKey) try: mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _muls] divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _divs] unitId = 'u-{:02}'.format(len(unitTbl) + 1) for _measures in mulQns, divQns: for _measure in _measures: addQnameValue(modelXbrl.modelDocument, _measure) _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId, beforeSibling=topTupleFact) except OIMException as ex: modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) unitTbl[unitKey] = _unit else: _unit = None attrs["contextRef"] = _cntx.id if fact.get("value") is None: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact["value"] if concept.isNumeric: if _unit is None: return # skip creating fact because unit was invalid attrs["unitRef"] = _unit.id if "accuracy" in attrs or attrs.get(XbrlConst.qnXsiNil, "false") != "true": attrs["decimals"] = fact.get("accuracy", "INF") else: text = None #tuple id = fact.get("id") if id is not None: attrs["id"] = fact["id"] # is value a QName? if concept.baseXbrliType == "QName": addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes)) f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text, parent=parentModelFact, validate=False) if id is not None and id in parentedFacts: # create child facts for i in sorted(parentedFacts[id], key=lambda j: facts[j].get("aspects", EMPTYDICT).get(oimTupleOrder,0)): createModelFact(facts[i], f, f if topTupleFact is None else topTupleFact) # validate after creating tuple contents xmlValidate(modelXbrl, f)
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri): from openpyxl import load_workbook from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname _return = None # modelDocument or an exception try: currentAction = "initializing" startingErrorCount = len(modelXbrl.errors) startedAt = time.time() if os.path.isabs(oimFile): # allow relative filenames to loading directory priorCWD = os.getcwd() os.chdir(os.path.dirname(oimFile)) else: priorCWD = None currentAction = "determining file type" isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") # this option is not currently supported isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls") isCSVorXL = isCSV or isXL instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl" _csvwContext = None if isJSON: errPrefix = "xbrlje" currentAction = "loading and parsing JSON OIM file" def loadDict(keyValuePairs): _dict = OrderedDict() # preserve fact order in resulting instance for key, value in keyValuePairs: if isinstance(value, dict): if DUPJSONKEY in value: for _errKey, _errValue, _otherValue in value[DUPJSONKEY]: if key == "prefixes": modelXbrl.error("xbrlje:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"), modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue) del value[DUPJSONKEY] if key in _dict: if DUPJSONKEY not in _dict: _dict[DUPJSONKEY] = [] _dict[DUPJSONKEY].append((key, value, _dict[key])) else: _dict[key] = value return _dict with io.open(oimFile, 'rt', encoding='utf-8') as f: oimObject = json.load(f, object_pairs_hook=loadDict) # check if it's a CSVW metadata _csvwContext = oimObject.get("@context") if _csvwContext == "http://www.w3.org/ns/csvw" or ( isinstance(_csvwContext, list) and "http://www.w3.org/ns/csvw" in _csvwContext): isJSON = False isCSV = True else: if oimObject.get("documentType", None) != "http://www.xbrl.org/WGWD/YYYY-MM-DD/xbrl-json": # is it likely to be an OIM document? if any(t in oimObject for t in ("dtsReferences", "prefixes", "facts")): raise OIMException("xbrlje:missingJSONdocumentType", _("Required documentType is missing from JSON input")) else: raise NotOIMException() # return None, not an OIM document missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject] if missing: raise OIMException("xbrlje:missingJSONElements", _("Required element(s) are missing from JSON input: %(missing)s"), missing = ", ".join(missing)) currentAction = "identifying JSON objects" dtsReferences = oimObject["dtsReferences"] prefixesList = oimObject["prefixes"].items() facts = oimObject["facts"] footnotes = oimObject["facts"] # shares this object if isCSV: errPrefix = "xbrlce" currentAction = "compiling metadata" if sys.version[0] >= '3': csvOpenMode = 'w' csvOpenNewline = '' else: csvOpenMode = 'wb' # for 2.7 csvOpenNewline = None if not _csvwContext: # json metadata wasn't specified, try to find it raise OIMException("xbrlce:missingCSVMetadata", _("Unable to identify CSV metadata file")) # process CSV metadata # mandatory sections of metadata file oimMetadata = oimObject.get(CSVmetadata) if not oimMetadata: raise OIMException("xbrlce:missingOIMMetadata", _("Unable to identify OIM metadata infile")) if oimMetadata.get("documentType") != CSVdocumentType: raise OIMException("xbrlce:documentType", _("Document type %(documentType)s not recognized, expecting %(expectedDocumentType)s"), documentType=oimMetadata.get("documentType"), expectedDocumentType=CSVdocumentType) dtsReferences = oimMetadata.get("dtsReferences", {}) prefixesList = oimMetadata.get("prefixes", {}).items() topLevelAspects = oimObject.get(CSVaspects, {}) currentAction = "loading CSV facts tables" facts = [] footnotes = [] _dir = os.path.dirname(oimFile) for oimTable in oimObject.get("tables", []): tableType = oimTable.get(CSVtableType) tableLevelAspects = oimTable.get(CSVaspects, {}) tableUrl = oimTable.get("url") tableColumns = oimTable.get("tableSchema",{}).get("columns", []) # compile column dependencies aspectCols = [] factCols = [] for iCol, col in enumerate(tableColumns): if CSVcolumnAspect in col: aspectCols.append(iCol) if CSVtupleFactAspects in col or CSVsimpleFactAspects in col: factCols.append(iCol) if tableType not in ("fact", "footnote"): modelXbrl.error("xbrlce:unrecognizedTableType", _("Table type %(tableType)s was not recognized, table URI %(uri)s."), modelObject=modelXbrl, uri=oimFile, tableType=tableType) continue if not tableColumns: modelXbrl.error("xbrlce:noTableColumns", _("Table has no columns, table URI %(uri)s."), modelObject=modelXbrl, uri=_uri) continue tableAspects = topLevelAspects.copy() tableAspects.update(tableLevelAspects) filepath = os.path.join(_dir, tableUrl) tupleIds = set() with io.open(filepath, 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: if tableType == "fact": colAspects = tableAspects.copy() specificColAspects = defaultdict(dict) for iCol in aspectCols: value = row[iCol] aspect = tableColumns[iCol][CSVcolumnAspect] if isinstance(aspect, str): # applies to all cols colAspects[aspect] = value elif isinstance(aspect, dict): # applies to specific cols for _aspect, _colNames in aspect.items(): for _colName in _colNames: specificColAspects[_colName][_aspect] = value for iCol in factCols: fact = {"aspects": {}} cellValue = row[iCol] if cellValue == "": # no fact produced for this cell continue tableCol = tableColumns[iCol] if CSVtupleFactAspects in tableCol: aspectsObjectName = CSVtupleFactAspects else: aspectsObjectName = CSVsimpleFactAspects cellAspects = (colAspects, specificColAspects.get(tableCol.get("name"), EMPTYDICT), tableColumns[iCol].get(aspectsObjectName), EMPTYDICT) inapplicableAspects = set() if tableCol.get(CSVtupleReferenceId) == "true": if cellValue: if cellValue in tupleIds: continue # don't duplicate parent tuple fact["id"] = cellValue tupleIds.add(cellValue) # prevent tuple duplication elif CSVsimpleFactAspects in tableCol: fact["value"] = csvCellValue(cellValue) if CSVtupleFactAspects in tableCol: inapplicableAspects.update(oimSimpleFactAspects) for _aspects in cellAspects: for aspectName, aspectValue in _aspects.items(): if not aspectName.startswith(oimPrefix): inapplicableAspects.add(aspectName) # block any row aspect produced by this column from this column's fact _colAspect = tableCol.get(CSVcolumnAspect) if isinstance(_colAspect, str): # applies to all cols inapplicableAspects.add(_colAspect) for _aspects in cellAspects: if _aspects: for aspectName, aspectValue in _aspects.items(): if aspectName not in inapplicableAspects and aspectValue != "": if ":" in aspectName: fact["aspects"][aspectName] = csvCellValue(aspectValue) else: fact[aspectName] = aspectValue facts.append(fact) elif tableType == "footnote": footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col)) del tupleIds elif isXL: errPrefix = "xbrlwe" currentAction = "identifying workbook input worksheets" oimWb = load_workbook(oimFile, read_only=True, data_only=True) sheetNames = oimWb.get_sheet_names() if (not any(sheetName == "prefixes" for sheetName in sheetNames) or not any(sheetName == "dtsReferences" for sheetName in sheetNames) or not any("facts" in sheetName for sheetName in sheetNames)): raise OIMException("xbrlwe:missingWorkbookWorksheets", _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts")) currentAction = "loading worksheet: dtsReferences" dtsReferences = [] for i, row in enumerate(oimWb["dtsReferences"]): if i == 0: header = [col.value for col in row] else: dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row))) currentAction = "loading worksheet: prefixes" prefixesList = [] for i, row in enumerate(oimWb["prefixes"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) else: prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value)) defaults = {} if "defaults" in sheetNames: currentAction = "loading worksheet: defaults" for i, row in enumerate(oimWb["defaults"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) fileCol = header["file"] else: defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol) facts = [] for sheetName in sheetNames: if sheetName == "facts" or "-facts" in sheetName: currentAction = "loading worksheet: {}".format(sheetName) tableDefaults = defaults.get(sheetName, {}) for i, row in enumerate(oimWb[sheetName]): if i == 0: header = [col.value for col in row] else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col.value is not None: if header[j]: # skip cols with no header if header[j].endswith("Value"): fact["value"] = str(col.value) else: fact[header[j]] = str(col.value) facts.append(fact) footnotes = [] if "footnotes" in sheetNames: currentAction = "loading worksheet: footnotes" for i, row in enumerate(oimWb["footnotes"]): if i == 0: header = [col.value for j,col in enumerate(row) if col.value] else: footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value)) currentAction = "identifying default dimensions" ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults currentAction = "validating OIM" prefixes = {} prefixedUris = {} for _prefix, _uri in prefixesList: if not _prefix: modelXbrl.error("{}:emptyPrefix".format(errPrefix), _("The empty string must not be used as a prefix, uri %(uri)s"), modelObject=modelXbrl, uri=_uri) elif not NCNamePattern.match(_prefix): modelXbrl.error("oime:prefixPattern", _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"), modelObject=modelXbrl, prefix=_prefix, uri=_uri) elif _prefix in prefixes: modelXbrl.error("{}:duplicatedPrefix".format(errPrefix), _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"), modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri) elif _uri in prefixedUris: modelXbrl.error("{}:duplicatedUri".format(errPrefix), _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"), modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix) else: prefixes[_prefix] = _uri prefixedUris[_uri] = _prefix for _nsOim in nsOim: if _nsOim in prefixedUris: _oimPrefix = prefixedUris[_nsOim] if not _oimPrefix: raise OIMException("oime:noOimPrefix", _("The oim namespace must have a declared prefix")) # create the instance document currentAction = "creating instance document" modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = _return = createModelDocument( modelXbrl, Type.INSTANCE, instanceFileName, schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef.get("type") == "schema" and dtsRef.get("href")], isEntry=True, initialComment="extracted from OIM {}".format(mappedUri), documentEncoding="utf-8") modelXbrl.modelDocument.inDTS = True # add linkbase, role and arcrole refs for refType in ("linkbase", "role", "arcrole"): for dtsRef in dtsReferences: if dtsRef.get("type") == refType and dtsRef.get("href"): elt = addChild(modelXbrl.modelDocument.xmlRootElement, qname(link, refType+"Ref"), attributes=(("{http://www.w3.org/1999/xlink}href", dtsRef["href"]), ("{http://www.w3.org/1999/xlink}type", "simple"))) href = modelXbrl.modelDocument.discoverHref(elt) if href: _elt, hrefDoc, hrefId = href _defElt = hrefDoc.idObjects.get(hrefId) _uriAttrName = refType + "URI" _uriAttrValue = _defElt.get(_uriAttrName) if _defElt is not None and _uriAttrValue: elt.set(_uriAttrName, _uriAttrValue) cntxTbl = {} unitTbl = {} # determine tuple dependency order idFacts = {} parentedFacts = defaultdict(list) for i, fact in enumerate(facts): aspects = fact["aspects"] id = fact.get("id") tupleParent = fact.get("aspects", EMPTYDICT).get(oimTupleParent) if id is not None: idFacts[id] = i parentedFacts[tupleParent].append(i) # root facts have tupleParent None currentAction = "creating facts" def createModelFact(fact, parentModelFact, topTupleFact): aspects = fact.get("aspects", EMPTYDICT) if oimConcept not in aspects: modelXbrl.error("{}:conceptQName".format(errPrefix), _("The concept QName could not be determined"), modelObject=modelXbrl) return conceptQn = qname(aspects[oimConcept], prefixes) concept = modelXbrl.qnameConcepts.get(conceptQn) attrs = {} if concept.isItem: missingAspects = [] if oimEntity not in aspects: missingAspects.append(oimEntity) if oimPeriod not in aspects and (oimPeriodStart not in aspects or oimPeriodEnd not in aspects): missingAspects.append(oimPeriod) if missingAspects: modelXbrl.error("{}:missingAspects".format(errPrefix), _("The concept %(element)s is missing aspects %(missingAspects)s"), modelObject=modelXbrl, element=conceptQn, missingAspects=", ".join(missingAspects)) return entityAsQn = qname(aspects[oimEntity], prefixes) or qname("error",fact[oimEntity]) if oimPeriod in aspects: periodStart = periodEnd = aspects[oimPeriod] if oimPeriodStart in aspects and oimPeriodEnd in aspects: periodStart = aspects[oimPeriodStart] periodEnd = aspects[oimPeriodEnd] cntxKey = ( # hashable context key ("periodType", concept.periodType), ("entity", entityAsQn), ("periodStart", periodStart), ("periodEnd", periodEnd)) + tuple(sorted( (dimName, dimVal["value"] if isinstance(dimVal,dict) else dimVal) for dimName, dimVal in aspects.items() if ":" in dimName and not dimName.startswith(oimPrefix))) if cntxKey in cntxTbl: _cntx = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) qnameDims = {} for dimName, dimVal in aspects.items(): if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal: dimQname = qname(dimName, prefixes) dimConcept = modelXbrl.qnameConcepts.get(dimQname) if dimConcept is None: modelXbrl.error("{}:taxonomyDefinedAspectQName".format(errPrefix), _("The taxonomy defined aspect concept QName %(qname)s could not be determined"), modelObject=modelXbrl, qname=dimQname) continue if isinstance(dimVal, dict): dimVal = dimVal["value"] else: dimVal = str(dimVal) # may be int or boolean if isinstance(dimVal,str) and ":" in dimVal and dimVal.partition(':')[0] in prefixes: mem = qname(dimVal, prefixes) # explicit dim elif dimConcept.isTypedDimension: # a modelObject xml element is needed for all of the instance functions to manage the typed dim mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment") _cntx = modelXbrl.createContext( entityAsQn.namespaceURI, entityAsQn.localName, concept.periodType, None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME), dateTime(periodEnd, type=DATETIME), None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId, beforeSibling=topTupleFact) cntxTbl[cntxKey] = _cntx if oimUnit in aspects and concept.isNumeric: unitKey = aspects[oimUnit] if unitKey in unitTbl: _unit = unitTbl[unitKey] else: _unit = None # validate unit unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey) if not UnitPattern.match(unitKeySub): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation is lexically invalid, %(unit)s"), modelObject=modelXbrl, unit=unitKey) else: _mul, _sep, _div = unitKey.partition('/') if _mul.startswith('('): _mul = _mul[1:-1] _muls = [u for u in _mul.split('*') if u] if _div.startswith('('): _div = _div[1:-1] _divs = [u for u in _div.split('*') if u] if _muls != sorted(_muls) or _divs != sorted(_divs): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation measures are not in alphabetical order, %(unit)s"), modelObject=modelXbrl, unit=unitKey) try: mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _muls] divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _divs] unitId = 'u-{:02}'.format(len(unitTbl) + 1) for _measures in mulQns, divQns: for _measure in _measures: addQnameValue(modelXbrl.modelDocument, _measure) _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId, beforeSibling=topTupleFact) except OIMException as ex: modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) unitTbl[unitKey] = _unit else: _unit = None attrs["contextRef"] = _cntx.id if fact.get("value") is None: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact["value"] if concept.isNumeric: if _unit is None: return # skip creating fact because unit was invalid attrs["unitRef"] = _unit.id if "accuracy" in attrs or attrs.get(XbrlConst.qnXsiNil, "false") != "true": attrs["decimals"] = fact.get("accuracy", "INF") else: text = None #tuple id = fact.get("id") if id is not None: attrs["id"] = fact["id"] # is value a QName? if concept.baseXbrliType == "QName": addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes)) f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text, parent=parentModelFact, validate=False) if id is not None and id in parentedFacts: # create child facts for i in sorted(parentedFacts[id], key=lambda j: facts[j].get("aspects", EMPTYDICT).get(oimTupleOrder,0)): createModelFact(facts[i], f, f if topTupleFact is None else topTupleFact) # validate after creating tuple contents xmlValidate(modelXbrl, f) for i in parentedFacts[None]: createModelFact(facts[i], None, None) currentAction = "creating footnotes" footnoteLinks = OrderedDict() # ELR elements factLocs = {} # index by (linkrole, factId) footnoteNbr = 0 locNbr = 0 for factOrFootnote in footnotes: if "factId" in factOrFootnote: factId = factOrFootnote["factId"] factFootnotes = (factOrFootnote,) # CSV or XL elif "id" in factOrFootnote and "footnotes" in factOrFootnote: factId = factOrFootnote["id"] factFootnotes = factOrFootnote["footnotes"] else: factFootnotes = () for footnote in factFootnotes: linkrole = footnote.get("group") arcrole = footnote.get("footnoteType") if not factId or not linkrole or not arcrole or not ( footnote.get("factRef") or footnote.get("footnote")): # invalid footnote continue if linkrole not in footnoteLinks: footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, XbrlConst.qnLinkFootnoteLink, attributes={"{http://www.w3.org/1999/xlink}type": "extended", "{http://www.w3.org/1999/xlink}role": linkrole}) footnoteLink = footnoteLinks[linkrole] if (linkrole, factId) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factId)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factId, XLINKLABEL: locLabel}) locFromLabel = factLocs[(linkrole, factId)] if footnote.get("footnote"): footnoteNbr += 1 footnoteToLabel = "f_{:02}".format(footnoteNbr) attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteToLabel} if footnote.get("language"): attrs[XMLLANG] = footnote["language"] # note, for HTML will need to build an element structure addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"]) elif footnote.get("factRef"): factRef = footnote.get("factRef") if (linkrole, factRef) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factRef)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factRef, XLINKLABEL: locLabel}) footnoteToLabel = factLocs[(linkrole, factRef)] footnoteArc = addChild(footnoteLink, XbrlConst.qnLinkFootnoteArc, attributes={XLINKTYPE: "arc", XLINKARCROLE: arcrole, XLINKFROM: locFromLabel, XLINKTO: footnoteToLabel}) if footnoteLinks: modelXbrl.modelDocument.linkbaseDiscover(footnoteLinks.values(), inInstance=True) currentAction = "done loading facts and footnotes" #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") except NotOIMException as ex: _return = ex # not an OIM document except Exception as ex: _return = ex if isinstance(ex, OIMException): modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) else: modelXbrl.error("arelleOIMloader:error", "Error while %(action)s, error %(error)s\ntraceback %(traceback)s", modelObject=modelXbrl, action=currentAction, error=ex, traceback=traceback.format_tb(sys.exc_info()[2])) if priorCWD: os.chdir(priorCWD) # restore prior current working directory startingErrorCount = len(modelXbrl.errors) #if startingErrorCount < len(modelXbrl.errors): # # had errors, don't allow ModelDocument.load to continue # return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors") return _return
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri): from openpyxl import load_workbook from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname try: currentAction = "initializing" startedAt = time.time() if os.path.isabs(oimFile): # allow relative filenames to loading directory priorCWD = os.getcwd() os.chdir(os.path.dirname(oimFile)) else: priorCWD = None currentAction = "determining file type" isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json") isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json") isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls") isCSVorXL = isCSV or isXL instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl" if isJSON: currentAction = "loading and parsing JSON OIM file" with io.open(oimFile, 'rt', encoding='utf-8') as f: oimObject = json.load(f) missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject] if missing: raise OIMException("oime:missingJSONelements", _("The required elements %(missing)s {} in JSON input"), missing = ", ".join(missing)) currentAction = "identifying JSON objects" dtsReferences = oimObject["dtsReferences"] prefixes = oimObject["prefixes"] facts = oimObject["facts"] footnotes = oimOject["facts"] # shares this object elif isCSV: currentAction = "identifying CSV input tables" if sys.version[0] >= '3': csvOpenMode = 'w' csvOpenNewline = '' else: csvOpenMode = 'wb' # for 2.7 csvOpenNewline = None oimFileBase = None if "-facts" in oimFile: oimFileBase = oimFile.partition("-facts")[0] else: for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"): if oimFile.endswith(suffix): oimFileBase = oimFile[:-length(suffix)] break if oimFileBase is None: raise OIMException("oime:missingCSVtables", _("Unable to identify CSV tables file name pattern")) if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or not os.path.exists(oimFileBase + "-prefixes.csv")): raise OIMException("oime:missingCSVtables", _("Unable to identify CSV tables for dtsReferences or prefixes")) instanceFileName = oimFileBase + ".xbrl" currentAction = "loading CSV dtsReferences table" dtsReferences = [] with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: dtsReferences.append(dict((header[j], col) for j, col in enumerate(row))) currentAction = "loading CSV prefixes table" prefixes = {} with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = dict((col,i) for i,col in enumerate(row)) else: prefixes[row[header["prefix"]]] = row[header["URI"]] currentAction = "loading CSV defaults table" defaults = {} if os.path.exists(oimFileBase + "-defaults.csv"): with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row fileCol = row.index("file") else: defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol) currentAction = "loading CSV facts tables" facts = [] _dir = os.path.dirname(oimFileBase) for filename in os.listdir(_dir): filepath = os.path.join(_dir, filename) if "-facts" in filename and filepath.startswith(oimFileBase): tableDefaults = defaults.get(filename, {}) with io.open(filepath, 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col is not None: if header[j].endswith("Value"): if col: # ignore empty columns (= null CSV value) fact["value"] = col else: fact[header[j]] = col facts.append(fact) footnotes = [] if os.path.exists(oimFileBase + "-footnotes.csv"): with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col)) elif isXL: oimWb = load_workbook(oimFile, read_only=True, data_only=True) sheetNames = oimWb.get_sheet_names() if (not any(sheetName == "prefixes" for sheetName in sheetNames) or not any(sheetName == "dtsReferences" for sheetName in sheetNames) or not any("facts" in sheetName for sheetName in sheetNames)): if priorCWD: os.chdir(priorCWD) return None try: dtsReferences = [] for i, row in oimWb["dtsReferences"]: if i == 0: header = [col.value for col in row] else: dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row))) prefixes = {} for i, row in oimWb["dtsReferences"]: if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) else: prefixes[row[header["prefix"]].value] = row[header["URI"].value] defaults = {} for i, row in oimW.get("defaults", ()): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) fileCol = header["file"] else: defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol) facts = [] _dir = os.path.dirpart(oimFileBase) for sheetName in sheetNames: if sheetName == "facts" or "-facts" in sheetName: for i, row in oimWb[sheetName]: if i == 0: header = [col.value for col in row] else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col.value is not None: if header[j].endswith("Value"): fact["value"] = str(col.value) else: fact[header[j]] = str(col.value) facts.append(fact) footnotes = [] for i, row in oimWb.get("footnotes", ()): if i == 0: header = dict((col.value,i) for i,col in enumerate(row) if col.value) else: footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value)) except Exception as ex: if priorCWD: os.chdir(priorCWD) return None ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults # create the instance document modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument( modelXbrl, Type.INSTANCE, instanceFileName, schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"], isEntry=True, initialComment="extracted from OIM {}".format(mappedUri), documentEncoding="utf-8") cntxTbl = {} unitTbl = {} for fact in facts: conceptQn = qname(fact["oim:concept"], prefixes) concept = modelXbrl.qnameConcepts.get(conceptQn) entityAsQn = qname(fact["oim:entity"], prefixes) if "oim:period" in fact: periodStart = fact["oim:period"]["start"] periodEnd = fact["oim:period"]["end"] else: periodStart = fact["oim:periodStart"] periodEnd = fact["oim:periodEnd"] cntxKey = ( # hashable context key ("periodType", concept.periodType), ("entity", entityAsQn), ("periodStart", periodStart), ("periodEnd", periodEnd)) + tuple(sorted( (dimName, dimVal) for dimName, dimVal in fact.items() if ":" in dimName and not dimName.startswith("oim:"))) if cntxKey in cntxTbl: _cntx = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) qnameDims = {} for dimName, dimVal in fact.items(): if ":" in dimName and not dimName.startswith("oim:"): dimQname = qname(dimName, prefixes) dimConcept = modelXbrl.qnameConcepts.get(dimQname) if ":" in dimVal and dimVal.partition[':'][0] in prefixes: mem = qname(dimVal, prefixes) # explicit dim elif dimConcept.isTypedDimension: # a modelObject xml element is needed for all of the instance functions to manage the typed dim mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment") _cntx = modelXbrl.createContext( entityAsQn.namespaceURI, entityAsQn.localName, concept.periodType, None if concept.periodType == "instnat" else dateTime(periodStart, type=DATETIME), dateTime(periodEnd, type=DATETIME), None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) cntxTbl[cntxKey] = _cntx if "oim:unit" in fact: unitKey = fact["oim:unit"] if unitKey in unitTbl: _unit = unitTbl[unitKey] else: _mul, _sep, _div = unitKey.partition('/') if _mul.startswith('('): _mul = _mul[1:-1] mulQns = [qname(u, prefixes) for u in _mul.split('*') if u] if _div.startswith('('): _div = _div[1:-1] divQns = [qname(u, prefixes) for u in _div.split('*') if u] unitId = 'u-{:02}'.format(len(unitTbl) + 1) for _measures in mulQns, divQns: for _measure in _measures: addQnameValue(modelXbrl.modelDocument, _measure) _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId) unitTbl[unitKey] = _unit else: _unit = None attrs = {"contextRef": _cntx.id} if fact.get("value") is None: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact["value"] if fact.get("id"): attrs["id"] = fact["id"] if concept.isNumeric: if _unit is not None: attrs["unitRef"] = _unit.id if "accuracy" in fact: attrs["decimals"] = fact["accuracy"] # is value a QName? if concept.baseXbrliType == "QName": addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes)) f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text) footnoteLinks = {} # ELR elements factLocs = {} # index by (linkrole, factId) footnoteNbr = 0 locNbr = 0 for factOrFootnote in footnotes: if "factId" in factOrFootnote: factId = factOrFootnote["factId"] factFootnotes = (factOrFootnote,) # CSV or XL elif "id" in factOrFootnote and "footnotes" in factOrFootnote: factId = factOrFootnote["id"] factFootnotes = factOrFootnote["footnotes"] else: factFootnotes = () for footnote in factFootnotes: linkrole = footnote.get("group") arcrole = footnote.get("footnoteType") if not factId or not linkrole or not arcrole or not ( footnote.get("factRef") or footnote.get("footnote")): # invalid footnote continue if linkrole not in footnoteLinks: footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, XbrlConst.qnLinkFootnoteLink, attributes={"{http://www.w3.org/1999/xlink}type": "extended", "{http://www.w3.org/1999/xlink}role": linkrole}) footnoteLink = footnoteLinks[linkrole] if (linkrole, factId) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factId)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factId, XLINKLABEL: locLabel}) locLabel = factLocs[(linkrole, factId)] if footnote.get("footnote"): footnoteNbr += 1 footnoteLabel = "f_{:02}".format(footnoteNbr) attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel} if footnote.get("language"): attrs[XMLLANG] = footnote["language"] # note, for HTML will need to build an element structure addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"]) elif footnote.get("factRef"): factRef = footnote.get("factRef") if (linkrole, factRef) not in factLocs: locNbr += 1 locLabel = "f_{:02}".format(footnoteNbr) factLoc[(linkrole, factRef)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factRef, XLINKLABEL: locLabel}) footnoteLabel = factLoc[(linkrole, factId)] footnoteArc = addChild(footnoteLink, XbrlConst.qnLinkFootnoteArc, attributes={XLINKTYPE: "arc", XLINKARCROLE: arcrole, XLINKFROM: locLabel, XLINKTO: footnoteLabel}) #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") except Exception as ex: if ex is OIMException: modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) else: modelXbrl.error("Error while %(action)s, error %(error)s", modelObject=modelXbrl, action=currentAction, error=ex) if priorCWD: os.chdir(priorCWD) # restore prior current working directory return modelXbrl.modelDocument
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri): from openpyxl import load_workbook from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname try: currentAction = "initializing" startingErrorCount = len(modelXbrl.errors) startedAt = time.time() if os.path.isabs(oimFile): # allow relative filenames to loading directory priorCWD = os.getcwd() os.chdir(os.path.dirname(oimFile)) else: priorCWD = None currentAction = "determining file type" isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json") isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json") isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls") isCSVorXL = isCSV or isXL instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl" if isJSON: currentAction = "loading and parsing JSON OIM file" def loadDict(keyValuePairs): _dict = OrderedDict() # preserve fact order in resulting instance for key, value in keyValuePairs: if isinstance(value, dict): if DUPJSONKEY in value: for _errKey, _errValue, _otherValue in value[DUPJSONKEY]: if key == "prefixes": modelXbrl.error("oime:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"), modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue) del value[DUPJSONKEY] if key in _dict: if DUPJSONKEY not in _dict: _dict[DUPJSONKEY] = [] _dict[DUPJSONKEY].append((key, value, _dict[key])) else: _dict[key] = value return _dict with io.open(oimFile, 'rt', encoding='utf-8') as f: oimObject = json.load(f, object_pairs_hook=loadDict) missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject] if missing: raise OIMException("oime:missingJSONElements", _("Required element(s) are missing from JSON input: %(missing)s"), missing = ", ".join(missing)) currentAction = "identifying JSON objects" dtsReferences = oimObject["dtsReferences"] prefixesList = oimObject["prefixes"].items() facts = oimObject["facts"] footnotes = oimObject["facts"] # shares this object elif isCSV: currentAction = "identifying CSV input tables" if sys.version[0] >= '3': csvOpenMode = 'w' csvOpenNewline = '' else: csvOpenMode = 'wb' # for 2.7 csvOpenNewline = None oimFileBase = None if "-facts" in oimFile: oimFileBase = oimFile.partition("-facts")[0] else: for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"): if oimFile.endswith(suffix): oimFileBase = oimFile[:-len(suffix)] break if oimFileBase is None: raise OIMException("oime:missingCSVTables", _("Unable to identify CSV tables file name pattern")) if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or not os.path.exists(oimFileBase + "-prefixes.csv")): raise OIMException("oime:missingCSVTables", _("Unable to identify CSV tables for dtsReferences or prefixes")) instanceFileName = oimFileBase + ".xbrl" currentAction = "loading CSV dtsReferences table" dtsReferences = [] with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: dtsReferences.append(dict((header[j], col) for j, col in enumerate(row))) currentAction = "loading CSV prefixes table" prefixesList = [] with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = dict((col,i) for i,col in enumerate(row)) else: prefixesList.append((row[header["prefix"]], row[header["URI"]])) defaults = {} if os.path.exists(oimFileBase + "-defaults.csv"): currentAction = "loading CSV defaults table" with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row fileCol = row.index("file") else: defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol) currentAction = "loading CSV facts tables" facts = [] _dir = os.path.dirname(oimFileBase) factsFileBasename = os.path.basename(oimFileBase) + "-facts" for filename in os.listdir(_dir): filepath = os.path.join(_dir, filename) if filename.startswith(factsFileBasename): currentAction = "loading CSV facts table {}".format(filename) tableDefaults = defaults.get(filename, {}) with io.open(filepath, 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col is not None: if header[j]: # skip cols with no header if header[j].endswith("Value"): if col: # ignore empty columns (= null CSV value) fact["value"] = col else: fact[header[j]] = col facts.append(fact) footnotes = [] if os.path.exists(oimFileBase + "-footnotes.csv"): currentAction = "loading CSV footnotes table" with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col)) elif isXL: currentAction = "identifying workbook input worksheets" oimWb = load_workbook(oimFile, read_only=True, data_only=True) sheetNames = oimWb.get_sheet_names() if (not any(sheetName == "prefixes" for sheetName in sheetNames) or not any(sheetName == "dtsReferences" for sheetName in sheetNames) or not any("facts" in sheetName for sheetName in sheetNames)): raise OIMException("oime:missingWorkbookWorksheets", _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts")) currentAction = "loading worksheet: dtsReferences" dtsReferences = [] for i, row in enumerate(oimWb["dtsReferences"]): if i == 0: header = [col.value for col in row] else: dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row))) currentAction = "loading worksheet: prefixes" prefixesList = [] for i, row in enumerate(oimWb["prefixes"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) else: prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value)) defaults = {} if "defaults" in sheetNames: currentAction = "loading worksheet: defaults" for i, row in enumerate(oimWb["defaults"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) fileCol = header["file"] else: defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol) facts = [] for sheetName in sheetNames: if sheetName == "facts" or "-facts" in sheetName: currentAction = "loading worksheet: {}".format(sheetName) tableDefaults = defaults.get(sheetName, {}) for i, row in enumerate(oimWb[sheetName]): if i == 0: header = [col.value for col in row] else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col.value is not None: if header[j]: # skip cols with no header if header[j].endswith("Value"): fact["value"] = str(col.value) else: fact[header[j]] = str(col.value) facts.append(fact) footnotes = [] if "footnotes" in sheetNames: currentAction = "loading worksheet: footnotes" for i, row in enumerate(oimWb["footnotes"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row) if col.value) else: footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value)) currentAction = "identifying default dimensions" ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults currentAction = "validating OIM" prefixes = {} prefixedUris = {} for _prefix, _uri in prefixesList: if not _prefix: modelXbrl.error("oime:emptyPrefix", _("The empty string must not be used as a prefix, uri %(uri)s"), modelObject=modelXbrl, uri=_uri) elif not NCNamePattern.match(_prefix): modelXbrl.error("oime:prefixPattern", _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"), modelObject=modelXbrl, prefix=_prefix, uri=_uri) elif _prefix in prefixes: modelXbrl.error("oime:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"), modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri) elif _uri in prefixedUris: modelXbrl.error("oime:duplicatedUri", _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"), modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix) else: prefixes[_prefix] = _uri prefixedUris[_uri] = _prefix oimPrefix = None for _nsOim in nsOim: if _nsOim in prefixedUris: oimPrefix = prefixedUris[_nsOim] if not oimPrefix: raise OIMException("oime:noOimPrefix", _("The oim namespace must have a declared prefix")) oimConcept = "{}:concept".format(oimPrefix) oimEntity = "{}:entity".format(oimPrefix) oimPeriod = "{}:period".format(oimPrefix) oimPeriodStart = "{}:periodStart".format(oimPrefix) oimPeriodEnd = "{}:periodEnd".format(oimPrefix) oimUnit = "{}:unit".format(oimPrefix) oimPrefix = "{}:".format(oimPrefix) # create the instance document currentAction = "creating instance document" modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument( modelXbrl, Type.INSTANCE, instanceFileName, schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"], isEntry=True, initialComment="extracted from OIM {}".format(mappedUri), documentEncoding="utf-8") cntxTbl = {} unitTbl = {} currentAction = "creating facts" for fact in facts: conceptQn = qname(fact[oimConcept], prefixes) concept = modelXbrl.qnameConcepts.get(conceptQn) entityAsQn = qname(fact[oimEntity], prefixes) if oimPeriod in fact: periodStart = fact[oimPeriod]["start"] periodEnd = fact[oimPeriod]["end"] else: periodStart = fact[oimPeriodStart] periodEnd = fact[oimPeriodEnd] cntxKey = ( # hashable context key ("periodType", concept.periodType), ("entity", entityAsQn), ("periodStart", periodStart), ("periodEnd", periodEnd)) + tuple(sorted( (dimName, dimVal) for dimName, dimVal in fact.items() if ":" in dimName and not dimName.startswith(oimPrefix))) if cntxKey in cntxTbl: _cntx = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) qnameDims = {} for dimName, dimVal in fact.items(): if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal: dimQname = qname(dimName, prefixes) dimConcept = modelXbrl.qnameConcepts.get(dimQname) if ":" in dimVal and dimVal.partition(':')[0] in prefixes: mem = qname(dimVal, prefixes) # explicit dim elif dimConcept.isTypedDimension: # a modelObject xml element is needed for all of the instance functions to manage the typed dim mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment") _cntx = modelXbrl.createContext( entityAsQn.namespaceURI, entityAsQn.localName, concept.periodType, None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME), dateTime(periodEnd, type=DATETIME), None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) cntxTbl[cntxKey] = _cntx if oimUnit in fact: unitKey = fact[oimUnit] if unitKey in unitTbl: _unit = unitTbl[unitKey] else: _unit = None # validate unit unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey) if not UnitPattern.match(unitKeySub): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation is lexically invalid, %(unit)s"), modelObject=modelXbrl, unit=unitKey) else: _mul, _sep, _div = unitKey.partition('/') if _mul.startswith('('): _mul = _mul[1:-1] _muls = [u for u in _mul.split('*') if u] if _div.startswith('('): _div = _div[1:-1] _divs = [u for u in _div.split('*') if u] if _muls != sorted(_muls) or _divs != sorted(_divs): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation measures are not in alphabetical order, %(unit)s"), modelObject=modelXbrl, unit=unitKey) try: mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _muls] divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _divs] unitId = 'u-{:02}'.format(len(unitTbl) + 1) for _measures in mulQns, divQns: for _measure in _measures: addQnameValue(modelXbrl.modelDocument, _measure) _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId) except OIMException as ex: modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) unitTbl[unitKey] = _unit else: _unit = None attrs = {"contextRef": _cntx.id} if fact.get("value") is None: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact["value"] if fact.get("id"): attrs["id"] = fact["id"] if concept.isNumeric: if _unit is None: continue # skip creating fact because unit was invalid attrs["unitRef"] = _unit.id if "accuracy" in fact: attrs["decimals"] = fact["accuracy"] # is value a QName? if concept.baseXbrliType == "QName": addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes)) f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text) currentAction = "creating footnotes" footnoteLinks = {} # ELR elements factLocs = {} # index by (linkrole, factId) footnoteNbr = 0 locNbr = 0 for factOrFootnote in footnotes: if "factId" in factOrFootnote: factId = factOrFootnote["factId"] factFootnotes = (factOrFootnote,) # CSV or XL elif "id" in factOrFootnote and "footnotes" in factOrFootnote: factId = factOrFootnote["id"] factFootnotes = factOrFootnote["footnotes"] else: factFootnotes = () for footnote in factFootnotes: linkrole = footnote.get("group") arcrole = footnote.get("footnoteType") if not factId or not linkrole or not arcrole or not ( footnote.get("factRef") or footnote.get("footnote")): # invalid footnote continue if linkrole not in footnoteLinks: footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, XbrlConst.qnLinkFootnoteLink, attributes={"{http://www.w3.org/1999/xlink}type": "extended", "{http://www.w3.org/1999/xlink}role": linkrole}) footnoteLink = footnoteLinks[linkrole] if (linkrole, factId) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factId)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factId, XLINKLABEL: locLabel}) locLabel = factLocs[(linkrole, factId)] if footnote.get("footnote"): footnoteNbr += 1 footnoteLabel = "f_{:02}".format(footnoteNbr) attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel} if footnote.get("language"): attrs[XMLLANG] = footnote["language"] # note, for HTML will need to build an element structure addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"]) elif footnote.get("factRef"): factRef = footnote.get("factRef") if (linkrole, factRef) not in factLocs: locNbr += 1 locLabel = "f_{:02}".format(footnoteNbr) factLoc[(linkrole, factRef)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factRef, XLINKLABEL: locLabel}) footnoteLabel = factLoc[(linkrole, factId)] footnoteArc = addChild(footnoteLink, XbrlConst.qnLinkFootnoteArc, attributes={XLINKTYPE: "arc", XLINKARCROLE: arcrole, XLINKFROM: locLabel, XLINKTO: footnoteLabel}) currentAction = "done loading facts and footnotes" #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") except Exception as ex: if isinstance(ex, OIMException): modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) else: modelXbrl.error("arelleOIMloader:error", "Error while %(action)s, error %(error)s\ntraceback %(traceback)s", modelObject=modelXbrl, action=currentAction, error=ex, traceback=traceback.format_tb(sys.exc_info()[2])) if priorCWD: os.chdir(priorCWD) # restore prior current working directory startingErrorCount = len(modelXbrl.errors) if startingErrorCount < len(modelXbrl.errors): # had errors, don't allow ModelDocument.load to continue return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors") return getattr(modelXbrl, "modelDocument", None) # none if returning from exception
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri): from openpyxl import load_workbook from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname try: currentAction = "initializing" startingErrorCount = len(modelXbrl.errors) startedAt = time.time() if os.path.isabs(oimFile): # allow relative filenames to loading directory priorCWD = os.getcwd() os.chdir(os.path.dirname(oimFile)) else: priorCWD = None currentAction = "determining file type" isJSON = oimFile.endswith( ".json") and not oimFile.endswith("-metadata.json") isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json") isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls") isCSVorXL = isCSV or isXL instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl" if isJSON: currentAction = "loading and parsing JSON OIM file" def loadDict(keyValuePairs): _dict = OrderedDict( ) # preserve fact order in resulting instance for key, value in keyValuePairs: if isinstance(value, dict): if DUPJSONKEY in value: for _errKey, _errValue, _otherValue in value[ DUPJSONKEY]: if key == "prefixes": modelXbrl.error( "oime:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s" ), modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue) del value[DUPJSONKEY] if key in _dict: if DUPJSONKEY not in _dict: _dict[DUPJSONKEY] = [] _dict[DUPJSONKEY].append((key, value, _dict[key])) else: _dict[key] = value return _dict with io.open(oimFile, 'rt', encoding='utf-8') as f: oimObject = json.load(f, object_pairs_hook=loadDict) missing = [ t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject ] if missing: raise OIMException( "oime:missingJSONElements", _("Required element(s) are missing from JSON input: %(missing)s" ), missing=", ".join(missing)) currentAction = "identifying JSON objects" dtsReferences = oimObject["dtsReferences"] prefixesList = oimObject["prefixes"].items() facts = oimObject["facts"] footnotes = oimObject["facts"] # shares this object elif isCSV: currentAction = "identifying CSV input tables" if sys.version[0] >= '3': csvOpenMode = 'w' csvOpenNewline = '' else: csvOpenMode = 'wb' # for 2.7 csvOpenNewline = None oimFileBase = None if "-facts" in oimFile: oimFileBase = oimFile.partition("-facts")[0] else: for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"): if oimFile.endswith(suffix): oimFileBase = oimFile[:-len(suffix)] break if oimFileBase is None: raise OIMException( "oime:missingCSVTables", _("Unable to identify CSV tables file name pattern")) if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or not os.path.exists(oimFileBase + "-prefixes.csv")): raise OIMException( "oime:missingCSVTables", _("Unable to identify CSV tables for dtsReferences or prefixes" )) instanceFileName = oimFileBase + ".xbrl" currentAction = "loading CSV dtsReferences table" dtsReferences = [] with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: dtsReferences.append( dict( (header[j], col) for j, col in enumerate(row))) currentAction = "loading CSV prefixes table" prefixesList = [] with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = dict((col, i) for i, col in enumerate(row)) else: prefixesList.append( (row[header["prefix"]], row[header["URI"]])) defaults = {} if os.path.exists(oimFileBase + "-defaults.csv"): currentAction = "loading CSV defaults table" with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row fileCol = row.index("file") else: defaults[row[fileCol]] = dict( (header[j], col) for j, col in enumerate(row) if j != fileCol) currentAction = "loading CSV facts tables" facts = [] _dir = os.path.dirname(oimFileBase) factsFileBasename = os.path.basename(oimFileBase) + "-facts" for filename in os.listdir(_dir): filepath = os.path.join(_dir, filename) if filename.startswith(factsFileBasename): currentAction = "loading CSV facts table {}".format( filename) tableDefaults = defaults.get(filename, {}) with io.open(filepath, 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col is not None: if header[ j]: # skip cols with no header if header[j].endswith("Value"): if col: # ignore empty columns (= null CSV value) fact["value"] = col else: fact[header[j]] = col facts.append(fact) footnotes = [] if os.path.exists(oimFileBase + "-footnotes.csv"): currentAction = "loading CSV footnotes table" with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: footnotes.append( dict((header[j], col) for j, col in enumerate(row) if col)) elif isXL: currentAction = "identifying workbook input worksheets" oimWb = load_workbook(oimFile, read_only=True, data_only=True) sheetNames = oimWb.get_sheet_names() if (not any(sheetName == "prefixes" for sheetName in sheetNames) or not any(sheetName == "dtsReferences" for sheetName in sheetNames) or not any("facts" in sheetName for sheetName in sheetNames)): raise OIMException( "oime:missingWorkbookWorksheets", _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts" )) currentAction = "loading worksheet: dtsReferences" dtsReferences = [] for i, row in enumerate(oimWb["dtsReferences"]): if i == 0: header = [col.value for col in row] else: dtsReferences.append( dict((header[j], col.value) for j, col in enumerate(row))) currentAction = "loading worksheet: prefixes" prefixesList = [] for i, row in enumerate(oimWb["prefixes"]): if i == 0: header = dict((col.value, i) for i, col in enumerate(row)) else: prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value)) defaults = {} if "defaults" in sheetNames: currentAction = "loading worksheet: defaults" for i, row in enumerate(oimWb["defaults"]): if i == 0: header = dict( (col.value, i) for i, col in enumerate(row)) fileCol = header["file"] else: defaults[row[fileCol].value] = dict( (header[j], col.value) for j, col in enumerate(row) if j != fileCol) facts = [] for sheetName in sheetNames: if sheetName == "facts" or "-facts" in sheetName: currentAction = "loading worksheet: {}".format(sheetName) tableDefaults = defaults.get(sheetName, {}) for i, row in enumerate(oimWb[sheetName]): if i == 0: header = [col.value for col in row] else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col.value is not None: if header[j]: # skip cols with no header if header[j].endswith("Value"): fact["value"] = str(col.value) else: fact[header[j]] = str(col.value) facts.append(fact) footnotes = [] if "footnotes" in sheetNames: currentAction = "loading worksheet: footnotes" for i, row in enumerate(oimWb["footnotes"]): if i == 0: header = dict((col.value, i) for i, col in enumerate(row) if col.value) else: footnotes.append( dict((header[j], col.value) for j, col in enumerate(row) if col.value)) currentAction = "identifying default dimensions" ValidateXbrlDimensions.loadDimensionDefaults( modelXbrl) # needs dimension defaults currentAction = "validating OIM" prefixes = {} prefixedUris = {} for _prefix, _uri in prefixesList: if not _prefix: modelXbrl.error( "oime:emptyPrefix", _("The empty string must not be used as a prefix, uri %(uri)s" ), modelObject=modelXbrl, uri=_uri) elif not NCNamePattern.match(_prefix): modelXbrl.error( "oime:prefixPattern", _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s" ), modelObject=modelXbrl, prefix=_prefix, uri=_uri) elif _prefix in prefixes: modelXbrl.error( "oime:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s" ), modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri) elif _uri in prefixedUris: modelXbrl.error( "oime:duplicatedUri", _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s" ), modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix) else: prefixes[_prefix] = _uri prefixedUris[_uri] = _prefix oimPrefix = None for _nsOim in nsOim: if _nsOim in prefixedUris: oimPrefix = prefixedUris[_nsOim] if not oimPrefix: raise OIMException( "oime:noOimPrefix", _("The oim namespace must have a declared prefix")) oimConcept = "{}:concept".format(oimPrefix) oimEntity = "{}:entity".format(oimPrefix) oimPeriod = "{}:period".format(oimPrefix) oimPeriodStart = "{}:periodStart".format(oimPrefix) oimPeriodEnd = "{}:periodEnd".format(oimPrefix) oimUnit = "{}:unit".format(oimPrefix) oimPrefix = "{}:".format(oimPrefix) # create the instance document currentAction = "creating instance document" modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument( modelXbrl, Type.INSTANCE, instanceFileName, schemaRefs=[ dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema" ], isEntry=True, initialComment="extracted from OIM {}".format(mappedUri), documentEncoding="utf-8") cntxTbl = {} unitTbl = {} currentAction = "creating facts" for fact in facts: conceptQn = qname(fact[oimConcept], prefixes) concept = modelXbrl.qnameConcepts.get(conceptQn) entityAsQn = qname(fact[oimEntity], prefixes) if oimPeriod in fact: periodStart = fact[oimPeriod]["start"] periodEnd = fact[oimPeriod]["end"] else: periodStart = fact[oimPeriodStart] periodEnd = fact[oimPeriodEnd] cntxKey = ( # hashable context key ("periodType", concept.periodType), ("entity", entityAsQn), ("periodStart", periodStart), ("periodEnd", periodEnd)) + tuple( sorted( (dimName, dimVal) for dimName, dimVal in fact.items() if ":" in dimName and not dimName.startswith(oimPrefix))) if cntxKey in cntxTbl: _cntx = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) qnameDims = {} for dimName, dimVal in fact.items(): if ":" in dimName and not dimName.startswith( oimPrefix) and dimVal: dimQname = qname(dimName, prefixes) dimConcept = modelXbrl.qnameConcepts.get(dimQname) if ":" in dimVal and dimVal.partition( ':')[0] in prefixes: mem = qname(dimVal, prefixes) # explicit dim elif dimConcept.isTypedDimension: # a modelObject xml element is needed for all of the instance functions to manage the typed dim mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False) qnameDims[dimQname] = DimValuePrototype( modelXbrl, None, dimQname, mem, "segment") _cntx = modelXbrl.createContext( entityAsQn.namespaceURI, entityAsQn.localName, concept.periodType, None if concept.periodType == "instant" else dateTime( periodStart, type=DATETIME), dateTime(periodEnd, type=DATETIME), None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) cntxTbl[cntxKey] = _cntx if oimUnit in fact: unitKey = fact[oimUnit] if unitKey in unitTbl: _unit = unitTbl[unitKey] else: _unit = None # validate unit unitKeySub = PrefixedQName.sub( UnitPrefixedQNameSubstitutionChar, unitKey) if not UnitPattern.match(unitKeySub): modelXbrl.error( "oime:unitStringRepresentation", _("Unit string representation is lexically invalid, %(unit)s" ), modelObject=modelXbrl, unit=unitKey) else: _mul, _sep, _div = unitKey.partition('/') if _mul.startswith('('): _mul = _mul[1:-1] _muls = [u for u in _mul.split('*') if u] if _div.startswith('('): _div = _div[1:-1] _divs = [u for u in _div.split('*') if u] if _muls != sorted(_muls) or _divs != sorted(_divs): modelXbrl.error( "oime:unitStringRepresentation", _("Unit string representation measures are not in alphabetical order, %(unit)s" ), modelObject=modelXbrl, unit=unitKey) try: mulQns = [ qname( u, prefixes, prefixException=OIMException( "oime:unitPrefix", _("Unit prefix is not declared: %(unit)s" ), unit=u)) for u in _muls ] divQns = [ qname( u, prefixes, prefixException=OIMException( "oime:unitPrefix", _("Unit prefix is not declared: %(unit)s" ), unit=u)) for u in _divs ] unitId = 'u-{:02}'.format(len(unitTbl) + 1) for _measures in mulQns, divQns: for _measure in _measures: addQnameValue(modelXbrl.modelDocument, _measure) _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId) except OIMException as ex: modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) unitTbl[unitKey] = _unit else: _unit = None attrs = {"contextRef": _cntx.id} if fact.get("value") is None: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact["value"] if fact.get("id"): attrs["id"] = fact["id"] if concept.isNumeric: if _unit is None: continue # skip creating fact because unit was invalid attrs["unitRef"] = _unit.id if "accuracy" in fact: attrs["decimals"] = fact["accuracy"] # is value a QName? if concept.baseXbrliType == "QName": addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes)) f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text) currentAction = "creating footnotes" footnoteLinks = {} # ELR elements factLocs = {} # index by (linkrole, factId) footnoteNbr = 0 locNbr = 0 for factOrFootnote in footnotes: if "factId" in factOrFootnote: factId = factOrFootnote["factId"] factFootnotes = (factOrFootnote, ) # CSV or XL elif "id" in factOrFootnote and "footnotes" in factOrFootnote: factId = factOrFootnote["id"] factFootnotes = factOrFootnote["footnotes"] else: factFootnotes = () for footnote in factFootnotes: linkrole = footnote.get("group") arcrole = footnote.get("footnoteType") if not factId or not linkrole or not arcrole or not ( footnote.get("factRef") or footnote.get("footnote")): # invalid footnote continue if linkrole not in footnoteLinks: footnoteLinks[linkrole] = addChild( modelXbrl.modelDocument.xmlRootElement, XbrlConst.qnLinkFootnoteLink, attributes={ "{http://www.w3.org/1999/xlink}type": "extended", "{http://www.w3.org/1999/xlink}role": linkrole }) footnoteLink = footnoteLinks[linkrole] if (linkrole, factId) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factId)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={ XLINKTYPE: "locator", XLINKHREF: "#" + factId, XLINKLABEL: locLabel }) locLabel = factLocs[(linkrole, factId)] if footnote.get("footnote"): footnoteNbr += 1 footnoteLabel = "f_{:02}".format(footnoteNbr) attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel} if footnote.get("language"): attrs[XMLLANG] = footnote["language"] # note, for HTML will need to build an element structure addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"]) elif footnote.get("factRef"): factRef = footnote.get("factRef") if (linkrole, factRef) not in factLocs: locNbr += 1 locLabel = "f_{:02}".format(footnoteNbr) factLoc[(linkrole, factRef)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={ XLINKTYPE: "locator", XLINKHREF: "#" + factRef, XLINKLABEL: locLabel }) footnoteLabel = factLoc[(linkrole, factId)] footnoteArc = addChild(footnoteLink, XbrlConst.qnLinkFootnoteArc, attributes={ XLINKTYPE: "arc", XLINKARCROLE: arcrole, XLINKFROM: locLabel, XLINKTO: footnoteLabel }) currentAction = "done loading facts and footnotes" #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") except Exception as ex: if isinstance(ex, OIMException): modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) else: modelXbrl.error( "arelleOIMloader:error", "Error while %(action)s, error %(error)s\ntraceback %(traceback)s", modelObject=modelXbrl, action=currentAction, error=ex, traceback=traceback.format_tb(sys.exc_info()[2])) if priorCWD: os.chdir( priorCWD ) # restore prior current working directory startingErrorCount = len(modelXbrl.errors) if startingErrorCount < len(modelXbrl.errors): # had errors, don't allow ModelDocument.load to continue return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors") return getattr(modelXbrl, "modelDocument", None) # none if returning from exception
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None, *args, **kwargs): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath) def addLocallyReferencedFile(elt,filingFiles): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrValue): attrValue = attrValue.partition('#')[0] # remove anchor if attrValue: # ignore anchor references to base document attrValue = os.path.normpath(attrValue) # change url path separators to host separators file = os.path.join(sourceDir,attrValue) if modelXbrl.fileSource.isInArchive(file, checkExistence=True) or os.path.exists(file): filingFiles.add(file) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl)) rootElt = modelXbrl.modelDocument.xmlRootElement # take baseXmlLang from <html> or <base> baseXmlLang = rootElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang") for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/1999/xhtml}body"): baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang") or baseXmlLang targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True, discover=False) # don't attempt to load DTS if baseXmlLang: targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang) ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in sorted(modelXbrl.contexts.values(), key=lambda c: elementChildSequence(c)): ignore = targetInstance.createContext(context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default xmlLang = fact.xmlLang if xmlLang is not None and xmlLang != baseXmlLang: attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) # if fact.isFraction, create numerator and denominator newFactForOldObjId[fact.objectIndex] = newFact if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references so that referenced files are included in the zip. for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter(): addLocallyReferencedFile(elt, filingFiles) except (XMLSyntaxError, UnicodeDecodeError): pass # TODO: Why ignore UnicodeDecodeError? elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) footnoteIdCount = {} for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format(attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) xmlLang = linkChild.xmlLang if xmlLang is not None and xmlLang != baseXmlLang: # default newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang) copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): addLocallyReferencedFile(elt,filingFiles) targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip) if getattr(modelXbrl, "isTestcaseVariation", False): modelXbrl.extractedInlineInstance = True # for validation comparison modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)