def lbTreeWalk(lbType, parentElt, lbList, roleRefs, locs=None, fromPrefix=None, fromName=None): order = 1.0 for toPrefix, toName, rel, list in lbList: if rel == "_ELR_": role = "unspecified" if toPrefix and toPrefix.startswith("http://"): # have a role specified role = toPrefix elif toName: #may be a definition for linkroleUri, modelRoleTypes in dts.roleTypes.items(): definition = modelRoleTypes[0].definition if toName == definition: role = linkroleUri break if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef roleType = modelRoleTypes[0] roleRef = ("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id) roleRefs.add(roleRef) linkElt = XmlUtil.addChild(parentElt, XbrlConst.link, lbType + "Link", attributes=(("{http://www.w3.org/1999/xlink}type", "extended"), ("{http://www.w3.org/1999/xlink}role", role))) locs = set() lbTreeWalk(lbType, linkElt, list, roleRefs, locs) else: toHref = extensionHref(toPrefix, toName) toLabel = toPrefix + "_" + toName if toHref not in locs: XmlUtil.addChild(parentElt, XbrlConst.link, "loc", attributes=(("{http://www.w3.org/1999/xlink}type", "locator"), ("{http://www.w3.org/1999/xlink}href", toHref), ("{http://www.w3.org/1999/xlink}label", toLabel))) locs.add(toHref) if rel != "_root_": fromLabel = fromPrefix + "_" + fromName if lbType == "calculation": otherAttrs = ( ("weight", list), ) else: otherAttrs = ( ) if rel == "_dimensions_": # pick proper consecutive arcrole fromConcept = hrefConcept(fromPrefix, fromName) toConcept = hrefConcept(toPrefix, toName) if toConcept is not None and toConcept.isHypercubeItem: rel = XbrlConst.all elif toConcept is not None and toConcept.isDimensionItem: rel = XbrlConst.hypercubeDimension elif fromConcept is not None and fromConcept.isDimensionItem: rel = XbrlConst.dimensionDomain else: rel = XbrlConst.domainMember XmlUtil.addChild(parentElt, XbrlConst.link, lbType + "Arc", attributes=(("{http://www.w3.org/1999/xlink}type", "arc"), ("{http://www.w3.org/1999/xlink}arcrole", rel), ("{http://www.w3.org/1999/xlink}from", fromLabel), ("{http://www.w3.org/1999/xlink}to", toLabel), ("order", order)) + otherAttrs ) order += 1.0 if lbType != "calculation" or rel == "_root_": lbTreeWalk(lbType, parentElt, list, roleRefs, locs, toPrefix, toName)
def createUnit(self, multiplyBy, divideBy, afterSibling=None, beforeSibling=None): """Creates new unit, by measures, as in formula usage, if any :param multiplyBy: List of multiply-by measure QNames (or top level measures if no divideBy) :type multiplyBy: [QName] :param divideBy: List of multiply-by measure QNames (or empty list if no divideBy) :type divideBy: [QName] :param beforeSibling: lxml element in instance to insert new concept before :type beforeSibling: ModelObject :param afterSibling: lxml element in instance to insert new concept after :type afterSibling: ModelObject :returns: ModelUnit -- New unit object """ xbrlElt = self.modelDocument.xmlRootElement if afterSibling == AUTO_LOCATE_ELEMENT: afterSibling = XmlUtil.lastChild(xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context", "unit")) unitId = 'u-{0:02n}'.format( len(self.units) + 1) newUnitElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId), afterSibling=afterSibling, beforeSibling=beforeSibling) if len(divideBy) == 0: for multiply in multiplyBy: XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) else: divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide") numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator") denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator") for multiply in multiplyBy: XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) for divide in divideBy: XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, divide)) self.modelDocument.unitDiscover(newUnitElt) XmlValidate.validate(self, newUnitElt) return newUnitElt
def createFact(self, conceptQname, attributes=None, text=None, parent=None, afterSibling=None, beforeSibling=None): if parent is None: parent = self.modelDocument.xmlRootElement newFact = XmlUtil.addChild(parent, conceptQname, attributes=attributes, text=text, afterSibling=afterSibling, beforeSibling=beforeSibling) self.modelDocument.factDiscover(newFact, parentElement=parent) XmlValidate.validate(self, newFact) return newFact
def addLinkbaseRef(lbType, lbFilename, lbDoc): role = "http://www.xbrl.org/2003/role/{0}LinkbaseRef".format(lbType) lbRefElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "linkbaseRef", attributes=(("{http://www.w3.org/1999/xlink}type", "simple"), ("{http://www.w3.org/1999/xlink}href", lbFilename), ("{http://www.w3.org/1999/xlink}role", role), ("{http://www.w3.org/1999/xlink}arcrole", "http://www.w3.org/1999/xlink/properties/linkbase"), )) dtsSchemaDocument.referencesDocument[lbDoc] = ModelDocumentReference("href", lbRefElt)
def createUnit(self, multiplyBy, divideBy, afterSibling=None, beforeSibling=None): xbrlElt = self.modelDocument.xmlRootElement if afterSibling == AUTO_LOCATE_ELEMENT: afterSibling = XmlUtil.lastChild( xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context", "unit")) unitId = 'u-{0:02n}'.format(len(self.units) + 1) newUnitElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId), afterSibling=afterSibling, beforeSibling=beforeSibling) if len(divideBy) == 0: for multiply in multiplyBy: XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) else: divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide") numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator") denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator") for multiply in multiplyBy: XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) for divide in divideBy: XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, divide)) self.modelDocument.unitDiscover(newUnitElt) XmlValidate.validate(self, newUnitElt) return newUnitElt
def createUnit(self, multiplyBy, divideBy, afterSibling=None, beforeSibling=None): xbrlElt = self.modelDocument.xmlRootElement if afterSibling == AUTO_LOCATE_ELEMENT: afterSibling = XmlUtil.lastChild(xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context", "unit")) unitId = 'u-{0:02n}'.format( len(self.units) + 1) newUnitElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId), afterSibling=afterSibling, beforeSibling=beforeSibling) if len(divideBy) == 0: for multiply in multiplyBy: XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) else: divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide") numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator") denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator") for multiply in multiplyBy: XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) for divide in divideBy: XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, divide)) self.modelDocument.unitDiscover(newUnitElt) XmlValidate.validate(self, newUnitElt) return newUnitElt
def createFact(self, conceptQname, attributes=None, text=None, parent=None, afterSibling=None, beforeSibling=None): """Creates new fact, as in formula output instance creation, and validates into object model :param conceptQname: QNames of concept :type conceptQname: QName :param attributes: Tuple of name, value, or tuples of name, value tuples (name,value) or ((name,value)[,(name,value...)]), where name is either QName or clark-notation name string :param text: Text content of fact :type text: str :param parent: lxml element in instance to append as child of :type parent: ModelObject :param beforeSibling: lxml element in instance to insert new concept before :type beforeSibling: ModelObject :param afterSibling: lxml element in instance to insert new concept after :type afterSibling: ModelObject :returns: ModelFact -- New fact object """ if parent is None: parent = self.modelDocument.xmlRootElement newFact = XmlUtil.addChild(parent, conceptQname, attributes=attributes, text=text, afterSibling=afterSibling, beforeSibling=beforeSibling) self.modelDocument.factDiscover(newFact, parentElement=parent) XmlValidate.validate(self, newFact) return newFact
def lbTreeWalk(lbType, parentElt, lbStruct, roleRefs, locs=None, arcsFromTo=None, fromPrefix=None, fromName=None): order = 1.0 for lbEntry in lbStruct: if lbEntry.isELR: role = "unspecified" if lbEntry.role and lbEntry.role.startswith("http://"): # have a role specified role = lbEntry.role elif lbEntry.name: # may be a definition for linkroleUri, modelRoleTypes in dts.roleTypes.items(): definition = modelRoleTypes[0].definition if lbEntry.name == definition: role = linkroleUri break if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef roleType = modelRoleTypes[0] roleRefs.add(("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id)) linkElt = XmlUtil.addChild( parentElt, XbrlConst.link, lbType + "Link", attributes=( ("{http://www.w3.org/1999/xlink}type", "extended"), ("{http://www.w3.org/1999/xlink}role", role), ), ) locs = set() arcsFromTo = set() lbTreeWalk(lbType, linkElt, lbEntry.childStruct, roleRefs, locs, arcsFromTo) else: toPrefix = lbEntry.prefix toName = lbEntry.name toHref = extensionHref(toPrefix, toName) toLabel = toPrefix + "_" + toName toLabelAlt = None if not lbEntry.isRoot: fromLabel = fromPrefix + "_" + fromName if (fromLabel, toLabel) in arcsFromTo: # need extra loc to prevent arc from/to duplication in ELR for i in range(1, 1000): toLabelAlt = "{}_{}".format(toLabel, i) if (fromLabel, toLabelAlt) not in arcsFromTo: toLabel = toLabelAlt break if toHref not in locs or toLabelAlt: XmlUtil.addChild( parentElt, XbrlConst.link, "loc", attributes=( ("{http://www.w3.org/1999/xlink}type", "locator"), ("{http://www.w3.org/1999/xlink}href", toHref), ("{http://www.w3.org/1999/xlink}label", toLabel), ), ) locs.add(toHref) if not lbEntry.isRoot: arcsFromTo.add((fromLabel, toLabel)) if lbType == "calculation" and lbEntry.weight is not None: otherAttrs = (("weight", lbEntry.weight),) elif lbType == "presentation" and lbEntry.role: otherAttrs = (("preferredLabel", lbEntry.role),) if lbEntry.role and lbEntry.role in dts.roleTypes: roleType = dts.roleTypes[lbEntry.role][0] roleRefs.add(("roleRef", lbEntry.role, roleType.modelDocument.uri + "#" + roleType.id)) else: otherAttrs = () if lbEntry.arcrole == "_dimensions_": # pick proper consecutive arcrole fromConcept = hrefConcept(fromPrefix, fromName) toConcept = hrefConcept(toPrefix, toName) if toConcept is not None and toConcept.isHypercubeItem: arcrole = XbrlConst.all otherAttrs += ((XbrlConst.qnXbrldtContextElement, "segment"),) elif toConcept is not None and toConcept.isDimensionItem: arcrole = XbrlConst.hypercubeDimension elif fromConcept is not None and fromConcept.isDimensionItem: arcrole = XbrlConst.dimensionDomain else: arcrole = XbrlConst.domainMember else: arcrole = lbEntry.arcrole XmlUtil.addChild( parentElt, XbrlConst.link, lbType + "Arc", attributes=( ("{http://www.w3.org/1999/xlink}type", "arc"), ("{http://www.w3.org/1999/xlink}arcrole", arcrole), ("{http://www.w3.org/1999/xlink}from", fromLabel), ("{http://www.w3.org/1999/xlink}to", toLabel), ("order", order), ) + otherAttrs, ) order += 1.0 if lbType != "calculation" or lbEntry.isRoot: lbTreeWalk(lbType, parentElt, lbEntry.childStruct, roleRefs, locs, arcsFromTo, toPrefix, toName)
def createContext(self, entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, priItem, dims, segOCCs, scenOCCs, afterSibling=None, beforeSibling=None): xbrlElt = self.modelDocument.xmlRootElement if afterSibling == AUTO_LOCATE_ELEMENT: afterSibling = XmlUtil.lastChild(xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context")) cntxId = 'c-{0:02n}'.format( len(self.contexts) + 1) newCntxElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId), afterSibling=afterSibling, beforeSibling=beforeSibling) entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity") XmlUtil.addChild(entityElt, XbrlConst.xbrli, "identifier", attributes=("scheme", entityIdentScheme), text=entityIdentValue) periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period") if periodType == "forever": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever") elif periodType == "instant": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "instant", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) elif periodType == "duration": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "startDate", text=XmlUtil.dateunionValue(periodStart)) XmlUtil.addChild(periodElt, XbrlConst.xbrli, "endDate", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) segmentElt = None scenarioElt = None from arelle.ModelInstanceObject import ModelDimensionValue if dims: # requires primary item to determin ambiguous concepts ''' in theory we have to check full set of dimensions for validity in source or any other context element, but for shortcut will see if each dimension is already reported in an unambiguous valid contextElement ''' from arelle.PrototypeInstanceObject import FactPrototype, ContextPrototype, DimValuePrototype fp = FactPrototype(self, priItem, dims.items()) # force trying a valid prototype's context Elements if not isFactDimensionallyValid(self, fp, setPrototypeContextElements=True): self.info("arelleLinfo", _("Create context for %(priItem)s, cannot determine valid context elements, no suitable hypercubes"), modelObject=self, priItem=priItem) fpDims = fp.context.qnameDims for dimQname in sorted(fpDims.keys()): dimValue = fpDims[dimQname] if isinstance(dimValue, DimValuePrototype): dimMemberQname = dimValue.memberQname # None if typed dimension contextEltName = dimValue.contextElement else: # qname for explicit or node for typed dimMemberQname = None contextEltName = None if contextEltName == "segment": if segmentElt is None: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") contextElt = segmentElt elif contextEltName == "scenario": if scenarioElt is None: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") contextElt = scenarioElt else: self.info("arelleLinfo", _("Create context, %(dimension)s, cannot determine context element, either no all relationship or validation issue"), modelObject=self, dimension=dimQname), continue dimConcept = self.qnameConcepts[dimQname] dimAttr = ("dimension", XmlUtil.addQnameValue(xbrlElt, dimConcept.qname)) if dimConcept.isTypedDimension: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:typedMember", attributes=dimAttr) if isinstance(dimValue, (ModelDimensionValue, DimValuePrototype)) and dimValue.isTyped: XmlUtil.copyNodes(dimElt, dimValue.typedMember) elif dimMemberQname: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:explicitMember", attributes=dimAttr, text=XmlUtil.addQnameValue(xbrlElt, dimMemberQname)) if segOCCs: if segmentElt is None: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") XmlUtil.copyNodes(segmentElt, segOCCs) if scenOCCs: if scenarioElt is None: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") XmlUtil.copyNodes(scenarioElt, scenOCCs) self.modelDocument.contextDiscover(newCntxElt) XmlValidate.validate(self, newCntxElt) return newCntxElt
def genFact(dts, concept, preferredLabel, arcrole, relationshipSet, level, visited, elrInfo): try: if concept is not None: if concept.isHypercubeItem: elrInfo["inCube"] = level elrInfo["dims"] = {} elrInfo["lineItems"] =False elrInfo.pop("instant", None) elrInfo.pop("duration", None) elif concept.isDimensionItem: elrInfo["currentDim"] = concept if concept.isTypedDimension: elrInfo["dims"][concept.qname] = (concept, concept.typedDomainElement) if concept.typedDomainElement.isNumeric: elrInfo["domainIter"] = 1 elif concept.name.endswith("Member") or concept.name.endswith("_member"): # don't generate entries for default dim (Domain) (for now) dimConcept = elrInfo["currentDim"] if dimConcept.qname not in elrInfo["dims"]: elrInfo["dims"][dimConcept.qname] = (dimConcept, concept) else: if concept.name.endswith("LineItems") or concept.name.endswith("_line_items"): elrInfo["lineItems"] = True elif ((not elrInfo["inCube"] or # before any hypercube elrInfo["lineItems"]) # in Cube and within Line Items and not concept.isAbstract): # or within line items # generate a fact sampVals = sampleDataValues[elrInfo.get("domainIter",1)] # use first entry if no domain iter if concept.periodType not in elrInfo: qnameDims = {} for _dimConcept, _domConcept in elrInfo["dims"].values(): if _dimConcept.isExplicitDimension: _memVal = _domConcept.qname else: if _domConcept.type is not None and not _domConcept.isNumeric: _memEltVal = genSampleValue(sampVals, _domConcept) else: _memEltVal = str(elrInfo["domainIter"]) _memVal = XmlUtil.addChild(dts.modelDocument.xmlRootElement, _domConcept.qname, text=_memEltVal, appendChild=False) _dimObj = DimValuePrototype(dts, None, _dimConcept.qname, _memVal, "segment") qnameDims[_dimConcept.qname] = _dimObj elrInfo[concept.periodType] = dts.createContext( dts.conceptSampleScheme or "http://www.treasury.gov", "entityId", concept.periodType, sampVals["periodStart"] if concept.periodType == "duration" else None, sampVals["periodEnd"], concept.qname, qnameDims, [], []) cntx = elrInfo[concept.periodType] cntxId = cntx.id if concept.isNumeric: if concept.isMonetary: unitMeasure = qname(XbrlConst.iso4217, "USD") unitMeasure.prefix = "iso4217" # want to save with a recommended prefix decimals = 2 elif concept.isShares: unitMeasure = XbrlConst.qnXbrliShares decimals = 0 else: unitMeasure = XbrlConst.qnXbrliPure decimals = 0 # check if utr unitId is specified utrUnitId = genSampleUtrUnitId(concept) if utrUnitId is not None: _utrEntries = dts.modelManager.disclosureSystem.utrItemTypeEntries[concept.type.name] if _utrEntries: for _utrEntry in _utrEntries.values(): if _utrEntry.unitId == utrUnitId and _utrEntry.isSimple: unitMeasure = qname(_utrEntry.nsUnit, _utrEntry.unitId) break prevUnit = dts.matchUnit([unitMeasure], []) if prevUnit is not None: unitId = prevUnit.id else: newUnit = dts.createUnit([unitMeasure], []) unitId = newUnit.id value = genSampleValue(sampVals, concept) attrs = [("contextRef", cntxId)] if concept.isNumeric: attrs.append(("unitRef", unitId)) attrs.append(("decimals", decimals)) value = Locale.atof(dts.locale, str(value), str.strip) newFact = dts.createFact(concept.qname, attributes=attrs, text=value) if concept not in visited: visited.add(concept) rels = relationshipSet.fromModelObject(concept) lenRels = len(rels) iRel = 0 iFirstLineItem = None while iRel <= lenRels: if iRel == lenRels: # check if cube needs re-iterating if iFirstLineItem is None or elrInfo.get("domainIter",0) >= 2: break reIterateCube = True # cube can re-iterate else: modelRel = rels[iRel] toConcept = modelRel.toModelObject reIterateCube = (toConcept.isHypercubeItem and # finished prior line items and hitting next table iFirstLineItem is not None and elrInfo["lineItems"] and 1 <= elrInfo.get("domainIter",0) < 2) if reIterateCube: # repeat typed dim container iRel = iFirstLineItem elrInfo["domainIter"] += 1 elrInfo.pop("instant", None) # want new contexts for next iteration elrInfo.pop("duration", None) isFirstLineItem = not elrInfo["lineItems"] genFact(dts, toConcept, modelRel.preferredLabel, arcrole, relationshipSet, level+1, visited, elrInfo) if isFirstLineItem and elrInfo["lineItems"] and elrInfo.get("domainIter",0) > 0: iFirstLineItem = iRel iRel += 1 visited.remove(concept) except AttributeError as ex: # bad relationship print ("[exception] {}".format(ex)) return
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( targetDocumentFilename, modelXbrl.modelDocument.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus( _("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults( targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in modelXbrl.contexts.values(): if context is not None: newCntx = targetInstance.createContext( context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): if unit is not None: measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) newFactForOldObjId[fact.objectIndex] = newFact if filingFiles and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML( "<body>\n{0}\n</body>\n".format(xmltext)): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src"): filingFiles.add(attrValue) except (XMLSyntaxError, UnicodeDecodeError): pass elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) # footnote links footnoteIdCount = {} modelXbrl.modelManager.showStatus( _("Creating and validating footnotes & relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any( lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = XmlUtil.addChild( targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance( linkChild, LocPrototype) and HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format( attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount XmlUtil.addChild(newLink, linkChild.qname, attributes=attributes, text=linkChild.textValue) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src"): filingFiles.add(attrValue) targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def createContext(self, entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, priItem, dims, segOCCs, scenOCCs, afterSibling=None, beforeSibling=None): xbrlElt = self.modelDocument.xmlRootElement if afterSibling == AUTO_LOCATE_ELEMENT: afterSibling = XmlUtil.lastChild( xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context")) cntxId = 'c-{0:02n}'.format(len(self.contexts) + 1) newCntxElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId), afterSibling=afterSibling, beforeSibling=beforeSibling) entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity") XmlUtil.addChild(entityElt, XbrlConst.xbrli, "identifier", attributes=("scheme", entityIdentScheme), text=entityIdentValue) periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period") if periodType == "forever": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever") elif periodType == "instant": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "instant", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) elif periodType == "duration": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "startDate", text=XmlUtil.dateunionValue(periodStart)) XmlUtil.addChild(periodElt, XbrlConst.xbrli, "endDate", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) segmentElt = None scenarioElt = None from arelle.ModelInstanceObject import ModelDimensionValue if dims: # requires primary item to determin ambiguous concepts ''' in theory we have to check full set of dimensions for validity in source or any other context element, but for shortcut will see if each dimension is already reported in an unambiguous valid contextElement ''' from arelle.PrototypeInstanceObject import FactPrototype, ContextPrototype, DimValuePrototype fp = FactPrototype(self, priItem, dims.items()) # force trying a valid prototype's context Elements if not isFactDimensionallyValid( self, fp, setPrototypeContextElements=True): self.info( "arelleLinfo", _("Create context for %(priItem)s, cannot determine valid context elements, no suitable hypercubes" ), modelObject=self, priItem=priItem) fpDims = fp.context.qnameDims for dimQname in sorted(fpDims.keys()): dimValue = fpDims[dimQname] if isinstance(dimValue, DimValuePrototype): dimMemberQname = dimValue.memberQname # None if typed dimension contextEltName = dimValue.contextElement else: # qname for explicit or node for typed dimMemberQname = None contextEltName = None if contextEltName == "segment": if segmentElt is None: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") contextElt = segmentElt elif contextEltName == "scenario": if scenarioElt is None: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") contextElt = scenarioElt else: self.info( "arelleLinfo", _("Create context, %(dimension)s, cannot determine context element, either no all relationship or validation issue" ), modelObject=self, dimension=dimQname), continue dimConcept = self.qnameConcepts[dimQname] dimAttr = ("dimension", XmlUtil.addQnameValue(xbrlElt, dimConcept.qname)) if dimConcept.isTypedDimension: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:typedMember", attributes=dimAttr) if isinstance(dimValue, (ModelDimensionValue, DimValuePrototype)) and dimValue.isTyped: XmlUtil.copyNodes(dimElt, dimValue.typedMember) elif dimMemberQname: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:explicitMember", attributes=dimAttr, text=XmlUtil.addQnameValue( xbrlElt, dimMemberQname)) if segOCCs: if segmentElt is None: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") XmlUtil.copyNodes(segmentElt, segOCCs) if scenOCCs: if scenarioElt is None: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") XmlUtil.copyNodes(scenarioElt, scenOCCs) self.modelDocument.contextDiscover(newCntxElt) XmlValidate.validate(self, newCntxElt) return newCntxElt
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( targetDocumentFilename, modelXbrl.modelDocument.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus( _("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults( targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in modelXbrl.contexts.values(): newCntx = targetInstance.createContext( context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in modelXbrl.facts: if fact.isItem: attrs = {"contextRef": fact.contextID} if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) newFactForOldObjId[fact.objectIndex] = newFact elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) # footnote links modelXbrl.modelManager.showStatus( _("Creating and validating footnotes & relationships")) for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: newLink = XmlUtil.addChild( targetInstance.modelDocument.xmlRootElement, linkqname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: if isinstance( linkChild, LocPrototype ) and "{http://www.w3.org/1999/xlink}href" not in linkChild.attributes: linkChild.attributes["{http://www.w3.org/1999/xlink}href"] = \ "#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) XmlUtil.addChild(newLink, linkChild.qname, attributes=linkChild.attributes, text=linkChild.textValue) targetInstance.saveInstance(overrideFilepath=targetUrl) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def genFact(dts, concept, preferredLabel, arcrole, relationshipSet, level, visited, elrInfo): try: if concept is not None: if concept.isHypercubeItem: elrInfo["inCube"] = level elrInfo["dims"] = {} elrInfo["lineItems"] =False elrInfo.pop("instant", None) elrInfo.pop("duration", None) elif concept.isDimensionItem: elrInfo["currentDim"] = concept if concept.isTypedDimension: elrInfo["dims"][concept.qname] = (concept, concept.typedDomainElement) if concept.typedDomainElement.isNumeric: elrInfo["domainIter"] = 1 elif concept.name.endswith("Member"): # don't generate entries for default dim (Domain) (for now) dimConcept = elrInfo["currentDim"] if dimConcept.qname not in elrInfo["dims"]: elrInfo["dims"][dimConcept.qname] = (dimConcept, concept) else: if concept.name.endswith("LineItems"): elrInfo["lineItems"] = True elif ((not elrInfo["inCube"] or # before any hypercube elrInfo["lineItems"]) # in Cube and within Line Items and not concept.isAbstract): # or within line items # generate a fact sampVals = sampleDataValues[elrInfo.get("domainIter",1)] # use first entry if no domain iter if concept.periodType not in elrInfo: qnameDims = {} for _dimConcept, _domConcept in elrInfo["dims"].values(): if _dimConcept.isExplicitDimension: _memVal = _domConcept.qname else: if _domConcept.type is not None and not _domConcept.isNumeric: _memEltVal = genSampleValue(sampVals, _domConcept) else: _memEltVal = str(elrInfo["domainIter"]) _memVal = XmlUtil.addChild(dts.modelDocument.xmlRootElement, _domConcept.qname, text=_memEltVal, appendChild=False) _dimObj = DimValuePrototype(dts, None, _dimConcept.qname, _memVal, "segment") qnameDims[_dimConcept.qname] = _dimObj elrInfo[concept.periodType] = dts.createContext( "http://treasury.gov", "entityId", concept.periodType, sampVals["periodStart"] if concept.periodType == "duration" else None, sampVals["periodEnd"], concept.qname, qnameDims, [], []) cntx = elrInfo[concept.periodType] cntxId = cntx.id if concept.isNumeric: if concept.isMonetary: unitMeasure = qname(XbrlConst.iso4217, "USD") unitMeasure.prefix = "iso4217" # want to save with a recommended prefix decimals = 2 elif concept.isShares: unitMeasure = XbrlConst.qnXbrliShares decimals = 0 else: unitMeasure = XbrlConst.qnXbrliPure decimals = 0 prevUnit = dts.matchUnit([unitMeasure], []) if prevUnit is not None: unitId = prevUnit.id else: newUnit = dts.createUnit([unitMeasure], []) unitId = newUnit.id value = genSampleValue(sampVals, concept) attrs = [("contextRef", cntxId)] if concept.isNumeric: attrs.append(("unitRef", unitId)) attrs.append(("decimals", decimals)) value = Locale.atof(dts.locale, str(value), str.strip) newFact = dts.createFact(concept.qname, attributes=attrs, text=value) if concept not in visited: visited.add(concept) rels = relationshipSet.fromModelObject(concept) lenRels = len(rels) iRel = 0 iFirstLineItem = None while iRel <= lenRels: if iRel == lenRels: # check if cube needs re-iterating if iFirstLineItem is None or elrInfo.get("domainIter",0) >= 2: break reIterateCube = True # cube can re-iterate else: modelRel = rels[iRel] toConcept = modelRel.toModelObject reIterateCube = (toConcept.isHypercubeItem and # finished prior line items and hitting next table iFirstLineItem is not None and elrInfo["lineItems"] and 1 <= elrInfo.get("domainIter",0) < 2) if reIterateCube: # repeat typed dim container iRel = iFirstLineItem elrInfo["domainIter"] += 1 elrInfo.pop("instant", None) # want new contexts for next iteration elrInfo.pop("duration", None) isFirstLineItem = not elrInfo["lineItems"] genFact(dts, toConcept, modelRel.preferredLabel, arcrole, relationshipSet, level+1, visited, elrInfo) if isFirstLineItem and elrInfo["lineItems"] and elrInfo.get("domainIter",0) > 0: iFirstLineItem = iRel iRel += 1 visited.remove(concept) except AttributeError as ex: # bad relationship print ("[exception] {}".format(ex)) return
def loadFromExcel(cntlr, excelFile): from openpyxl import load_workbook from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname startedAt = time.time() if os.path.isabs(excelFile): # allow relative filenames to loading directory priorCWD = os.getcwd() os.chdir(os.path.dirname(excelFile)) else: priorCWD = None importExcelBook = load_workbook(excelFile, read_only=True, data_only=True) sheetNames = importExcelBook.get_sheet_names() if "DTS" in sheetNames: dtsWs = importExcelBook["DTS"] elif "Sheet2" in sheetNames: dtsWs = importExcelBook["Sheet2"] else: dtsWs = None imports = {"xbrli": ( ("namespace", XbrlConst.xbrli), ("schemaLocation", "http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd") )} # xml of imports importXmlns = {} linkbaseRefs = [] labelLinkbases = [] hasPreLB = hasCalLB = hasDefLB = False # xxxLB structure [ (elr1, def1, "_ELR_", [roots]), (elr2, def2, "_ELR_", [rootw]) ...] # roots = (rootHref, None, "_root_", [children]) # children = (childPrefix, childName, arcrole, [grandChildren]) preLB = [] defLB = [] calLB = [] def lbDepthList(lbStruct, depth, parentList=None): if depth == topDepth: if len(lbStruct) > 0: return lbStruct[-1].childStruct else: cntlr.addToLog("Depth error, Excel row: {excelRow}" .format(excelRow=iRow), messageCode="importExcel:depth") return None return lbDepthList(lbStruct[-1].childStruct, depth-1, list) extensionElements = {} extensionRoles = {} # key is roleURI, value is role definition extensionLabels = {} # key = (prefix, name, lang, role), value = label text importSheetName = None skipRows = [] # [(from,to),(from,to)] row number starting at 1 def extensionHref(prefix, name): if prefix == extensionSchemaPrefix: filename = extensionSchemaFilename elif prefix in imports: filename = imports[prefix][1][1] else: return None return "{0}#{1}_{2}".format(filename, prefix, name) isUSGAAP = False for iRow, row in enumerate(dtsWs.rows if dtsWs else ()): try: if (len(row) < 1 or row[0].value is None): # skip if col 1 is empty continue action = filetype = prefix = filename = namespaceURI = None if len(row) > 0: action = row[0].value if len(row) > 1: filetype = row[1].value if len(row) > 2: prefix = row[2].value if len(row) > 3: filename = row[3].value if len(row) > 4: namespaceURI = row[4].value lbType = lang = None if action == "import": imports[prefix] = ( ("namespace", namespaceURI), ("schemaLocation", filename) ) importXmlns[prefix] = namespaceURI if re.match(r"http://[^/]+/us-gaap/", namespaceURI): isUSGAAP = True elif action == "extension": if filetype == "schema": extensionSchemaPrefix = prefix extensionSchemaFilename = filename extensionSchemaNamespaceURI = namespaceURI elif filetype == "linkbase": typeLang = prefix.split() if len(typeLang) > 0: lbType = typeLang[0] else: lbType = "unknown" if len(typeLang) > 1: lang = typeLang[1] else: lang = "en" if lbType == "label": labelLinkbases.append((lang, filename)) elif lbType == "presentation": hasPreLB = True elif lbType == "definition": hasDefLB = True elif lbType == "calculation": hasCalLB = True linkbaseRefs.append( (lbType, filename) ) elif filetype == "role" and namespaceURI: extensionRoles[namespaceURI] = filename elif action == "worksheet" and filename: importSheetName = filename elif action == "colheader" and filename and namespaceURI: importColHeaderMap[filename].append(namespaceURI) elif action == "skip rows": if filename: fromRow, _sep, toRow = filename.partition("-") try: skipRows.append((int(fromRow), int(toRow) if toRow else int(fromRow))) except (ValueError, TypeError): cntlr.addToLog("Exception: {error}, Excel row: {excelRow}" .format(error=err, excelRow=iRow), messageCode="importExcel:skip rows") except Exception as err: cntlr.addToLog("Exception: {error}, Excel row: {excelRow}" .format(error=err, excelRow=iRow), messageCode="importExcel:exception") if not isUSGAAP: # need extra namespace declaration importXmlns["iod"] = "http://disclosure.edinet-fsa.go.jp/taxonomy/common/2013-03-31/iod" # find column headers row headerCols = {} hasLinkroleSeparateRow = True headerRows = set() topDepth = 999999 if importSheetName and importSheetName in sheetNames: conceptsWs = importExcelBook[importSheetName] elif "Concepts" in sheetNames: conceptsWs = importExcelBook["Concepts"] elif "Sheet1" in sheetNames: conceptsWs = importExcelBook["Sheet1"] else: conceptsWs = None def setHeaderCols(row): headerCols.clear() for iCol, colCell in enumerate(row): v = colCell.value if v in importColHeaderMap: for hdr in importColHeaderMap[v]: if hdr in importColumnHeaders: headerCols[importColumnHeaders[hdr]] = iCol elif v in importColumnHeaders: headerCols[importColumnHeaders[v]] = iCol # find out which rows are header rows for iRow, row in enumerate(conceptsWs.rows if conceptsWs else ()): if any(fromRow <= iRow+1 <= toRow for fromRow,toRow in skipRows): continue for iCol, colCell in enumerate(row): setHeaderCols(row) if all(colName in headerCols for colName in ("name", "type", "depth")): # must have these to be a header col # it's a header col headerRows.add(iRow+1) if 'linkrole' in headerCols: hasLinkroleSeparateRow = False headerCols.clear() def cellHasValue(row, header, _type): if header in headerCols: iCol = headerCols[header] return iCol < len(row) and isinstance(row[iCol].value, _type) return False def cellValue(row, header, strip=False, nameChars=False): if header in headerCols: iCol = headerCols[header] if iCol < len(row): v = row[iCol].value if strip and isinstance(v, str): v = v.strip() if nameChars and isinstance(v, str): v = ''.join(c for c in v if c.isalnum() or c in ('.', '_', '-')) return v return None def checkImport(qname): prefix, sep, localName = qname.partition(":") if sep: if prefix not in imports: if prefix == "xbrldt": imports["xbrldt"] = ("namespace", XbrlConst.xbrldt), ("schemaLocation", "http://www.xbrl.org/2005/xbrldt-2005.xsd") elif prefix == "nonnum": imports["nonnum"] = ("namespace", "http://www.xbrl.org/dtr/type/non-numeric"), ("schemaLocation", "http://www.xbrl.org/dtr/type/nonNumeric-2009-12-16.xsd") else: cntlr.addToLog("Warning: prefix schema file is not imported for: {qname}" .format(qname=qname), messageCode="importExcel:warning") # find top depth for iRow, row in enumerate(conceptsWs.rows if conceptsWs else ()): if (iRow + 1) in headerRows: setHeaderCols(row) elif not (hasLinkroleSeparateRow and (iRow + 1) in headerRows) and 'depth' in headerCols: depth = cellValue(row, 'depth') if isinstance(depth, int) and depth < topDepth: topDepth = depth # find header rows currentELR = currentELRdefinition = None for iRow, row in enumerate(conceptsWs.rows if conceptsWs else ()): useLabels = False if any(fromRow <= iRow+1 <= toRow for fromRow,toRow in skipRows): continue if (all(col is None for col in row) or all(isinstance(row[i].value, str) and row[i].value.strip() == "n/a" for i in (headerCols.get("name"), headerCols.get("type"), headerCols.get("value")) if i)): continue # skip blank row try: isHeaderRow = (iRow + 1) in headerRows isELRrow = hasLinkroleSeparateRow and (iRow + 2) in headerRows if isHeaderRow: setHeaderCols(row) elif isELRrow: currentELR = currentELRdefinition = None for colCell in row: v = str(colCell.value or '') if v.startswith("http://"): currentELR = v elif not currentELRdefinition and v.endswith(" 科目一覧"): currentELRdefinition = v[0:-5] elif not currentELRdefinition: currentELRdefinition = v if currentELR or currentELRdefinition: if hasPreLB: preLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) ) if hasDefLB: defLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) ) if hasCalLB: calLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) ) calRels = set() # prevent duplications when same rel in different parts of tree elif headerCols: if "linkrole" in headerCols and cellHasValue(row, 'linkrole', str): v = cellValue(row, 'linkrole', strip=True) _trialELR = _trialELRdefinition = None if v.startswith("http://"): _trialELR = v elif v.endswith(" 科目一覧"): _trialELRdefinition = v[0:-5] else: _trialELRdefinition = v if (_trialELR and _trialELR != currentELR) or (_trialELRdefinition and _trialELRdefinition != currentELRdefinition): currentELR = _trialELR currentELRdefinition = _trialELRdefinition if currentELR or currentELRdefinition: if hasPreLB: preLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) ) if hasDefLB: defLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) ) if hasCalLB: calLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) ) calRels = set() # prevent duplications when same rel in different parts of tree prefix = cellValue(row, 'prefix', nameChars=True) or extensionSchemaPrefix if cellHasValue(row, 'name', str): name = cellValue(row, 'name', nameChars=True) else: name = None if cellHasValue(row, 'depth', int): depth = cellValue(row, 'depth') else: depth = None if (not prefix or prefix == extensionSchemaPrefix) and name not in extensionElements and name: # elements row eltType = cellValue(row, 'type') eltTypePrefix = cellValue(row, 'typePrefix') if not eltType: eltType = 'xbrli:stringItemType' elif eltTypePrefix and ':' not in eltType: eltType = eltTypePrefix + ':' + eltType elif ':' not in eltType and eltType.endswith("ItemType"): eltType = 'xbrli:' + eltType subsGrp = cellValue(row, 'substitutionGroup') or 'xbrli:item' abstract = cellValue(row, 'abstract') or (cellValue(row, 'abstractMarker') is not None) nillable = cellValue(row, 'nillable') balance = cellValue(row, 'balance') periodType = cellValue(row, 'periodType') newElt = [ ("name", name), ("id", (prefix or "") + "_" + name) ] if eltType: newElt.append( ("type", eltType) ) checkImport(eltType) if subsGrp: newElt.append( ("substitutionGroup", subsGrp) ) checkImport(subsGrp) if abstract or subsGrp in ("xbrldt:hypercubeItem", "xbrldt:dimensionItem"): newElt.append( ("abstract", abstract or "true") ) if nillable: newElt.append( ("nillable", nillable) ) if balance: newElt.append( ("{http://www.xbrl.org/2003/instance}balance", balance) ) if periodType: newElt.append( ("{http://www.xbrl.org/2003/instance}periodType", periodType) ) extensionElements[name] = newElt useLabels = True if depth is not None: if hasPreLB: entryList = lbDepthList(preLB, depth) preferredLabel = cellValue(row, 'preferredLabel') if preferredLabel and not preferredLabel.startswith("http://"): preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel if entryList is not None: if depth == topDepth: entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) ) else: entryList.append( LBentry(prefix=prefix, name=name, arcrole=XbrlConst.parentChild, role=preferredLabel) ) if hasDefLB: entryList = lbDepthList(defLB, depth) if entryList is not None: if depth == topDepth: entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) ) else: if (not preferredLabel or # prevent start/end labels from causing duplicate dim-mem relationships not any(lbEntry.prefix == prefix and lbEntry.name == name for lbEntry in entryList)): entryList.append( LBentry(prefix=prefix, name=name, arcrole="_dimensions_") ) if hasCalLB: calcParents = cellValue(row, 'calculationParent').split() calcWeights = (str(cellValue(row, 'calculationWeight')) or '').split() # may be float or string if calcParents and calcWeights: # may be multiple parents split by whitespace for i, calcParent in enumerate(calcParents): calcWeight = calcWeights[i] if i < len(calcWeights) else calcWeights[-1] calcParentPrefix, sep, calcParentName = calcParent.partition(":") entryList = lbDepthList(calLB, topDepth) if entryList is not None: calRel = (calcParentPrefix, calcParentName, prefix, name) if calRel not in calRels: entryList.append( LBentry(prefix=calcParentPrefix, name=calcParentName, isRoot=True, childStruct= [LBentry(prefix=prefix, name=name, arcrole=XbrlConst.summationItem, weight=calcWeight )]) ) calRels.add(calRel) else: pass # accumulate extension labels if useLabels: prefix = cellValue(row, 'prefix', nameChars=True) or extensionSchemaPrefix name = cellValue(row, 'name', nameChars=True) if name is not None: preferredLabel = cellValue(row, 'preferredLabel') if preferredLabel and not preferredLabel.startswith("http://"): preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel for colItem, iCol in headerCols.items(): if isinstance(colItem, tuple): colItemType = colItem[0] role = colItem[1] lang = colItem[2] cell = row[iCol] if cell.value is None: values = () elif colItemType == "label": values = (cell.value,) elif colItemType == "labels": values = cell.value.split('\n') else: values = () if preferredLabel and "indented" in colItem: # indented column sets preferredLabel if any role = preferredLabel for value in values: extensionLabels[prefix, name, lang, role] = value.strip() except Exception as err: cntlr.addToLog("Exception: {error}, Excel row: {excelRow}" .format(error=err, excelRow=iRow), messageCode="importExcel:exception") if isUSGAAP and hasDefLB: # move line items above table def fixUsggapTableDims(lvl1Struct): foundLineItems = False for i1, lvl1Entry in enumerate(lvl1Struct): for i2, lvl2Entry in enumerate(lvl1Entry.childStruct): for i3, lvl3Entry in enumerate(lvl2Entry.childStruct): if lvl3Entry.name.endswith("LineItems") and lvl2Entry.name.endswith("Table"): foundLineItems = True break if foundLineItems: break else: fixUsggapTableDims(lvl1Entry.childStruct) if foundLineItems: lvl1Struct.insert(i1 + 1, LBentry(prefix=lvl3Entry.prefix, name=lvl3Entry.name, arcrole=lvl1Entry.arcrole, childStruct=lvl3Entry.childStruct)) # must keep lvl1Rel if it is __root__ lvl3Entry.childStruct.insert(0, lvl2Entry) if lvl1Entry.name.endswith("Abstract"): del lvl1Struct[i1] if i3 < len(lvl2Entry.childStruct): del lvl2Entry.childStruct[i3] pass fixUsggapTableDims(defLB) dts = cntlr.modelManager.create(newDocumentType=ModelDocument.Type.SCHEMA, url=extensionSchemaFilename, isEntry=True, base='', # block pathname from becomming absolute initialXml=''' <schema xmlns="http://www.w3.org/2001/XMLSchema" targetNamespace="{targetNamespace}" attributeFormDefault="unqualified" elementFormDefault="qualified" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:{extensionPrefix}="{targetNamespace}" {importXmlns} xmlns:nonnum="http://www.xbrl.org/dtr/type/non-numeric" xmlns:link="http://www.xbrl.org/2003/linkbase" xmlns:xbrli="http://www.xbrl.org/2003/instance" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xbrldt="http://xbrl.org/2005/xbrldt"/> '''.format(targetNamespace=extensionSchemaNamespaceURI, extensionPrefix=extensionSchemaPrefix, importXmlns=''.join('xmlns:{0}="{1}"\n'.format(prefix, namespaceURI) for prefix, namespaceURI in importXmlns.items()) ) ) dtsSchemaDocument = dts.modelDocument dtsSchemaDocument.inDTS = True # entry document always in DTS dtsSchemaDocument.targetNamespace = extensionSchemaNamespaceURI # not set until schemaDiscover too late otherwise schemaElt = dtsSchemaDocument.xmlRootElement #foreach linkbase annotationElt = XmlUtil.addChild(schemaElt, XbrlConst.xsd, "annotation") appinfoElt = XmlUtil.addChild(annotationElt, XbrlConst.xsd, "appinfo") # add linkbaseRefs appinfoElt = XmlUtil.descendant(schemaElt, XbrlConst.xsd, "appinfo") # don't yet add linkbase refs, want to process imports first to get roleType definitions # add imports for importAttributes in sorted(imports.values()): XmlUtil.addChild(schemaElt, XbrlConst.xsd, "import", attributes=importAttributes) # add elements for eltName, eltAttrs in sorted(extensionElements.items(), key=lambda item: item[0]): XmlUtil.addChild(schemaElt, XbrlConst.xsd, "element", attributes=eltAttrs) # add role definitions (for discovery) for roleURI, roleDefinition in extensionRoles.items(): roleElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "roleType", attributes=(("roleURI", roleURI), ("id", "roleType_" + roleURI.rpartition("/")[2]))) if roleDefinition: XmlUtil.addChild(roleElt, XbrlConst.link, "definition", text=roleDefinition) if hasPreLB: XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:presentationLink") if hasDefLB: XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:definitionLink") if hasCalLB: XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:calculationLink") dtsSchemaDocument.schemaDiscover(schemaElt, False, extensionSchemaNamespaceURI) def addLinkbaseRef(lbType, lbFilename, lbDoc): role = "http://www.xbrl.org/2003/role/{0}LinkbaseRef".format(lbType) lbRefElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "linkbaseRef", attributes=(("{http://www.w3.org/1999/xlink}type", "simple"), ("{http://www.w3.org/1999/xlink}href", lbFilename), ("{http://www.w3.org/1999/xlink}role", role), ("{http://www.w3.org/1999/xlink}arcrole", "http://www.w3.org/1999/xlink/properties/linkbase"), )) dtsSchemaDocument.referencesDocument[lbDoc] = ModelDocumentReference("href", lbRefElt) # label linkbase for lang, filename in labelLinkbases: lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base="", initialXml=""" <linkbase xmlns="http://www.xbrl.org/2003/linkbase" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xbrli="http://www.xbrl.org/2003/instance"/> """) lbDoc.inDTS = True addLinkbaseRef("label", filename, lbDoc) lbElt = lbDoc.xmlRootElement linkElt = XmlUtil.addChild(lbElt, XbrlConst.link, "labelLink", attributes=(("{http://www.w3.org/1999/xlink}type", "extended"), ("{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/link"))) firstLinkElt = linkElt locs = set() roleRefs = set() for labelKey, text in extensionLabels.items(): prefix, name, labelLang, role = labelKey if lang == labelLang: locLabel = prefix + "_" + name if locLabel not in locs: locs.add(locLabel) XmlUtil.addChild(linkElt, XbrlConst.link, "loc", attributes=(("{http://www.w3.org/1999/xlink}type", "locator"), ("{http://www.w3.org/1999/xlink}href", extensionHref(prefix, name)), ("{http://www.w3.org/1999/xlink}label", locLabel))) XmlUtil.addChild(linkElt, XbrlConst.link, "labelArc", attributes=(("{http://www.w3.org/1999/xlink}type", "arc"), ("{http://www.w3.org/1999/xlink}arcrole", "http://www.xbrl.org/2003/arcrole/concept-label"), ("{http://www.w3.org/1999/xlink}from", locLabel), ("{http://www.w3.org/1999/xlink}to", "label_" + locLabel), ("order", 1.0))) XmlUtil.addChild(linkElt, XbrlConst.link, "label", attributes=(("{http://www.w3.org/1999/xlink}type", "resource"), ("{http://www.w3.org/1999/xlink}label", "label_" + locLabel), ("{http://www.w3.org/1999/xlink}role", role), ("{http://www.w3.org/XML/1998/namespace}lang", lang)), text=text) if role: if role in dts.roleTypes: roleType = dts.roleTypes[role][0] roleRefs.add(("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id)) elif role.startswith("http://www.xbrl.org/2009/role/negated"): roleRefs.add(("roleRef", role, "http://www.xbrl.org/lrr/role/negated-2009-12-16.xsd#" + role.rpartition("/")[2])) # add arcrole references for roleref, roleURI, href in roleRefs: XmlUtil.addChild(lbElt, XbrlConst.link, roleref, attributes=(("arcroleURI" if roleref == "arcroleRef" else "roleURI", roleURI), ("{http://www.w3.org/1999/xlink}type", "simple"), ("{http://www.w3.org/1999/xlink}href", href)), beforeSibling=firstLinkElt) lbDoc.linkbaseDiscover(lbElt) def hrefConcept(prefix, name): qn = schemaElt.prefixedNameQname(prefix + ":" + name) if qn in dts.qnameConcepts: return dts.qnameConcepts[qn] return None def lbTreeWalk(lbType, parentElt, lbStruct, roleRefs, locs=None, arcsFromTo=None, fromPrefix=None, fromName=None): order = 1.0 for lbEntry in lbStruct: if lbEntry.isELR: role = "unspecified" if lbEntry.role and lbEntry.role.startswith("http://"): # have a role specified role = lbEntry.role elif lbEntry.name: #may be a definition for linkroleUri, modelRoleTypes in dts.roleTypes.items(): definition = modelRoleTypes[0].definition if lbEntry.name == definition: role = linkroleUri break if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef roleType = modelRoleTypes[0] roleRefs.add(("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id)) linkElt = XmlUtil.addChild(parentElt, XbrlConst.link, lbType + "Link", attributes=(("{http://www.w3.org/1999/xlink}type", "extended"), ("{http://www.w3.org/1999/xlink}role", role))) locs = set() arcsFromTo = set() lbTreeWalk(lbType, linkElt, lbEntry.childStruct, roleRefs, locs, arcsFromTo) else: toPrefix = lbEntry.prefix toName = lbEntry.name toHref = extensionHref(toPrefix, toName) toLabel = toPrefix + "_" + toName toLabelAlt = None if not lbEntry.isRoot: fromLabel = fromPrefix + "_" + fromName if (fromLabel, toLabel) in arcsFromTo: # need extra loc to prevent arc from/to duplication in ELR for i in range(1, 1000): toLabelAlt = "{}_{}".format(toLabel, i) if (fromLabel, toLabelAlt) not in arcsFromTo: toLabel = toLabelAlt break if toHref not in locs or toLabelAlt: XmlUtil.addChild(parentElt, XbrlConst.link, "loc", attributes=(("{http://www.w3.org/1999/xlink}type", "locator"), ("{http://www.w3.org/1999/xlink}href", toHref), ("{http://www.w3.org/1999/xlink}label", toLabel))) locs.add(toHref) if not lbEntry.isRoot: arcsFromTo.add( (fromLabel, toLabel) ) if lbType == "calculation" and lbEntry.weight is not None: otherAttrs = ( ("weight", lbEntry.weight), ) elif lbType == "presentation" and lbEntry.role: otherAttrs = ( ("preferredLabel", lbEntry.role), ) if lbEntry.role and lbEntry.role in dts.roleTypes: roleType = dts.roleTypes[lbEntry.role][0] roleRefs.add(("roleRef", lbEntry.role, roleType.modelDocument.uri + "#" + roleType.id)) else: otherAttrs = ( ) if lbEntry.arcrole == "_dimensions_": # pick proper consecutive arcrole fromConcept = hrefConcept(fromPrefix, fromName) toConcept = hrefConcept(toPrefix, toName) if toConcept is not None and toConcept.isHypercubeItem: arcrole = XbrlConst.all otherAttrs += ( (XbrlConst.qnXbrldtContextElement, "segment"), ) elif toConcept is not None and toConcept.isDimensionItem: arcrole = XbrlConst.hypercubeDimension elif fromConcept is not None and fromConcept.isDimensionItem: arcrole = XbrlConst.dimensionDomain else: arcrole = XbrlConst.domainMember else: arcrole = lbEntry.arcrole XmlUtil.addChild(parentElt, XbrlConst.link, lbType + "Arc", attributes=(("{http://www.w3.org/1999/xlink}type", "arc"), ("{http://www.w3.org/1999/xlink}arcrole", arcrole), ("{http://www.w3.org/1999/xlink}from", fromLabel), ("{http://www.w3.org/1999/xlink}to", toLabel), ("order", order)) + otherAttrs ) order += 1.0 if lbType != "calculation" or lbEntry.isRoot: lbTreeWalk(lbType, parentElt, lbEntry.childStruct, roleRefs, locs, arcsFromTo, toPrefix, toName) for hasLB, lbType, lbLB in ((hasPreLB, "presentation", preLB), (hasDefLB, "definition", defLB), (hasCalLB, "calculation", calLB)): if hasLB: for lbRefType, filename in linkbaseRefs: if lbType == lbRefType: # output presentation linkbase lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base='', initialXml=""" <linkbase xmlns="http://www.xbrl.org/2003/linkbase" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xbrli="http://www.xbrl.org/2003/instance"/> """) lbDoc.inDTS = True addLinkbaseRef(lbRefType, filename, lbDoc) lbElt = lbDoc.xmlRootElement roleRefs = set() if lbType == "definition": roleRefs.update((("arcroleRef", XbrlConst.all, "http://www.xbrl.org/2005/xbrldt-2005.xsd#all"), ("arcroleRef", XbrlConst.dimensionDefault, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-default"), ("arcroleRef", XbrlConst.dimensionDomain, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-domain"), ("arcroleRef", XbrlConst.domainMember, "http://www.xbrl.org/2005/xbrldt-2005.xsd#domain-member"), ("arcroleRef", XbrlConst.hypercubeDimension, "http://www.xbrl.org/2005/xbrldt-2005.xsd#hypercube-dimension"))) lbTreeWalk(lbType, lbElt, lbLB, roleRefs) firstLinkElt = None for firstLinkElt in lbElt.iterchildren(): break # add arcrole references for roleref, roleURI, href in roleRefs: XmlUtil.addChild(lbElt, XbrlConst.link, roleref, attributes=(("arcroleURI" if roleref == "arcroleRef" else "roleURI", roleURI), ("{http://www.w3.org/1999/xlink}type", "simple"), ("{http://www.w3.org/1999/xlink}href", href)), beforeSibling=firstLinkElt) lbDoc.linkbaseDiscover(lbElt) break #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") if priorCWD: os.chdir(priorCWD) # restore prior current working directory return dts
def produceOutputFact(xpCtx, formula, result): priorErrorCount = len(xpCtx.modelXbrl.errors) # assemble context conceptQname = aspectValue(xpCtx, formula, Aspect.CONCEPT, "xbrlfe:missingConceptRule") if isinstance(conceptQname, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} concept: {1}").format( formula, conceptQname.msg), "err", conceptQname.err) modelConcept = None else: modelConcept = xpCtx.modelXbrl.qnameConcepts[conceptQname] if modelConcept is None or not modelConcept.isItem: xpCtx.modelXbrl.error( _("Formula {0} concept {1} is not an item").format( formula, conceptQname), "err", "xbrlfe:missingConceptRule") # entity entityIdentScheme = aspectValue(xpCtx, formula, Aspect.SCHEME, "xbrlfe:missingEntityIdentifierRule") if isinstance(entityIdentScheme, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} entity identifier scheme: {1}").format( formula, entityIdentScheme.msg ), "err", str(entityIdentScheme)) entityIdentValue = None else: entityIdentValue = aspectValue(xpCtx, formula, Aspect.VALUE, "xbrlfe:missingEntityIdentifierRule") if isinstance(entityIdentValue, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} entity identifier value: {1}").format( formula, entityIdentValue.msg ), "err", str(entityIdentScheme)) # period periodType = aspectValue(xpCtx, formula, Aspect.PERIOD_TYPE, "xbrlfe:missingPeriodRule") periodStart = None periodEndInstant = None if isinstance(periodType, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} period type: {1}").format( formula, periodType.msg ), "err", str(periodType)) elif periodType == "instant": periodEndInstant = aspectValue(xpCtx, formula, Aspect.INSTANT, "xbrlfe:missingPeriodRule") if isinstance(periodEndInstant, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} period start: {1}").format( formula, periodEndInstant.msg ), "err", str(periodEndInstant)) elif periodType == "duration": periodStart = aspectValue(xpCtx, formula, Aspect.START, "xbrlfe:missingPeriodRule") if isinstance(periodStart, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} period start: {1}").format( formula, periodStart.msg ), "err", str(periodStart)) periodEndInstant = aspectValue(xpCtx, formula, Aspect.END, "xbrlfe:missingPeriodRule") if isinstance(periodEndInstant, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} period end: {1}").format( formula, periodEndInstant.msg ), "err", str(periodEndInstant)) # unit if modelConcept and modelConcept.isNumeric: unitSource = aspectValue(xpCtx, formula, Aspect.UNIT_MEASURES, None) multDivBy = aspectValue(xpCtx, formula, Aspect.MULTIPLY_BY, "xbrlfe:missingUnitRule") if isinstance(multDivBy, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} unit: {1}").format( formula, multDivBy.msg ), "err", str(multDivBy) if isinstance(multDivBy, VariableBindingError) else "xbrlfe:missingUnitRule") multiplyBy = (); divideBy = () # prevent errors later if bad else: divMultBy = aspectValue(xpCtx, formula, Aspect.DIVIDE_BY, "xbrlfe:missingUnitRule") if isinstance(divMultBy, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} unit: {1}").format( formula, divMultBy.msg ), "err", str(multDivBy) if isinstance(divMultBy, VariableBindingError) else "xbrlfe:missingUnitRule") multiplyBy = (); divideBy = () # prevent errors later if bad else: multiplyBy = unitSource[0] + multDivBy[0] + divMultBy[1] divideBy = unitSource[1] + multDivBy[1] + divMultBy[0] # remove cancelling mult/div units lookForCommonUnits = True while lookForCommonUnits: lookForCommonUnits = False for commonUnit in multiplyBy: if commonUnit in divideBy: multiplyBy.remove(commonUnit) divideBy.remove(commonUnit) lookForCommonUnits = True break if len(multiplyBy) == 0: # if no units add pure multiplyBy.append(XbrlConst.qnXbrliPure) # dimensions segOCCs = [] scenOCCs = [] if formula.aspectModel == "dimensional": dimAspects = {} dimQnames = aspectValue(xpCtx, formula, Aspect.DIMENSIONS, None) if dimQnames: for dimQname in dimQnames: dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname] dimErr = "xbrlfe:missing{0}DimensionRule".format("typed" if dimConcept and dimConcept.isTypedDimension else "explicit") dimValue = aspectValue(xpCtx, formula, dimQname, dimErr) if isinstance(dimValue, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} dimension {1}: {2}").format( formula, dimQname, dimValue.msg ), "err", dimErr) elif dimValue and xpCtx.modelXbrl.qnameDimensionDefaults.get(dimQname) != dimValue: dimAspects[dimQname] = dimValue segOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SEGMENT, None) scenOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SCENARIO, None) else: dimAspects = None # non-dimensional segOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SEGMENT, None) scenOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SCENARIO, None) if priorErrorCount < len(xpCtx.modelXbrl.errors): return None # had errors, don't produce output fact # does context exist in out instance document outputInstanceQname = formula.outputInstanceQname outputXbrlInstance = xpCtx.inScopeVars[outputInstanceQname] xbrlElt = outputXbrlInstance.modelDocument.xmlRootElement # in source instance document # add context prevCntx = outputXbrlInstance.matchContext( entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, dimAspects, segOCCs, scenOCCs) if prevCntx: cntxId = prevCntx.id newCntxElt = prevCntx.element else: cntxId = 'c-{0:02n}'.format( len(outputXbrlInstance.contexts) + 1) newCntxElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId), afterSibling=xpCtx.outputLastContext.get(outputInstanceQname)) xpCtx.outputLastContext[outputInstanceQname] = newCntxElt entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity") XmlUtil.addChild(entityElt, XbrlConst.xbrli, "identifier", attributes=("scheme", entityIdentScheme), text=entityIdentValue) periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period") if periodType == "forever": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever") elif periodType == "instant": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "instant", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) elif periodType == "duration": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "startDate", text=XmlUtil.dateunionValue(periodStart)) XmlUtil.addChild(periodElt, XbrlConst.xbrli, "endDate", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) segmentElt = None scenarioElt = None from arelle.ModelObject import ModelDimensionValue if dimAspects: for dimQname in sorted(dimAspects.keys()): dimValue = dimAspects[dimQname] if isinstance(dimValue, ModelDimensionValue): if dimValue.isExplicit: dimMemberQname = dimValue.memberQname contextEltName = dimValue.contextElement else: # qname for explicit or node for typed dimMemberQname = dimValue contextEltName = xpCtx.modelXbrl.qnameDimensionContextElement.get(dimQname) if contextEltName == "segment": if not segmentElt: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") contextElt = segmentElt elif contextEltName == "scenario": if not scenarioElt: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") contextElt = scenarioElt else: continue dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname] dimAttr = ("dimension", XmlUtil.addQnameValue(xbrlElt, dimConcept.qname)) if dimConcept.isTypedDimension: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "typedMember", attributes=dimAttr) if isinstance(dimValue, ModelDimensionValue) and dimValue.isTyped: XmlUtil.copyChildren(dimElt, dimValue.typedMember) elif dimMemberQname: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "explicitMember", attributes=dimAttr, text=XmlUtil.addQnameValue(xbrlElt, dimMemberQname)) if segOCCs: if not segmentElt: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") XmlUtil.copyNodes(segmentElt, segOCCs) if scenOCCs: if not scenarioElt: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") XmlUtil.copyNodes(scenarioElt, scenOCCs) outputXbrlInstance.modelDocument.contextDiscover(newCntxElt) # does unit exist # add unit if modelConcept.isNumeric: prevUnit = outputXbrlInstance.matchUnit(multiplyBy, divideBy) if prevUnit: unitId = prevUnit.id newUnitElt = prevUnit.element else: unitId = 'u-{0:02n}'.format( len(outputXbrlInstance.units) + 1) newUnitElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId), afterSibling=xpCtx.outputLastUnit.get(outputInstanceQname)) xpCtx.outputLastUnit[outputInstanceQname] = newUnitElt if len(divideBy) == 0: for multiply in multiplyBy: XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) else: divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide") numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator") denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator") for multiply in multiplyBy: XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) for divide in divideBy: XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, divide)) outputXbrlInstance.modelDocument.unitDiscover(newUnitElt) # add fact attrs = [("contextRef", cntxId)] precision = None decimals = None if modelConcept.isNumeric: attrs.append(("unitRef", unitId)) value = formula.evaluate(xpCtx) valueSeqLen = len(value) if valueSeqLen > 1: xpCtx.modelXbrl.error( _("Formula {0} value is a sequence of length {1}").format( formula, valueSeqLen ), "err", "xbrlfe:nonSingletonOutputValue") else: if valueSeqLen == 0: #xsi:nil if no value attrs.append((XbrlConst.qnXsiNil, "true")) v = None else: # add precision/decimals for non-fraction numerics if modelConcept.isNumeric and not modelConcept.isFraction: if formula.hasDecimals: decimals = formula.evaluateRule(xpCtx, Aspect.DECIMALS) attrs.append(("decimals", decimals)) else: if formula.hasPrecision: precision = formula.evaluateRule(xpCtx, Aspect.PRECISION) else: precision = 0 attrs.append(("precision", precision)) x = value[0] if isinstance(x,float): from math import (log10, isnan, isinf, fabs) if (isnan(x) or (precision and (isinf(precision) or precision == 0)) or (decimals and isinf(decimals))): v = string(xpCtx, x) elif decimals is not None: v = "%.*f" % ( int(decimals), x) elif precision is not None: a = fabs(x) log = log10(a) if a != 0 else 0 v = "%.*f" % ( int(precision) - int(log) - (1 if a >= 1 else 0), x) else: # no implicit precision yet v = string(xpCtx, x) elif isinstance(x,QName): v = XmlUtil.addQnameValue(xbrlElt, x) elif isinstance(x,datetime.datetime): v = XmlUtil.dateunionValue(x) else: v = string(xpCtx, x) itemElt = XmlUtil.addChild(xbrlElt, conceptQname, attributes=attrs, text=v, afterSibling=xpCtx.outputLastFact.get(outputInstanceQname)) xpCtx.outputLastFact[outputInstanceQname] = itemElt newFact = outputXbrlInstance.modelDocument.factDiscover(itemElt, outputXbrlInstance.facts) return newFact
def runFromExcel(self, options): testGenFileName = options.excelfilename #testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls" testGenDir = os.path.dirname(testGenFileName) timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) if options.testfiledate: today = options.testfiledate else: today = XmlUtil.dateunionValue(datetime.date.today()) startedAt = time.time() self.logMessages = [] logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt' modelTestcases = ModelXbrl.create(self.modelManager) testcaseIndexBook = xlrd.open_workbook(testGenFileName) testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0) self.addToLog( _("[info] xls loaded in {0:.2} secs at {1}").format( time.time() - startedAt, timeNow)) # start index file indexFiles = [ testGenDir + os.sep + 'creationTestcasesIndex.xml', testGenDir + os.sep + 'consumptionTestcasesIndex.xml' ] indexDocs = [ xml.dom.minidom.parseString( '<?xml version="1.0" encoding="UTF-8"?>' '<!-- XBRL Versioning 1.0 {0} Tests -->' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>' '<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" ' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">' '</testcases>'.format(purpose, today)) for purpose in ("Creation", "Consumption") ] testcasesElements = [ XmlUtil.child(indexDoc, None, "testcases") for indexDoc in indexDocs ] priorTestcasesDir = None testcaseFiles = None testcaseDocs = None for iRow in range(1, testcaseIndexSheet.nrows): row = testcaseIndexSheet.row(iRow) if row[0].ctype == xlrd.XL_CELL_EMPTY or row[ 1].ctype == xlrd.XL_CELL_EMPTY or row[ 2].ctype == xlrd.XL_CELL_EMPTY: continue testDir = row[0].value uriFrom = row[1].value uriTo = row[2].value intention = row[3].value if intention is None or len(intention) == 0: continue # test not ready to run reason = row[4].value expectedEvent = row[5].value base = os.path.join(os.path.dirname(testGenFileName), testDir) + os.sep self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom)) if uriFrom and uriTo and reason.lower() not in ( "n.a.", "error") and expectedEvent != "N.A.": for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)): if ',' not in URIs: modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base) else: modelDTS = ModelXbrl.create( self.modelManager, ModelDocument.Type.DTSENTRIES, self.webCache.normalizeUrl( URIs.replace(", ", "_") + ".dts", base), isEntry=True) DTSdoc = modelDTS.modelDocument DTSdoc.inDTS = True for uri in URIs.split(','): doc = ModelDocument.load(modelDTS, uri.strip(), base=base) DTSdoc.referencesDocument[ doc] = "import" #fake import doc.inDTS = True if isFrom: modelDTSfrom = modelDTS else: modelDTSto = modelDTS if modelDTSfrom and modelDTSto: # generate differences report reportUri = uriFrom.partition(',')[0] # first file reportDir = os.path.dirname(reportUri) if reportDir: reportDir += os.sep reportName = os.path.basename(reportUri).replace( "from.xsd", "report.xml") reportFile = reportDir + "report" + os.sep + reportName reportFullPath = self.webCache.normalizeUrl( reportFile, base) testcasesDir = os.path.dirname( os.path.dirname(reportFullPath)) if testcasesDir != priorTestcasesDir: # close prior report if priorTestcasesDir: for i, testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") testcaseName = os.path.basename(testcasesDir) testcaseFiles = [ testcasesDir + os.sep + testcaseName + "-creation-testcase.xml", testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml" ] for i, testcaseFile in enumerate(testcaseFiles): XmlUtil.addChild( testcasesElements[i], None, "testcase", ("uri", testcaseFile[len(testGenDir) + 1:].replace("\\", "/"))) # start testcase file testcaseDocs = [ xml.dom.minidom.parseString( '<?xml version="1.0" encoding="UTF-8"?>' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>' '<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" ' ' xmlns="http://xbrl.org/2008/conformance"' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">' '<creator>' '<name>Roland Hommes</name>' '<email>[email protected]</email>' '</creator>' '<name>{0}</name>' '<description>{0}</description>' '</testcase>'.format(testcaseName, purpose, today)) for purpose in ("Creation", "Consumption") ] testcaseElements = [ XmlUtil.child(testcaseDoc, conformanceNS, "testcase") for testcaseDoc in testcaseDocs ] priorTestcasesDir = testcasesDir variationID = 1 try: os.makedirs(os.path.dirname(reportFullPath)) except WindowsError: pass # dir already exists modelVersReport = ModelVersReport.ModelVersReport( modelTestcases) modelVersReport.diffDTSes(reportFullPath, modelDTSfrom, modelDTSto) # check for expected elements if expectedEvent and expectedEvent not in ("No change", "N.A."): if len( modelVersReport.xmlDocument. getElementsByTagNameNS('*', expectedEvent)) == 0: modelTestcases.error( "Generated test case {0} missing expected event {1}" .format(reportName, expectedEvent), "wrn", "missingEvent") modelVersReport.close([]) for i, testcaseElt in enumerate(testcaseElements): variationElement = XmlUtil.addChild( testcaseElt, conformanceNS, "variation", attributes=("id", "_{0:02n}".format(variationID))) XmlUtil.addChild(variationElement, conformanceNS, "name", text=intention) dataElement = XmlUtil.addChild(variationElement, conformanceNS, "data") for schemaURIs, dtsAttr in ((uriFrom, "from"), (uriTo, "to")): for schemaURI in schemaURIs.split(","): XmlUtil.addChild( dataElement, conformanceNS, "schema", attributes=((("dts", dtsAttr), ) + ((("readMeFirst", "true"), ) if i == 0 else ())), text=os.path.basename(schemaURI.strip())) resultElement = XmlUtil.addChild( variationElement, conformanceNS, "result") XmlUtil.addChild( resultElement if i == 0 else dataElement, conformanceNS, "versioningReport", attributes=(("readMeFirst", "true") if i == 1 else ()), text="report/" + reportName) variationID += 1 with open(logMessagesFile, "w") as fh: fh.writelines(self.logMessages) if priorTestcasesDir: for i, testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") for i, indexFile in enumerate(indexFiles): with open(indexFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def runFromExcel(self, options): testGenFileName = options.excelfilename #testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls" testGenDir = os.path.dirname(testGenFileName) timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) if options.testfiledate: today = options.testfiledate else: today = XmlUtil.dateunionValue(datetime.date.today()) startedAt = time.time() self.logMessages = [] logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt' modelTestcases = ModelXbrl.create(self.modelManager) testcaseIndexBook = xlrd.open_workbook(testGenFileName) testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0) self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow)) # start index file indexFiles = [testGenDir + os.sep + 'creationTestcasesIndex.xml', testGenDir + os.sep + 'consumptionTestcasesIndex.xml'] indexDocs = [xml.dom.minidom.parseString( '<?xml version="1.0" encoding="UTF-8"?>' '<!-- XBRL Versioning 1.0 {0} Tests -->' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>' '<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" ' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">' '</testcases>'.format(purpose, today) ) for purpose in ("Creation","Consumption")] testcasesElements = [XmlUtil.child(indexDoc, None, "testcases") for indexDoc in indexDocs] priorTestcasesDir = None testcaseFiles = None testcaseDocs = None for iRow in range(1, testcaseIndexSheet.nrows): row = testcaseIndexSheet.row(iRow) if row[0].ctype == xlrd.XL_CELL_EMPTY or row[1].ctype == xlrd.XL_CELL_EMPTY or row[2].ctype == xlrd.XL_CELL_EMPTY: continue testDir = row[0].value uriFrom = row[1].value uriTo = row[2].value intention = row[3].value if intention is None or len(intention) == 0: continue # test not ready to run reason = row[4].value expectedEvent = row[5].value base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom)) if uriFrom and uriTo and reason.lower() not in ("n.a.", "error") and expectedEvent != "N.A.": for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)): if ',' not in URIs: modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base) else: modelDTS = ModelXbrl.create(self.modelManager, ModelDocument.Type.DTSENTRIES, self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts", base), isEntry=True) DTSdoc = modelDTS.modelDocument DTSdoc.inDTS = True for uri in URIs.split(','): doc = ModelDocument.load(modelDTS, uri.strip(), base=base) DTSdoc.referencesDocument[doc] = "import" #fake import doc.inDTS = True if isFrom: modelDTSfrom = modelDTS else: modelDTSto = modelDTS if modelDTSfrom and modelDTSto: # generate differences report reportUri = uriFrom.partition(',')[0] # first file reportDir = os.path.dirname(reportUri) if reportDir: reportDir += os.sep reportName = os.path.basename(reportUri).replace("from.xsd","report.xml") reportFile = reportDir + "report" + os.sep + reportName reportFullPath = self.webCache.normalizeUrl( reportFile, base) testcasesDir = os.path.dirname(os.path.dirname(reportFullPath)) if testcasesDir != priorTestcasesDir: # close prior report if priorTestcasesDir: for i,testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") testcaseName = os.path.basename(testcasesDir) testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml", testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"] for i,testcaseFile in enumerate(testcaseFiles): XmlUtil.addChild(testcasesElements[i], None, "testcase", ("uri", testcaseFile[len(testGenDir)+1:].replace("\\","/")) ) # start testcase file testcaseDocs = [xml.dom.minidom.parseString( '<?xml version="1.0" encoding="UTF-8"?>' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>' '<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" ' ' xmlns="http://xbrl.org/2008/conformance"' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">' '<creator>' '<name>Roland Hommes</name>' '<email>[email protected]</email>' '</creator>' '<name>{0}</name>' '<description>{0}</description>' '</testcase>'.format(testcaseName,purpose,today) ) for purpose in ("Creation","Consumption")] testcaseElements = [XmlUtil.child(testcaseDoc, conformanceNS, "testcase") for testcaseDoc in testcaseDocs] priorTestcasesDir = testcasesDir variationID = 1 try: os.makedirs(os.path.dirname(reportFullPath)) except WindowsError: pass # dir already exists modelVersReport = ModelVersReport.ModelVersReport(modelTestcases) modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto) # check for expected elements if expectedEvent and expectedEvent not in ( "No change", "N.A."): if len(modelVersReport.xmlDocument.getElementsByTagNameNS('*',expectedEvent)) == 0: modelTestcases.error( "Generated test case {0} missing expected event {1}".format( reportName, expectedEvent), "wrn", "missingEvent") modelVersReport.close([]) for i,testcaseElt in enumerate(testcaseElements): variationElement = XmlUtil.addChild(testcaseElt, conformanceNS, "variation", attributes=("id", "_{0:02n}".format(variationID))) XmlUtil.addChild(variationElement, conformanceNS, "name", text=intention) dataElement = XmlUtil.addChild(variationElement, conformanceNS, "data") for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")): for schemaURI in schemaURIs.split(","): XmlUtil.addChild(dataElement, conformanceNS, "schema", attributes=((("dts",dtsAttr),) + ((("readMeFirst","true"),) if i == 0 else ())), text=os.path.basename(schemaURI.strip())) resultElement = XmlUtil.addChild(variationElement, conformanceNS, "result") XmlUtil.addChild(resultElement if i == 0 else dataElement, conformanceNS, "versioningReport", attributes=(("readMeFirst","true") if i == 1 else ()), text="report/" + reportName) variationID += 1 with open(logMessagesFile, "w") as fh: fh.writelines(self.logMessages) if priorTestcasesDir: for i,testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") for i,indexFile in enumerate(indexFiles): with open(indexFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def produceOutputFact(xpCtx, formula, result): priorErrorCount = len(xpCtx.modelXbrl.errors) # assemble context conceptQname = aspectValue(xpCtx, formula, Aspect.CONCEPT, "xbrlfe:missingConceptRule") if isinstance(conceptQname, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} concept: {1}").format(formula, conceptQname.msg), "err", conceptQname.err) modelConcept = None else: modelConcept = xpCtx.modelXbrl.qnameConcepts[conceptQname] if modelConcept is None or not modelConcept.isItem: xpCtx.modelXbrl.error( _("Formula {0} concept {1} is not an item").format( formula, conceptQname), "err", "xbrlfe:missingConceptRule") # entity entityIdentScheme = aspectValue(xpCtx, formula, Aspect.SCHEME, "xbrlfe:missingEntityIdentifierRule") if isinstance(entityIdentScheme, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} entity identifier scheme: {1}").format( formula, entityIdentScheme.msg), "err", str(entityIdentScheme)) entityIdentValue = None else: entityIdentValue = aspectValue(xpCtx, formula, Aspect.VALUE, "xbrlfe:missingEntityIdentifierRule") if isinstance(entityIdentValue, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} entity identifier value: {1}").format( formula, entityIdentValue.msg), "err", str(entityIdentScheme)) # period periodType = aspectValue(xpCtx, formula, Aspect.PERIOD_TYPE, "xbrlfe:missingPeriodRule") periodStart = None periodEndInstant = None if isinstance(periodType, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} period type: {1}").format(formula, periodType.msg), "err", str(periodType)) elif periodType == "instant": periodEndInstant = aspectValue(xpCtx, formula, Aspect.INSTANT, "xbrlfe:missingPeriodRule") if isinstance(periodEndInstant, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} period start: {1}").format( formula, periodEndInstant.msg), "err", str(periodEndInstant)) elif periodType == "duration": periodStart = aspectValue(xpCtx, formula, Aspect.START, "xbrlfe:missingPeriodRule") if isinstance(periodStart, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} period start: {1}").format( formula, periodStart.msg), "err", str(periodStart)) periodEndInstant = aspectValue(xpCtx, formula, Aspect.END, "xbrlfe:missingPeriodRule") if isinstance(periodEndInstant, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} period end: {1}").format( formula, periodEndInstant.msg), "err", str(periodEndInstant)) # unit if modelConcept and modelConcept.isNumeric: unitSource = aspectValue(xpCtx, formula, Aspect.UNIT_MEASURES, None) multDivBy = aspectValue(xpCtx, formula, Aspect.MULTIPLY_BY, "xbrlfe:missingUnitRule") if isinstance(multDivBy, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} unit: {1}").format(formula, multDivBy.msg), "err", str(multDivBy) if isinstance(multDivBy, VariableBindingError) else "xbrlfe:missingUnitRule") multiplyBy = () divideBy = () # prevent errors later if bad else: divMultBy = aspectValue(xpCtx, formula, Aspect.DIVIDE_BY, "xbrlfe:missingUnitRule") if isinstance(divMultBy, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} unit: {1}").format(formula, divMultBy.msg), "err", str(multDivBy) if isinstance(divMultBy, VariableBindingError) else "xbrlfe:missingUnitRule") multiplyBy = () divideBy = () # prevent errors later if bad else: multiplyBy = unitSource[0] + multDivBy[0] + divMultBy[1] divideBy = unitSource[1] + multDivBy[1] + divMultBy[0] # remove cancelling mult/div units lookForCommonUnits = True while lookForCommonUnits: lookForCommonUnits = False for commonUnit in multiplyBy: if commonUnit in divideBy: multiplyBy.remove(commonUnit) divideBy.remove(commonUnit) lookForCommonUnits = True break if len(multiplyBy) == 0: # if no units add pure multiplyBy.append(XbrlConst.qnXbrliPure) # dimensions segOCCs = [] scenOCCs = [] if formula.aspectModel == "dimensional": dimAspects = {} dimQnames = aspectValue(xpCtx, formula, Aspect.DIMENSIONS, None) if dimQnames: for dimQname in dimQnames: dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname] dimErr = "xbrlfe:missing{0}DimensionRule".format( "typed" if dimConcept and dimConcept.isTypedDimension else "explicit") dimValue = aspectValue(xpCtx, formula, dimQname, dimErr) if isinstance(dimValue, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} dimension {1}: {2}").format( formula, dimQname, dimValue.msg), "err", dimErr) elif dimValue and xpCtx.modelXbrl.qnameDimensionDefaults.get( dimQname) != dimValue: dimAspects[dimQname] = dimValue segOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SEGMENT, None) scenOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SCENARIO, None) else: dimAspects = None # non-dimensional segOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SEGMENT, None) scenOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SCENARIO, None) if priorErrorCount < len(xpCtx.modelXbrl.errors): return None # had errors, don't produce output fact # does context exist in out instance document outputInstanceQname = formula.outputInstanceQname outputXbrlInstance = xpCtx.inScopeVars[outputInstanceQname] xbrlElt = outputXbrlInstance.modelDocument.xmlRootElement # in source instance document # add context prevCntx = outputXbrlInstance.matchContext(entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, dimAspects, segOCCs, scenOCCs) if prevCntx: cntxId = prevCntx.id newCntxElt = prevCntx.element else: cntxId = 'c-{0:02n}'.format(len(outputXbrlInstance.contexts) + 1) newCntxElt = XmlUtil.addChild( xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId), afterSibling=xpCtx.outputLastContext.get(outputInstanceQname)) xpCtx.outputLastContext[outputInstanceQname] = newCntxElt entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity") XmlUtil.addChild(entityElt, XbrlConst.xbrli, "identifier", attributes=("scheme", entityIdentScheme), text=entityIdentValue) periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period") if periodType == "forever": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever") elif periodType == "instant": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "instant", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) elif periodType == "duration": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "startDate", text=XmlUtil.dateunionValue(periodStart)) XmlUtil.addChild(periodElt, XbrlConst.xbrli, "endDate", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) segmentElt = None scenarioElt = None from arelle.ModelObject import ModelDimensionValue if dimAspects: for dimQname in sorted(dimAspects.keys()): dimValue = dimAspects[dimQname] if isinstance(dimValue, ModelDimensionValue): if dimValue.isExplicit: dimMemberQname = dimValue.memberQname contextEltName = dimValue.contextElement else: # qname for explicit or node for typed dimMemberQname = dimValue contextEltName = xpCtx.modelXbrl.qnameDimensionContextElement.get( dimQname) if contextEltName == "segment": if not segmentElt: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") contextElt = segmentElt elif contextEltName == "scenario": if not scenarioElt: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") contextElt = scenarioElt else: continue dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname] dimAttr = ("dimension", XmlUtil.addQnameValue(xbrlElt, dimConcept.qname)) if dimConcept.isTypedDimension: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "typedMember", attributes=dimAttr) if isinstance(dimValue, ModelDimensionValue) and dimValue.isTyped: XmlUtil.copyChildren(dimElt, dimValue.typedMember) elif dimMemberQname: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "explicitMember", attributes=dimAttr, text=XmlUtil.addQnameValue( xbrlElt, dimMemberQname)) if segOCCs: if not segmentElt: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") XmlUtil.copyNodes(segmentElt, segOCCs) if scenOCCs: if not scenarioElt: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") XmlUtil.copyNodes(scenarioElt, scenOCCs) outputXbrlInstance.modelDocument.contextDiscover(newCntxElt) # does unit exist # add unit if modelConcept.isNumeric: prevUnit = outputXbrlInstance.matchUnit(multiplyBy, divideBy) if prevUnit: unitId = prevUnit.id newUnitElt = prevUnit.element else: unitId = 'u-{0:02n}'.format(len(outputXbrlInstance.units) + 1) newUnitElt = XmlUtil.addChild( xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId), afterSibling=xpCtx.outputLastUnit.get(outputInstanceQname)) xpCtx.outputLastUnit[outputInstanceQname] = newUnitElt if len(divideBy) == 0: for multiply in multiplyBy: XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue( xbrlElt, multiply)) else: divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide") numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator") denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator") for multiply in multiplyBy: XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue( xbrlElt, multiply)) for divide in divideBy: XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue( xbrlElt, divide)) outputXbrlInstance.modelDocument.unitDiscover(newUnitElt) # add fact attrs = [("contextRef", cntxId)] precision = None decimals = None if modelConcept.isNumeric: attrs.append(("unitRef", unitId)) value = formula.evaluate(xpCtx) valueSeqLen = len(value) if valueSeqLen > 1: xpCtx.modelXbrl.error( _("Formula {0} value is a sequence of length {1}").format( formula, valueSeqLen), "err", "xbrlfe:nonSingletonOutputValue") else: if valueSeqLen == 0: #xsi:nil if no value attrs.append((XbrlConst.qnXsiNil, "true")) v = None else: # add precision/decimals for non-fraction numerics if modelConcept.isNumeric and not modelConcept.isFraction: if formula.hasDecimals: decimals = formula.evaluateRule(xpCtx, Aspect.DECIMALS) attrs.append(("decimals", decimals)) else: if formula.hasPrecision: precision = formula.evaluateRule( xpCtx, Aspect.PRECISION) else: precision = 0 attrs.append(("precision", precision)) x = value[0] if isinstance(x, float): from math import (log10, isnan, isinf, fabs) if (isnan(x) or (precision and (isinf(precision) or precision == 0)) or (decimals and isinf(decimals))): v = string(xpCtx, x) elif decimals is not None: v = "%.*f" % (int(decimals), x) elif precision is not None: a = fabs(x) log = log10(a) if a != 0 else 0 v = "%.*f" % (int(precision) - int(log) - (1 if a >= 1 else 0), x) else: # no implicit precision yet v = string(xpCtx, x) elif isinstance(x, QName): v = XmlUtil.addQnameValue(xbrlElt, x) elif isinstance(x, datetime.datetime): v = XmlUtil.dateunionValue(x) else: v = string(xpCtx, x) itemElt = XmlUtil.addChild( xbrlElt, conceptQname, attributes=attrs, text=v, afterSibling=xpCtx.outputLastFact.get(outputInstanceQname)) xpCtx.outputLastFact[outputInstanceQname] = itemElt newFact = outputXbrlInstance.modelDocument.factDiscover( itemElt, outputXbrlInstance.facts) return newFact
def loadFromExcel(cntlr, excelFile): from arelle import xlrd from arelle.xlrd.sheet import empty_cell from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname startedAt = time.time() importExcelBook = xlrd.open_workbook(excelFile) controlSheet = importExcelBook.sheet_by_index(1) imports = {"xbrli": ( ("namespace", XbrlConst.xbrli), ("schemaLocation", "http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd") )} # xml of imports importXmlns = {} linkbaseRefs = [] labelLinkbases = [] hasPreLB = hasCalLB = hasDefLB = False # xxxLB structure [ (elr1, def1, "_ELR_", [roots]), (elr2, def2, "_ELR_", [rootw]) ...] # roots = (rootHref, None, "_root_", [children]) # children = (childPrefix, childName, arcrole, [grandChildren]) preLB = [] defLB = [] calLB = [] def lbDepthList(lbStruct, depth, parentList=None): if depth == 0: return lbStruct[-1].childStruct return lbDepthList(lbStruct[-1].childStruct, depth-1, list) extensionElements = {} extensionRoles = {} # key is roleURI, value is role definition extensionLabels = {} # key = (prefix, name, lang, role), value = label text def extensionHref(prefix, name): if prefix == extensionSchemaPrefix: filename = extensionSchemaFilename elif prefix in imports: filename = imports[prefix][1][1] else: return None return "{0}#{1}_{2}".format(filename, prefix, name) isUSGAAP = False for iRow in range(1, controlSheet.nrows): try: row = controlSheet.row(iRow) if (row[0].ctype == xlrd.XL_CELL_EMPTY): # skip if col 1 is empty continue action = row[0].value filetype = row[1].value prefix = row[2].value filename = row[3].value namespaceURI = row[4].value lbType = lang = None if action == "import": imports[prefix] = ( ("namespace", namespaceURI), ("schemaLocation", filename) ) importXmlns[prefix] = namespaceURI if re.match(r"http://[^/]+/us-gaap/", namespaceURI): isUSGAAP = True elif action == "extension": if filetype == "schema": extensionSchemaPrefix = prefix extensionSchemaFilename = filename extensionSchemaNamespaceURI = namespaceURI elif filetype == "linkbase": typeLang = prefix.split() if len(typeLang) > 0: lbType = typeLang[0] else: lbType = "unknown" if len(typeLang) > 1: lang = typeLang[1] else: lang = "en" if lbType == "label": labelLinkbases.append((lang, filename)) elif lbType == "presentation": hasPreLB = True elif lbType == "definition": hasDefLB = True elif lbType == "calculation": hasCalLB = True linkbaseRefs.append( (lbType, filename) ) elif filetype == "role" and namespaceURI: extensionRoles[namespaceURI] = filename except Exception as err: cntlr.addToLog("Exception: {error}, Excel row: {excelRow}" .format(error=err, excelRow=iRow), messageCode="importExcel:exception") importExcelSheet = importExcelBook.sheet_by_index(0) # find column headers row headerCols = {} headerRows = set() # find out which rows are header rows for iRow in range(0, importExcelSheet.nrows): row = importExcelSheet.row(iRow) for iCol, colCell in enumerate(row): v = colCell.value if v in importColumnHeaders: headerCols[importColumnHeaders[v]] = iCol if all(colName in headerCols for colName in ("name", "type", "depth")): # must have these to be a header col # it's a header col headerRows.add(iRow) headerCols.clear() def cellValue(row, header): if header in headerCols: iCol = headerCols[header] if iCol < len(row): return row[iCol].value return '' def checkImport(qname): prefix, sep, localName = qname.partition(":") if sep: if prefix not in imports: if prefix == "xbrldt": imports["xbrldt"] = ("namespace", XbrlConst.xbrldt), ("schemaLocation", "http://www.xbrl.org/2005/xbrldt-2005.xsd") elif prefix == "nonnum": imports["nonnum"] = ("namespace", "http://www.xbrl.org/dtr/type/non-numeric"), ("schemaLocation", "http://www.xbrl.org/dtr/type/nonNumeric-2009-12-16.xsd") else: cntlr.addToLog("Warning: prefix schema file is not imported for: {qname}" .format(qname=qname), messageCode="importExcel:warning") # find header rows currentELR = currentELRdefinition = None for iRow in range(0, importExcelSheet.nrows): useLabels = False try: row = importExcelSheet.row(iRow) isHeaderRow = iRow in headerRows isELRrow = (iRow + 1) in headerRows if isHeaderRow: headerCols.clear() for iCol, colCell in enumerate(row): v = colCell.value if v in importColumnHeaders: headerCols[importColumnHeaders[v]] = iCol elif isELRrow: currentELR = currentELRdefinition = None for colCell in row: v = colCell.value if v.startswith("http://"): currentELR = v elif not currentELRdefinition and v.endswith(" 科目一覧"): currentELRdefinition = v[0:-5] elif not currentELRdefinition: currentELRdefinition = v if currentELR or currentELRdefinition: if hasPreLB: preLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) ) if hasDefLB: defLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) ) if hasCalLB: calLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) ) elif headerCols: prefix = cellValue(row, 'prefix').strip() name = cellValue(row, 'name').strip() if "depth" in headerCols: try: depth = int(cellValue(row, 'depth')) except ValueError: depth = None else: depth = None if prefix == extensionSchemaPrefix and name not in extensionElements: # elements row eltType = cellValue(row, 'type') subsGrp = cellValue(row, 'substitutionGroup') abstract = cellValue(row, 'abstract') nillable = cellValue(row, 'nillable') balance = cellValue(row, 'balance') periodType = cellValue(row, 'periodType') newElt = [ ("name", name), ("id", prefix + "_" + name) ] if eltType: newElt.append( ("type", eltType) ) checkImport(eltType) if subsGrp: newElt.append( ("substitutionGroup", subsGrp) ) checkImport(subsGrp) if abstract: newElt.append( ("abstract", abstract) ) if nillable: newElt.append( ("nillable", nillable) ) if balance: newElt.append( ("{http://www.xbrl.org/2003/instance}balance", balance) ) if periodType: newElt.append( ("{http://www.xbrl.org/2003/instance}periodType", periodType) ) extensionElements[name] = newElt useLabels = True if depth is not None: if hasPreLB: entryList = lbDepthList(preLB, depth) preferredLabel = cellValue(row, 'preferredLabel') if preferredLabel and not preferredLabel.startswith("http://"): preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel if entryList is not None: if depth == 0: entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) ) else: entryList.append( LBentry(prefix=prefix, name=name, arcrole=XbrlConst.parentChild, role=preferredLabel) ) if hasDefLB: entryList = lbDepthList(defLB, depth) if entryList is not None: if depth == 0: entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) ) else: entryList.append( LBentry(prefix=prefix, name=name, arcrole="_dimensions_") ) if hasCalLB: calcParent = cellValue(row, 'calculationParent') calcWeight = cellValue(row, 'calculationWeight') if calcParent and calcWeight: calcParentPrefix, sep, calcParentName = calcParent.partition(":") entryList = lbDepthList(calLB, 0) if entryList is not None: entryList.append( LBentry(prefix=calcParentPrefix, name=calcParentName, isRoot=True, childStruct= [LBentry(prefix=prefix, name=name, arcrole=XbrlConst.summationItem, weight=calcWeight )]) ) # accumulate extension labels if useLabels: prefix = cellValue(row, 'prefix').strip() name = cellValue(row, 'name').strip() preferredLabel = cellValue(row, 'preferredLabel') if preferredLabel and not preferredLabel.startswith("http://"): preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel for colItem, iCol in headerCols.items(): if isinstance(colItem, tuple): colItemType, role, lang = colItem cell = row[iCol] if cell.ctype == xlrd.XL_CELL_EMPTY: values = () elif colItemType == "label": values = (cell.value,) elif colItemType == "labels": values = cell.value.split('\n') else: values = () if preferredLabel: # first label column sets preferredLabel if any role = preferredLabel preferredLabel = None for value in values: extensionLabels[prefix, name, lang, role] = value.strip() except Exception as err: cntlr.addToLog("Exception: {error}, Excel row: {excelRow}" .format(error=err, excelRow=iRow), messageCode="importExcel:exception") if isUSGAAP and hasDefLB: # move line items above table def fixUsggapTableDims(lvl1Struct): foundLineItems = False for i1, lvl1Entry in enumerate(lvl1Struct): for i2, lvl2Entry in enumerate(lvl1Entry.childStruct): for i3, lvl3Entry in enumerate(lvl2Entry.childStruct): if lvl3Entry.name.endswith("LineItems") and lvl2Entry.name.endswith("Table"): foundLineItems = True break if foundLineItems: break else: fixUsggapTableDims(lvl1Entry.childStruct) if foundLineItems: lvl1Struct.insert(i1 + 1, LBentry(prefix=lvl3Entry.prefix, name=lvl3Entry.name, arcrole=lvl1Entry.arcrole, childStruct=lvl3Entry.childStruct)) # must keep lvl1Rel if it is __root__ lvl3Entry.childStruct.insert(0, lvl2Entry) if lvl1Entry.name.endswith("Abstract"): del lvl1Struct[i1] del lvl2Entry.childStruct[i3] pass fixUsggapTableDims(defLB) dts = cntlr.modelManager.create(newDocumentType=ModelDocument.Type.SCHEMA, url=extensionSchemaFilename, isEntry=True, base='', # block pathname from becomming absolute initialXml=''' <schema xmlns="http://www.w3.org/2001/XMLSchema" targetNamespace="{targetNamespace}" attributeFormDefault="unqualified" elementFormDefault="qualified" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:{extensionPrefix}="{targetNamespace}" {importXmlns} xmlns:iod="http://disclosure.edinet-fsa.go.jp/taxonomy/common/2013-03-31/iod" xmlns:nonnum="http://www.xbrl.org/dtr/type/non-numeric" xmlns:link="http://www.xbrl.org/2003/linkbase" xmlns:xbrli="http://www.xbrl.org/2003/instance" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xbrldt="http://xbrl.org/2005/xbrldt"/> '''.format(targetNamespace=extensionSchemaNamespaceURI, extensionPrefix=extensionSchemaPrefix, importXmlns=''.join('xmlns:{0}="{1}"\n'.format(prefix, namespaceURI) for prefix, namespaceURI in importXmlns.items()) ) ) dtsSchemaDocument = dts.modelDocument dtsSchemaDocument.inDTS = True # entry document always in DTS dtsSchemaDocument.targetNamespace = extensionSchemaNamespaceURI # not set until schemaDiscover too late otherwise schemaElt = dtsSchemaDocument.xmlRootElement #foreach linkbase annotationElt = XmlUtil.addChild(schemaElt, XbrlConst.xsd, "annotation") appinfoElt = XmlUtil.addChild(annotationElt, XbrlConst.xsd, "appinfo") for iRow in range(0, importExcelSheet.nrows): try: row = importExcelSheet.row(iRow) if (row[0].ctype == xlrd.XL_CELL_EMPTY): # skip if col 1 is empty continue testDir = row[0].value uriFrom = row[1].value uriTo = row[2].value except Exception as err: cntlr.addToLog("Exception: {error}, Excel row: {excelRow}" .format(error=err, excelRow=iRow), messageCode="loadFromExcel:exception") # add linkbaseRefs appinfoElt = XmlUtil.descendant(schemaElt, XbrlConst.xsd, "appinfo") # don't yet add linkbase refs, want to process imports first to get roleType definitions # add imports for importAttributes in sorted(imports.values()): XmlUtil.addChild(schemaElt, XbrlConst.xsd, "import", attributes=importAttributes) # add elements for eltName, eltAttrs in sorted(extensionElements.items(), key=lambda item: item[0]): XmlUtil.addChild(schemaElt, XbrlConst.xsd, "element", attributes=eltAttrs) # add role definitions (for discovery) for roleURI, roleDefinition in extensionRoles.items(): roleElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "roleType", attributes=(("roleURI", roleURI), ("id", "roleType_" + roleURI.rpartition("/")[2]))) if roleDefinition: XmlUtil.addChild(roleElt, XbrlConst.link, "definition", text=roleDefinition) if hasPreLB: XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:presentationLink") if hasDefLB: XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:definitionLink") if hasCalLB: XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:calculationLink") dtsSchemaDocument.schemaDiscover(schemaElt, False, extensionSchemaNamespaceURI) def addLinkbaseRef(lbType, lbFilename, lbDoc): role = "http://www.xbrl.org/2003/role/{0}LinkbaseRef".format(lbType) lbRefElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "linkbaseRef", attributes=(("{http://www.w3.org/1999/xlink}type", "simple"), ("{http://www.w3.org/1999/xlink}href", lbFilename), ("{http://www.w3.org/1999/xlink}role", role), ("{http://www.w3.org/1999/xlink}arcrole", "http://www.w3.org/1999/xlink/properties/linkbase"), )) dtsSchemaDocument.referencesDocument[lbDoc] = ModelDocumentReference("href", lbRefElt) # label linkbase for lang, filename in labelLinkbases: lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base="", initialXml=""" <link:linkbase xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd" xmlns:link="http://www.xbrl.org/2003/linkbase" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xbrli="http://www.xbrl.org/2003/instance"/> """) lbDoc.inDTS = True addLinkbaseRef("label", filename, lbDoc) lbElt = lbDoc.xmlRootElement linkElt = XmlUtil.addChild(lbElt, XbrlConst.link, "labelLink", attributes=(("{http://www.w3.org/1999/xlink}type", "extended"), ("{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/link"))) locs = set() for labelKey, text in extensionLabels.items(): prefix, name, labelLang, role = labelKey if lang == labelLang: locLabel = prefix + "_" + name if locLabel not in locs: locs.add(locLabel) XmlUtil.addChild(linkElt, XbrlConst.link, "loc", attributes=(("{http://www.w3.org/1999/xlink}type", "locator"), ("{http://www.w3.org/1999/xlink}href", extensionHref(prefix, name)), ("{http://www.w3.org/1999/xlink}label", locLabel))) XmlUtil.addChild(linkElt, XbrlConst.link, "labelArc", attributes=(("{http://www.w3.org/1999/xlink}type", "arc"), ("{http://www.w3.org/1999/xlink}arcrole", "http://www.xbrl.org/2003/arcrole/concept-label"), ("{http://www.w3.org/1999/xlink}from", locLabel), ("{http://www.w3.org/1999/xlink}to", "label_" + locLabel), ("order", 1.0))) XmlUtil.addChild(linkElt, XbrlConst.link, "label", attributes=(("{http://www.w3.org/1999/xlink}type", "resource"), ("{http://www.w3.org/1999/xlink}label", "label_" + locLabel), ("{http://www.w3.org/1999/xlink}role", role), ("{http://www.w3.org/XML/1998/namespace}lang", lang)), text=text) lbDoc.linkbaseDiscover(lbElt) def hrefConcept(prefix, name): qn = schemaElt.prefixedNameQname(prefix + ":" + name) if qn in dts.qnameConcepts: return dts.qnameConcepts[qn] return None def lbTreeWalk(lbType, parentElt, lbStruct, roleRefs, locs=None, fromPrefix=None, fromName=None): order = 1.0 for lbEntry in lbStruct: if lbEntry.isELR: role = "unspecified" if lbEntry.role and lbEntry.role.startswith("http://"): # have a role specified role = lbEntry.role elif lbEntry.name: #may be a definition for linkroleUri, modelRoleTypes in dts.roleTypes.items(): definition = modelRoleTypes[0].definition if lbEntry.name == definition: role = linkroleUri break if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef roleType = modelRoleTypes[0] roleRef = ("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id) roleRefs.add(roleRef) linkElt = XmlUtil.addChild(parentElt, XbrlConst.link, lbType + "Link", attributes=(("{http://www.w3.org/1999/xlink}type", "extended"), ("{http://www.w3.org/1999/xlink}role", role))) locs = set() lbTreeWalk(lbType, linkElt, lbEntry.childStruct, roleRefs, locs) else: toPrefix = lbEntry.prefix toName = lbEntry.name toHref = extensionHref(toPrefix, toName) toLabel = toPrefix + "_" + toName if toHref not in locs: XmlUtil.addChild(parentElt, XbrlConst.link, "loc", attributes=(("{http://www.w3.org/1999/xlink}type", "locator"), ("{http://www.w3.org/1999/xlink}href", toHref), ("{http://www.w3.org/1999/xlink}label", toLabel))) locs.add(toHref) if not lbEntry.isRoot: fromLabel = fromPrefix + "_" + fromName if lbType == "calculation" and lbEntry.weight is not None: otherAttrs = ( ("weight", lbEntry.weight), ) elif lbType == "presentation" and lbEntry.role is not None: otherAttrs = ( ("preferredLabel", lbEntry.role), ) else: otherAttrs = ( ) if lbEntry.arcrole == "_dimensions_": # pick proper consecutive arcrole fromConcept = hrefConcept(fromPrefix, fromName) toConcept = hrefConcept(toPrefix, toName) if toConcept is not None and toConcept.isHypercubeItem: arcrole = XbrlConst.all elif toConcept is not None and toConcept.isDimensionItem: arcrole = XbrlConst.hypercubeDimension elif fromConcept is not None and fromConcept.isDimensionItem: arcrole = XbrlConst.dimensionDomain else: arcrole = XbrlConst.domainMember else: arcrole = lbEntry.arcrole XmlUtil.addChild(parentElt, XbrlConst.link, lbType + "Arc", attributes=(("{http://www.w3.org/1999/xlink}type", "arc"), ("{http://www.w3.org/1999/xlink}arcrole", arcrole), ("{http://www.w3.org/1999/xlink}from", fromLabel), ("{http://www.w3.org/1999/xlink}to", toLabel), ("order", order)) + otherAttrs ) order += 1.0 if lbType != "calculation" or lbEntry.isRoot: lbTreeWalk(lbType, parentElt, lbEntry.childStruct, roleRefs, locs, toPrefix, toName) for hasLB, lbType, lbLB in ((hasPreLB, "presentation", preLB), (hasDefLB, "definition", defLB), (hasCalLB, "calculation", calLB)): if hasLB: for lbRefType, filename in linkbaseRefs: if lbType == lbRefType: # output presentation linkbase lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base='', initialXml=""" <link:linkbase xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd" xmlns:link="http://www.xbrl.org/2003/linkbase" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xbrli="http://www.xbrl.org/2003/instance"/> """) lbDoc.inDTS = True addLinkbaseRef(lbRefType, filename, lbDoc) lbElt = lbDoc.xmlRootElement roleRefs = set() if lbType == "definition": roleRefs.update((("arcroleRef", XbrlConst.all, "http://www.xbrl.org/2005/xbrldt-2005.xsd#all"), ("arcroleRef", XbrlConst.dimensionDefault, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-default"), ("arcroleRef", XbrlConst.dimensionDomain, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-domain"), ("arcroleRef", XbrlConst.domainMember, "http://www.xbrl.org/2005/xbrldt-2005.xsd#domain-member"), ("arcroleRef", XbrlConst.hypercubeDimension, "http://www.xbrl.org/2005/xbrldt-2005.xsd#hypercube-dimension"))) lbTreeWalk(lbType, lbElt, lbLB, roleRefs) firstLinkElt = None for firstLinkElt in lbElt.iterchildren(): break # add arcrole references for roleref, roleURI, href in roleRefs: XmlUtil.addChild(lbElt, XbrlConst.link, roleref, attributes=(("arcroleURI" if roleref == "arcroleRef" else "roleURI", roleURI), ("{http://www.w3.org/1999/xlink}type", "simple"), ("{http://www.w3.org/1999/xlink}href", href)), beforeSibling=firstLinkElt) lbDoc.linkbaseDiscover(lbElt) break #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") return dts
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in modelXbrl.contexts.values(): newCntx = targetInstance.createContext(context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in modelXbrl.facts: if fact.isItem: attrs = {"contextRef": fact.contextID} if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) newFactForOldObjId[fact.objectIndex] = newFact elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) # footnote links modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships")) for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: newLink = XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, linkqname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: if isinstance(linkChild, LocPrototype) and "{http://www.w3.org/1999/xlink}href" not in linkChild.attributes: linkChild.attributes["{http://www.w3.org/1999/xlink}href"] = \ "#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) XmlUtil.addChild(newLink, linkChild.qname, attributes=linkChild.attributes, text=linkChild.textValue) targetInstance.saveInstance(overrideFilepath=targetUrl) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def lbTreeWalk(lbType, parentElt, lbStruct, roleRefs, locs=None, fromPrefix=None, fromName=None): order = 1.0 for lbEntry in lbStruct: if lbEntry.isELR: role = "unspecified" if lbEntry.role and lbEntry.role.startswith("http://"): # have a role specified role = lbEntry.role elif lbEntry.name: #may be a definition for linkroleUri, modelRoleTypes in dts.roleTypes.items(): definition = modelRoleTypes[0].definition if lbEntry.name == definition: role = linkroleUri break if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef roleType = modelRoleTypes[0] roleRef = ("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id) roleRefs.add(roleRef) linkElt = XmlUtil.addChild(parentElt, XbrlConst.link, lbType + "Link", attributes=(("{http://www.w3.org/1999/xlink}type", "extended"), ("{http://www.w3.org/1999/xlink}role", role))) locs = set() lbTreeWalk(lbType, linkElt, lbEntry.childStruct, roleRefs, locs) else: toPrefix = lbEntry.prefix toName = lbEntry.name toHref = extensionHref(toPrefix, toName) toLabel = toPrefix + "_" + toName if toHref not in locs: XmlUtil.addChild(parentElt, XbrlConst.link, "loc", attributes=(("{http://www.w3.org/1999/xlink}type", "locator"), ("{http://www.w3.org/1999/xlink}href", toHref), ("{http://www.w3.org/1999/xlink}label", toLabel))) locs.add(toHref) if not lbEntry.isRoot: fromLabel = fromPrefix + "_" + fromName if lbType == "calculation" and lbEntry.weight is not None: otherAttrs = ( ("weight", lbEntry.weight), ) elif lbType == "presentation" and lbEntry.role is not None: otherAttrs = ( ("preferredLabel", lbEntry.role), ) else: otherAttrs = ( ) if lbEntry.arcrole == "_dimensions_": # pick proper consecutive arcrole fromConcept = hrefConcept(fromPrefix, fromName) toConcept = hrefConcept(toPrefix, toName) if toConcept is not None and toConcept.isHypercubeItem: arcrole = XbrlConst.all elif toConcept is not None and toConcept.isDimensionItem: arcrole = XbrlConst.hypercubeDimension elif fromConcept is not None and fromConcept.isDimensionItem: arcrole = XbrlConst.dimensionDomain else: arcrole = XbrlConst.domainMember else: arcrole = lbEntry.arcrole XmlUtil.addChild(parentElt, XbrlConst.link, lbType + "Arc", attributes=(("{http://www.w3.org/1999/xlink}type", "arc"), ("{http://www.w3.org/1999/xlink}arcrole", arcrole), ("{http://www.w3.org/1999/xlink}from", fromLabel), ("{http://www.w3.org/1999/xlink}to", toLabel), ("order", order)) + otherAttrs ) order += 1.0 if lbType != "calculation" or lbEntry.isRoot: lbTreeWalk(lbType, parentElt, lbEntry.childStruct, roleRefs, locs, toPrefix, toName)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in modelXbrl.contexts.values(): newCntx = targetInstance.createContext(context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) newFactForOldObjId[fact.objectIndex] = newFact if filingFiles and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src"): filingFiles.add(attrValue) except (XMLSyntaxError, UnicodeDecodeError): pass elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) # footnote links footnoteIdCount = {} modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype) and HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format(attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount XmlUtil.addChild(newLink, linkChild.qname, attributes=attributes, text=linkChild.textValue) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src"): filingFiles.add(attrValue) targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)