def createFact(self, conceptQname, attributes=None, text=None, parent=None, afterSibling=None, beforeSibling=None): if parent is None: parent = self.modelDocument.xmlRootElement newFact = XmlUtil.addChild(parent, conceptQname, attributes=attributes, text=text, afterSibling=afterSibling, beforeSibling=beforeSibling) self.modelDocument.factDiscover(newFact, parentElement=parent) XmlValidate.validate(self, newFact) return newFact
def contextDiscover(self, cntxElement): id = cntxElement.getAttribute("id") self.modelXbrl.contexts[id] = modelContext = ModelObject.createContext( self, cntxElement) for container in (("segment", modelContext.segDimValues, modelContext.segNonDimValues), ("scenario", modelContext.scenDimValues, modelContext.scenNonDimValues)): containerName, containerDimValues, containerNonDimValues = container for containerElement in XmlUtil.descendants( cntxElement, XbrlConst.xbrli, containerName): for sElt in containerElement.childNodes: if sElt.nodeType == 1: if sElt.namespaceURI == XbrlConst.xbrldi and sElt.localName in ( "explicitMember", "typedMember"): XmlValidate.validate(self.modelXbrl, sElt) modelDimValue = ModelObject.createDimensionValue( self, sElt) dimension = modelDimValue.dimension if dimension and dimension not in containerDimValues: containerDimValues[dimension] = modelDimValue else: modelContext.errorDimValues.append( modelDimValue) modelContext.qnameDims[ modelDimValue. dimensionQname] = modelDimValue # both seg and scen else: containerNonDimValues.append(sElt)
def attributeDict(modelXbrl, elt, exclusions=set(), equalMode=S_EQUAL, excludeIDs=NO_IDs_EXCLUDED, ns2ns1Tbl=None, keyByTag=False, distinguishNaNs=False): if not hasattr(elt,"xValid"): XmlValidate.validate(modelXbrl, elt) attrs = {} # TBD: replace with validated attributes for modelAttribute in elt.xAttributes.values(): attrTag = modelAttribute.attrTag ns, sep, localName = attrTag.partition('}') attrNsURI = ns[1:] if sep else None if ns2ns1Tbl and attrNsURI in ns2ns1Tbl: attrNsURI = ns2ns1Tbl[attrNsURI] if (attrTag not in exclusions and (attrNsURI is None or attrNsURI not in exclusions)): if keyByTag: qname = attrTag elif attrNsURI is not None: qname = QName(None, attrNsURI, localName) else: qname = QName(None, None, attrTag) try: if excludeIDs and modelAttribute.xValid == XmlValidate.VALID_ID: continue if modelAttribute.xValid != XmlValidate.UNKNOWN: value = modelAttribute.sValue if equalMode <= S_EQUAL2 else modelAttribute.xValue else: # unable to validate, no schema definition, use string value of attribute value = modelAttribute.text if distinguishNaNs and isinstance(value,float) and math.isnan(value): value = (value,elt) attrs[qname] = value except KeyError: pass # what should be done if attribute failed to have psvi value return attrs
def doc(xc, p, contextItem, args): if len(args) != 1: raise XPathContext.FunctionNumArgs() uri = stringArg(xc, args, 0, "xs:string", emptyFallback=None) if uri is None: return () if xc.progHeader is None or xc.progHeader.element is None: raise XPathContext.XPathException( p, 'err:FODC0005', _('Function xf:doc no formula resource element for {0}').format( uri)) if not UrlUtil.isValid(uri): raise XPathContext.XPathException( p, 'err:FODC0005', _('Function xf:doc $uri is not valid {0}').format(uri)) normalizedUri = xc.modelXbrl.modelManager.cntlr.webCache.normalizeUrl( uri, xc.progHeader.element.modelDocument.baseForElement( xc.progHeader.element)) if normalizedUri in xc.modelXbrl.urlDocs: return xc.modelXbrl.urlDocs[normalizedUri].xmlDocument modelDocument = ModelDocument.load(xc.modelXbrl, normalizedUri) if modelDocument is None: raise XPathContext.XPathException( p, 'err:FODC0005', _('Function xf:doc $uri not successfully loaded {0}').format(uri)) # assure that document is validated XmlValidate.validate(xc.modelXbrl, modelDocument.xmlRootElement) return modelDocument.xmlDocument
def sEqual(dts1, elt1, elt2, equalMode=S_EQUAL, excludeIDs=NO_IDs_EXCLUDED, dts2=None, ns2ns1Tbl=None): if dts2 is None: dts2 = dts1 if elt1.localName != elt2.localName: return False if ns2ns1Tbl and elt2.namespaceURI in ns2ns1Tbl: if elt1.namespaceURI != ns2ns1Tbl[elt2.namespaceURI]: return False elif elt1.namespaceURI != elt2.namespaceURI: return False if not hasattr(elt1,u"xValid"): XmlValidate.validate(dts1, elt1) if not hasattr(elt2,u"xValid"): XmlValidate.validate(dts2, elt2) children1 = childElements(elt1) children2 = childElements(elt2) if len(children1) != len(children2): return False if (not xEqual(elt1, elt2, # must use stringValue for nested contents of mixed content # ... this is now in xValue for mixed content # VALIDATE_BY_STRING_VALUE if len(children1) and elt1.xValid == VALID else equalMode ) or attributeDict(dts1, elt1, (), equalMode, excludeIDs) != attributeDict(dts2, elt2, (), equalMode, excludeIDs, ns2ns1Tbl)): return False excludeChildIDs = excludeIDs if excludeIDs != TOP_IDs_EXCLUDED else NO_IDs_EXCLUDED for i in xrange( len(children1) ): if not sEqual(dts1, children1[i], children2[i], equalMode, excludeChildIDs, dts2, ns2ns1Tbl): return False return True
def sEqual(dts1, elt1, elt2, equalMode=S_EQUAL, excludeIDs=NO_IDs_EXCLUDED, dts2=None, ns2ns1Tbl=None): if dts2 is None: dts2 = dts1 if elt1.localName != elt2.localName: return False if ns2ns1Tbl and elt2.namespaceURI in ns2ns1Tbl: if elt1.namespaceURI != ns2ns1Tbl[elt2.namespaceURI]: return False elif elt1.namespaceURI != elt2.namespaceURI: return False if not hasattr(elt1, "xValid"): XmlValidate.validate(dts1, elt1) if not hasattr(elt2, "xValid"): XmlValidate.validate(dts2, elt2) if (not xEqual(elt1, elt2, equalMode) or attributeDict(dts1, elt1, (), equalMode, excludeIDs) != attributeDict(dts2, elt2, (), equalMode, excludeIDs, ns2ns1Tbl)): return False children1 = childElements(elt1) children2 = childElements(elt2) if len(children1) != len(children2): return False excludeChildIDs = excludeIDs if excludeIDs != TOP_IDs_EXCLUDED else NO_IDs_EXCLUDED for i in range(len(children1)): if not sEqual(dts1, children1[i], children2[i], equalMode, excludeChildIDs, dts2, ns2ns1Tbl): return False return True
def equalityHash(elt, equalMode=S_EQUAL, excludeIDs=NO_IDs_EXCLUDED): if isinstance(elt, ModelObject): try: if equalMode == S_EQUAL: return elt._hashSEqual else: return elt._hashXpathEqual except AttributeError: dts = elt.modelXbrl if not hasattr(elt,"xValid"): XmlValidate.validate(dts, elt) hashableValue = elt.sValue if equalMode == S_EQUAL else elt.xValue if isinstance(hashableValue,float) and math.isnan(hashableValue): hashableValue = (hashableValue,elt) # ensure this NaN only compares to itself and no other NaN _hash = hash((elt.elementQname, hashableValue, tuple(attributeDict(dts, elt, (), equalMode, excludeIDs, distinguishNaNs=True).items()), tuple(equalityHash(child,equalMode,excludeIDs) for child in childElements(elt)) )) if equalMode == S_EQUAL: elt._hashSEqual = _hash else: elt._hashXpathEqual = _hash return _hash elif isinstance(elt, (tuple,list,set)): return hash( tuple(equalityHash(i) for i in elt) ) else: return hash(None)
def sEqual(dts1, elt1, elt2, equalMode=S_EQUAL, excludeIDs=NO_IDs_EXCLUDED, dts2=None, ns2ns1Tbl=None): if dts2 is None: dts2 = dts1 if elt1.localName != elt2.localName: return False if ns2ns1Tbl and elt2.namespaceURI in ns2ns1Tbl: if elt1.namespaceURI != ns2ns1Tbl[elt2.namespaceURI]: return False elif elt1.namespaceURI != elt2.namespaceURI: return False if not hasattr(elt1,"xValid"): XmlValidate.validate(dts1, elt1) if not hasattr(elt2,"xValid"): XmlValidate.validate(dts2, elt2) if (not xEqual(elt1, elt2, equalMode) or attributeDict(dts1, elt1, (), equalMode, excludeIDs) != attributeDict(dts2, elt2, (), equalMode, excludeIDs, ns2ns1Tbl)): return False children1 = childElements(elt1) children2 = childElements(elt2) if len(children1) != len(children2): return False excludeChildIDs = excludeIDs if excludeIDs != TOP_IDs_EXCLUDED else NO_IDs_EXCLUDED for i in range( len(children1) ): if not sEqual(dts1, children1[i], children2[i], equalMode, excludeChildIDs, dts2, ns2ns1Tbl): return False return True
def attributeDict(modelXbrl, elt, exclusions=set(), equalMode=S_EQUAL, excludeIDs=NO_IDs_EXCLUDED, ns2ns1Tbl=None, keyByTag=False, distinguishNaNs=False): if not hasattr(elt,"xValid"): XmlValidate.validate(modelXbrl, elt) attrs = {} # TBD: replace with validated attributes for modelAttribute in elt.xAttributes.values(): attrTag = modelAttribute.attrTag ns, sep, localName = attrTag.partition('}') attrNsURI = ns[1:] if sep else None if ns2ns1Tbl and attrNsURI in ns2ns1Tbl: attrNsURI = ns2ns1Tbl[attrNsURI] if (attrTag not in exclusions and (attrNsURI is None or attrNsURI not in exclusions)): if keyByTag: qname = attrTag elif attrNsURI is not None: qname = QName(None, attrNsURI, localName) else: qname = QName(None, None, attrTag) try: if excludeIDs and modelAttribute.xValid == XmlValidate.VALID_ID: continue value = modelAttribute.sValue if equalMode <= S_EQUAL2 else modelAttribute.xValue if distinguishNaNs and isinstance(value,float) and math.isnan(value): value = (value,elt) attrs[qname] = value except KeyError: pass # what should be done if attribute failed to have psvi value return attrs
def equalityHash(elt, equalMode=S_EQUAL, excludeIDs=NO_IDs_EXCLUDED): if isinstance(elt, ModelObject): try: if equalMode == S_EQUAL: return elt._hashSEqual else: return elt._hashXpathEqual except AttributeError: dts = elt.modelXbrl if not hasattr(elt, "xValid"): XmlValidate.validate(dts, elt) hashableValue = elt.sValue if equalMode == S_EQUAL else elt.xValue if isinstance(hashableValue, float) and math.isnan(hashableValue): hashableValue = ( hashableValue, elt ) # ensure this NaN only compares to itself and no other NaN _hash = hash((elt.elementQname, hashableValue, tuple( attributeDict(dts, elt, (), equalMode, excludeIDs, distinguishNaNs=True).items()), tuple( equalityHash(child, equalMode, excludeIDs) for child in childElements(elt)))) if equalMode == S_EQUAL: elt._hashSEqual = _hash else: elt._hashXpathEqual = _hash return _hash elif isinstance(elt, (tuple, list, set)): return hash(tuple(equalityHash(i) for i in elt)) else: return hash(None)
def createUnit(self, multiplyBy, divideBy, afterSibling=None, beforeSibling=None): """Creates new unit, by measures, as in formula usage, if any :param multiplyBy: List of multiply-by measure QNames (or top level measures if no divideBy) :type multiplyBy: [QName] :param divideBy: List of multiply-by measure QNames (or empty list if no divideBy) :type divideBy: [QName] :param beforeSibling: lxml element in instance to insert new concept before :type beforeSibling: ModelObject :param afterSibling: lxml element in instance to insert new concept after :type afterSibling: ModelObject :returns: ModelUnit -- New unit object """ xbrlElt = self.modelDocument.xmlRootElement if afterSibling == AUTO_LOCATE_ELEMENT: afterSibling = XmlUtil.lastChild(xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context", "unit")) unitId = 'u-{0:02n}'.format( len(self.units) + 1) newUnitElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId), afterSibling=afterSibling, beforeSibling=beforeSibling) if len(divideBy) == 0: for multiply in multiplyBy: XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) else: divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide") numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator") denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator") for multiply in multiplyBy: XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) for divide in divideBy: XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, divide)) self.modelDocument.unitDiscover(newUnitElt) XmlValidate.validate(self, newUnitElt) return newUnitElt
def xfxc_element(xc, p, contextItem, args): if not 2 <= len(args) <= 4: raise XPathContext.FunctionNumArgs() qn = qnameArg(xc, p, args, 0, 'QName', emptyFallback=None) attrArg = args[1] if isinstance(args[1], (list, tuple)) else (args[1], ) # attributes have to be pairs if attrArg: if len(attrArg) & 1 or any( not isinstance(attrArg[i], (QName, _STR_BASE)) for i in range(0, len(attrArg), 2)): raise XPathContext.FunctionArgType( 1, "((xs:qname|xs:string),xs:anyAtomicValue)", errCode="xfxce:AttributesNotNameValuePairs") else: attrParam = [ (attrArg[i], attrArg[i + 1] ) # need name-value pairs for XmlUtil function for i in range(0, len(attrArg), 2) ] else: attrParam = None value = atomicArg(xc, p, args, 2, "xs:anyAtomicType", emptyFallback='') if not value: # be sure '' is None so no text node is created value = None if len(args) < 4: childElements = None else: childElements = xc.flattenSequence(args[3]) # scratchpad instance document emulates fn:doc( ) to hold XML nodes scratchpadXmlDocUrl = "http://www.xbrl.org/2012/function/creation/xml_scratchpad.xml" if scratchpadXmlDocUrl in xc.modelXbrl.urlDocs: modelDocument = xc.modelXbrl.urlDocs[scratchpadXmlDocUrl] else: # create scratchpad xml document # this will get the fake instance document in the list of modelXbrl docs so that it is garbage collected from arelle import ModelDocument modelDocument = ModelDocument.create( xc.modelXbrl, ModelDocument.Type.UnknownXML, scratchpadXmlDocUrl, initialXml= "<xfc:dummy xmlns:xfc='http://www.xbrl.org/2012/function/creation'/>" ) newElement = XmlUtil.addChild(modelDocument.xmlRootElement, qn, attributes=attrParam, text=value) if childElements: for element in childElements: if isinstance(element, etree.ElementBase): newElement.append(element) # node myst be validated for use in instance creation (typed dimension references) XmlValidate.validate(xc.modelXbrl, newElement) return newElement
def instanceDiscover(self, xbrlElement): self.schemaLinkbaseRefsDiscover(xbrlElement) self.linkbaseDiscover( xbrlElement, inInstance=True) # for role/arcroleRefs and footnoteLinks self.instanceContentsDiscover(xbrlElement) XmlValidate.validate(self.modelXbrl, xbrlElement) # validate instance elements
def usedOns(self): try: return self._usedOns except AttributeError: XmlValidate.validate(self.modelXbrl, self) self._usedOns = set(usedOn.xValue for usedOn in self.iterdescendants("{http://www.xbrl.org/2003/linkbase}usedOn") if isinstance(usedOn,ModelObject)) return self._usedOns
def xEqual(elt1, elt2, equalMode=S_EQUAL): if not hasattr(elt1,"xValid"): XmlValidate.validate(elt1.modelXbrl, elt1) if not hasattr(elt2,"xValid"): XmlValidate.validate(elt2.modelXbrl, elt2) if equalMode == S_EQUAL or (equalMode == S_EQUAL2 and not isinstance(elt1.sValue, QName)): return elt1.sValue == elt2.sValue else: return elt1.xValue == elt2.xValue
def usedOns(self): try: return self._usedOns except AttributeError: XmlValidate.validate(self.modelXbrl, self) self._usedOns = set( usedOn.xValue for usedOn in self.iterdescendants( "{http://www.xbrl.org/2003/linkbase}usedOn") if isinstance(usedOn, ModelObject)) return self._usedOns
def checkAttribute(elt, isIxElt, attrTag, attrValue): ixEltAttrDefs = ixAttrDefined.get(elt.namespaceURI, EMPTYDICT).get(elt.localName, ()) if attrTag.startswith("{"): ns, sep, localName = attrTag[1:].partition("}") else: ns = None localName = attrTag if ns is not None and ns not in XbrlConst.ixbrlAll and attrTag not in ixEltAttrDefs: if isIxElt: allowedNs = allowedNonIxAttrNS.get(elt.localName, None) if allowedNs != "##other" and ns != allowedNs: modelXbrl.error(ixMsgCode("qualifiedAttributeNotExpected", elt), _("Inline XBRL element %(element)s has qualified attribute %(name)s"), modelObject=elt, element=str(elt.elementQname), name=attrTag) if ns == XbrlConst.xbrli and elt.localName in { "fraction", "nonFraction", "nonNumeric", "references", "relationship", "tuple"}: modelXbrl.error(ixMsgCode("qualifiedAttributeDisallowed", elt), _("Inline XBRL element %(element)s has disallowed attribute %(name)s"), modelObject=elt, element=str(elt.elementQname), name=attrTag) else: if ns in XbrlConst.ixbrlAll: modelXbrl.error(ixMsgCode("inlineAttributeMisplaced", elt, name="other"), _("Inline XBRL attributes are not allowed on html elements: ix:%(name)s"), modelObject=elt, name=localName) elif ns not in {XbrlConst.xml, XbrlConst.xsi, XbrlConst.xhtml}: modelXbrl.error(ixMsgCode("extensionAttributeMisplaced", ns=_ixNS), _("Extension attributes are not allowed on html elements: %(tag)s"), modelObject=elt, tag=attrTag) elif isIxElt: try: _xsdType = ixAttrType[elt.namespaceURI][localName] if isinstance(_xsdType, dict): baseXsdType = _xsdType["type"] facets = _xsdType else: baseXsdType = _xsdType facets = None XmlValidate.validateValue(modelXbrl, elt, attrTag, baseXsdType, attrValue, facets=facets) if not (attrTag in ixEltAttrDefs or (localName in ixEltAttrDefs and (not ns or ns in XbrlConst.ixbrlAll))): raise KeyError disallowedXbrliAttrs = ({"scheme", "periodType", "balance", "contextRef", "unitRef", "precision", "decimals"} - {"fraction": {"contextRef", "unitRef"}, "nonFraction": {"contextRef", "unitRef", "decimals", "precision"}, "nonNumeric": {"contextRef"}}.get(elt.localName, set())) disallowedAttrs = set(a for a in disallowedXbrliAttrs if elt.get(a) is not None) if disallowedAttrs: modelXbrl.error(ixMsgCode("inlineElementAttributes",elt), _("Inline XBRL element %(element)s has disallowed attributes %(attributes)s"), modelObject=elt, element=elt.elementQname, attributes=", ".join(disallowedAttrs)) except KeyError: modelXbrl.error(ixMsgCode("attributeNotExpected",elt), _("Attribute %(attribute)s is not expected on element ix:%(element)s"), modelObject=elt, attribute=attrTag, element=elt.localName)
def xEqual(elt1, elt2, equalMode=S_EQUAL): if not hasattr(elt1,u"xValid"): XmlValidate.validate(elt1.modelXbrl, elt1) if not hasattr(elt2,u"xValid"): XmlValidate.validate(elt2.modelXbrl, elt2) if equalMode == VALIDATE_BY_STRING_VALUE: return elt1.stringValue == elt2.stringValue elif equalMode == S_EQUAL or (equalMode == S_EQUAL2 and not isinstance(elt1.sValue, QName)): return elt1.sValue == elt2.sValue else: return elt1.xValue == elt2.xValue
def typedValue(dts, element, attrQname=None): try: if attrQname: node = element.getAttributeNodeNS(attrQname.namespaceURI,attrQname.localName) else: node = element if node.xValid == XmlValidate.VALID: return node.xValue except AttributeError: if dts: XmlValidate.validate(dts, element, recurse=False, attrQname=attrQname) return typedValue(None, element, attrQname=attrQname) return None
def typedValue(dts, element, attrQname=None): try: if attrQname: # PSVI attribute value modelAttribute = element.xAttributes[attrQname.clarkNotation] if modelAttribute.xValid >= XmlValidate.VALID: return modelAttribute.xValue else: # PSVI element value (of text) if element.xValid >= XmlValidate.VALID: return element.xValue except (AttributeError, KeyError): if dts: XmlValidate.validate(dts, element, recurse=False, attrQname=attrQname) return typedValue(None, element, attrQname=attrQname) return None
def xfxc_element(xc, p, contextItem, args): if not 2 <= len(args) <= 4: raise XPathContext.FunctionNumArgs() qn = qnameArg(xc, p, args, 0, 'QName', emptyFallback=None) attrArg = args[1] if isinstance(args[1],(list,tuple)) else (args[1],) # attributes have to be pairs if attrArg: if len(attrArg) & 1 or any(not isinstance(attrArg[i], (QName, _STR_BASE)) for i in range(0, len(attrArg),2)): raise XPathContext.FunctionArgType(1,"((xs:qname|xs:string),xs:anyAtomicValue)", errCode="xfxce:AttributesNotNameValuePairs") else: attrParam = [(attrArg[i],attrArg[i+1]) # need name-value pairs for XmlUtil function for i in range(0, len(attrArg),2)] else: attrParam = None value = atomicArg(xc, p, args, 2, "xs:anyAtomicType", emptyFallback='') if not value: # be sure '' is None so no text node is created value = None if len(args) < 4: childElements = None else: childElements = xc.flattenSequence(args[3]) # scratchpad instance document emulates fn:doc( ) to hold XML nodes scratchpadXmlDocUrl = "http://www.xbrl.org/2012/function/creation/xml_scratchpad.xml" if scratchpadXmlDocUrl in xc.modelXbrl.urlDocs: modelDocument = xc.modelXbrl.urlDocs[scratchpadXmlDocUrl] else: # create scratchpad xml document # this will get the fake instance document in the list of modelXbrl docs so that it is garbage collected from arelle import ModelDocument modelDocument = ModelDocument.create(xc.modelXbrl, ModelDocument.Type.UnknownXML, scratchpadXmlDocUrl, initialXml="<xfc:dummy xmlns:xfc='http://www.xbrl.org/2012/function/creation'/>") newElement = XmlUtil.addChild(modelDocument.xmlRootElement, qn, attributes=attrParam, text=value) if childElements: for element in childElements: if isinstance(element, etree.ElementBase): newElement.append(element) # node myst be validated for use in instance creation (typed dimension references) XmlValidate.validate(xc.modelXbrl, newElement) return newElement
def constrainingFacets(self, facetValues=None): facetValues = facetValues if facetValues else {} for facetElt in XmlUtil.schemaFacets( self, ("{http://www.w3.org/2001/XMLSchema}length", "{http://www.w3.org/2001/XMLSchema}minLength", "{http://www.w3.org/2001/XMLSchema}maxLength", "{http://www.w3.org/2001/XMLSchema}pattern", "{http://www.w3.org/2001/XMLSchema}whiteSpace", "{http://www.w3.org/2001/XMLSchema}maxInclusive", "{http://www.w3.org/2001/XMLSchema}maxExclusive", "{http://www.w3.org/2001/XMLSchema}minExclusive", "{http://www.w3.org/2001/XMLSchema}totalDigits", "{http://www.w3.org/2001/XMLSchema}fractionDigits")): facetValue = XmlValidate.validateFacet(self, facetElt) facetName = facetElt.localName if facetName not in facetValues and facetValue is not None: # facetValue can be zero but not None facetValues[facetName] = facetValue if "enumeration" not in facetValues: for facetElt in XmlUtil.schemaFacets( self, ("{http://www.w3.org/2001/XMLSchema}enumeration", )): facetValues.setdefault("enumeration", set()).add(facetElt.get("value")) typeDerivedFrom = self.typeDerivedFrom if isinstance(typeDerivedFrom, ModelType): typeDerivedFrom.constrainingFacets(facetValues) return facetValues
def typedValue(dts, element, attrQname=None): try: if attrQname: node = element.getAttributeNodeNS(attrQname.namespaceURI, attrQname.localName) else: node = element if node.xValid == XmlValidate.VALID: return node.xValue except AttributeError: if dts: XmlValidate.validate(dts, element, recurse=False, attrQname=attrQname) return typedValue(None, element, attrQname=attrQname) return None
def checkAttribute(elt, isIxElt, attrTag, attrValue): if attrTag.startswith("{"): ns, sep, localName = attrTag[1:].partition("}") if isIxElt: allowedNs = nonIxAttrNS.get(elt.localName, None) if allowedNs != "##other" and ns != allowedNs: modelXbrl.error("ix:qualifiedAttributeNotExpected", _("Inline XBRL element %(element)s: has qualified attribute %(name)s"), modelObject=elt, element=str(elt.elementQname), name=attrTag) else: if ns in XbrlConst.ixbrlAll: modelXbrl.error("ix:inlineAttributeMisplaced", _("Inline XBRL attributes are not allowed on html elements: ix:%(name)s"), modelObject=elt, name=localName) elif ns not in {XbrlConst.xml, XbrlConst.xsi, XbrlConst.xhtml}: modelXbrl.error("ix:extensionAttributeMisplaced", _("Extension attributes are not allowed on html elements: %(tag)s"), modelObject=elt, tag=attrTag) elif isIxElt: try: _xsdType = ixAttrType[elt.namespaceURI][attrTag] if isinstance(_xsdType, dict): baseXsdType = _xsdType["type"] facets = _xsdType else: baseXsdType = _xsdType facets = None XmlValidate.validateValue(modelXbrl, elt, attrTag, baseXsdType, attrValue, facets=facets) disallowedXbrliAttrs = ({"scheme", "periodType", "balance", "contextRef", "unitRef", "precision", "decimals"} - {"fraction": {"contextRef", "unitRef"}, "nonFraction": {"contextRef", "unitRef", "decimals", "precision"}, "nonNumeric": {"contextRef"}}.get(elt.localName, set())) disallowedAttrs = [a for a in disallowedXbrliAttrs if elt.get(a) is not None] if disallowedAttrs: modelXbrl.error("ix:inlineElementAttributes", _("Inline XBRL element %(element)s has disallowed attributes %(attributes)s"), modelObject=elt, element=elt.elementQname, attributes=", ".join(disallowedAttrs)) except KeyError: modelXbrl.error("ix:attributeNotExpected", _("Attribute %(attribute)s is not expected on element element ix:%(element)s"), modelObject=elt, attribute=attrTag, element=elt.localName)
def contextDiscover(self, cntxElement): id = cntxElement.getAttribute("id") self.modelXbrl.contexts[id] = modelContext = ModelObject.createContext(self,cntxElement) for container in (("segment", modelContext.segDimValues, modelContext.segNonDimValues), ("scenario", modelContext.scenDimValues, modelContext.scenNonDimValues)): containerName, containerDimValues, containerNonDimValues = container for containerElement in XmlUtil.descendants(cntxElement, XbrlConst.xbrli, containerName): for sElt in containerElement.childNodes: if sElt.nodeType == 1: if sElt.namespaceURI == XbrlConst.xbrldi and sElt.localName in ("explicitMember","typedMember"): XmlValidate.validate(self.modelXbrl, sElt) modelDimValue = ModelObject.createDimensionValue(self,sElt) dimension = modelDimValue.dimension if dimension and dimension not in containerDimValues: containerDimValues[dimension] = modelDimValue else: modelContext.errorDimValues.append(modelDimValue) modelContext.qnameDims[modelDimValue.dimensionQname] = modelDimValue # both seg and scen else: containerNonDimValues.append(sElt)
def endElementNS(self, name, qname): thisQname = QName(None, *name) if self.qnameStack and self.qnameStack[0].elementQname == thisQname: elt = self.qnameStack.pop(0) if elt.namespaceURI == XbrlConst.xbrli: if elt.localName == "unit": elt._measures = (sorted(elt._measures[0]), sorted(elt._measures[1])) if elt.id in self.unitRefedFacts: for fact in self.unitRefedFacts[elt.id]: fact._unit = elt del self.unitRefedFacts[elt.id] elif elt.localName == "context": if elt.id in self.contextRefedFacts: for fact in self.contextRefedFacts[elt.id]: fact._context = elt del self.contextRefedFacts[elt.id] self.currentNamespaceURI = None self.currentLocalName = None XmlValidate.validate(self.modelXbrl, elt, recurse=False) pass
def doc(xc, p, contextItem, args): if len(args) != 1: raise XPathContext.FunctionNumArgs() uri = stringArg(xc, args, 0, "xs:string", emptyFallback=None) if uri is None: return () if xc.progHeader is None or xc.progHeader.element is None: raise XPathContext.XPathException(p, 'err:FODC0005', _('Function xf:doc no formula resource element for {0}').format(uri)) if not UrlUtil.isValid(uri): raise XPathContext.XPathException(p, 'err:FODC0005', _('Function xf:doc $uri is not valid {0}').format(uri)) normalizedUri = xc.modelXbrl.modelManager.cntlr.webCache.normalizeUrl( uri, xc.progHeader.element.modelDocument.baseForElement(xc.progHeader.element)) if normalizedUri in xc.modelXbrl.urlDocs: return xc.modelXbrl.urlDocs[normalizedUri].xmlDocument modelDocument = ModelDocument.load(xc.modelXbrl, normalizedUri) if modelDocument is None: raise XPathContext.XPathException(p, 'err:FODC0005', _('Function xf:doc $uri not successfully loaded {0}').format(uri)) # assure that document is validated XmlValidate.validate(xc.modelXbrl, modelDocument.xmlRootElement) return modelDocument.xmlDocument
def checkAttribute(elt, isIxElt, attrTag, attrValue): if attrTag.startswith("{"): ns, sep, localName = attrTag[1:].partition("}") if isIxElt: if ns not in (XbrlConst.xml, XbrlConst.xsi): modelXbrl.error("ix:qualifiedAttributeNotExpected", _("Inline XBRL element %(element)s: has qualified attribute %(name)s"), modelObject=elt, element=str(elt.elementQname), name=attrTag) else: if ns in XbrlConst.ixbrlAll: modelXbrl.error("ix:inlineAttributeMisplaced", _("Inline XBRL attributes are not allowed on html elements: ix:%(name)s"), modelObject=elt, name=localName) elif ns not in {XbrlConst.xml, XbrlConst.xsi, XbrlConst.xhtml}: modelXbrl.error("ix:extensionAttributeMisplaced", _("Extension attributes are not allowed on html elements: %(tag)s"), modelObject=elt, tag=attrTag) elif isIxElt: try: _xsdType = ixAttrType[elt.namespaceURI][attrTag] if isinstance(_xsdType, dict): baseXsdType = _xsdType["type"] facets = _xsdType else: baseXsdType = _xsdType facets = None XmlValidate.validateValue(modelXbrl, elt, attrTag, baseXsdType, attrValue, facets=facets) disallowedXbrliAttrs = ({"scheme", "periodType", "balance", "contextRef", "unitRef", "precision", "decimals"} - {"fraction": {"contextRef", "unitRef"}, "nonFraction": {"contextRef", "unitRef", "decimals", "precision"}, "nonNumeric": {"contextRef"}}.get(elt.localName, set())) disallowedAttrs = [a for a in disallowedXbrliAttrs if elt.get(a) is not None] if disallowedAttrs: modelXbrl.error("ix:inlineElementAttributes", _("Inline XBRL element %(element)s has disallowed attributes %(attributes)s"), modelObject=elt, element=elt.elementQname, attributes=", ".join(disallowedAttrs)) except KeyError: modelXbrl.error("ix:attributeNotExpected", _("Attribute %(attribute)s is not expected on element element ix:%(element)s"), modelObject=elt, attribute=attrTag, element=elt.localName)
def createUnit(self, multiplyBy, divideBy, afterSibling=None, beforeSibling=None): xbrlElt = self.modelDocument.xmlRootElement if afterSibling == AUTO_LOCATE_ELEMENT: afterSibling = XmlUtil.lastChild( xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context", "unit")) unitId = 'u-{0:02n}'.format(len(self.units) + 1) newUnitElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId), afterSibling=afterSibling, beforeSibling=beforeSibling) if len(divideBy) == 0: for multiply in multiplyBy: XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) else: divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide") numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator") denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator") for multiply in multiplyBy: XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) for divide in divideBy: XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, divide)) self.modelDocument.unitDiscover(newUnitElt) XmlValidate.validate(self, newUnitElt) return newUnitElt
def constrainingFacets(self, facetValues=None): facetValues = facetValues if facetValues else {} for facetElt in XmlUtil.schemaFacets(self, ( "{http://www.w3.org/2001/XMLSchema}length", "{http://www.w3.org/2001/XMLSchema}minLength", "{http://www.w3.org/2001/XMLSchema}maxLength", "{http://www.w3.org/2001/XMLSchema}pattern", "{http://www.w3.org/2001/XMLSchema}whiteSpace", "{http://www.w3.org/2001/XMLSchema}maxInclusive", "{http://www.w3.org/2001/XMLSchema}maxExclusive", "{http://www.w3.org/2001/XMLSchema}minExclusive", "{http://www.w3.org/2001/XMLSchema}totalDigits", "{http://www.w3.org/2001/XMLSchema}fractionDigits")): XmlValidate.validateFacet(self, facetElt) facetName = facetElt.localName if facetName not in facetValues: facetValue = XmlValidate.validateFacet(self, facetElt) if facetValue: facetValues[facetName] = facetValue if "enumeration" not in facetValues: for facetElt in XmlUtil.schemaFacets(self, ("{http://www.w3.org/2001/XMLSchema}enumeration",)): facetValues.setdefault("enumeration",set()).add(facetElt.get("value")) typeDerivedFrom = self.typeDerivedFrom if typeDerivedFrom is not None: typeDerivedFrom.constrainingFacets(facetValues) return facetValues
def createUnit(self, multiplyBy, divideBy, afterSibling=None, beforeSibling=None): xbrlElt = self.modelDocument.xmlRootElement if afterSibling == AUTO_LOCATE_ELEMENT: afterSibling = XmlUtil.lastChild(xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context", "unit")) unitId = 'u-{0:02n}'.format( len(self.units) + 1) newUnitElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId), afterSibling=afterSibling, beforeSibling=beforeSibling) if len(divideBy) == 0: for multiply in multiplyBy: XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) else: divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide") numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator") denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator") for multiply in multiplyBy: XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) for divide in divideBy: XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, divide)) self.modelDocument.unitDiscover(newUnitElt) XmlValidate.validate(self, newUnitElt) return newUnitElt
def sEqual(dts1, elt1, elt2, equalMode=S_EQUAL, excludeIDs=NO_IDs_EXCLUDED, dts2=None, ns2ns1Tbl=None): if dts2 is None: dts2 = dts1 if elt1.localName != elt2.localName: return False if ns2ns1Tbl and elt2.namespaceURI in ns2ns1Tbl: if elt1.namespaceURI != ns2ns1Tbl[elt2.namespaceURI]: return False elif elt1.namespaceURI != elt2.namespaceURI: return False if not hasattr(elt1, "xValid"): XmlValidate.validate(dts1, elt1) if not hasattr(elt2, "xValid"): XmlValidate.validate(dts2, elt2) children1 = childElements(elt1) children2 = childElements(elt2) if len(children1) != len(children2): return False if (not xEqual( elt1, elt2, # must use stringValue for nested contents of mixed content # ... this is now in xValue for mixed content # VALIDATE_BY_STRING_VALUE if len(children1) and elt1.xValid == VALID else equalMode) or attributeDict(dts1, elt1, (), equalMode, excludeIDs) != attributeDict(dts2, elt2, (), equalMode, excludeIDs, ns2ns1Tbl)): return False excludeChildIDs = excludeIDs if excludeIDs != TOP_IDs_EXCLUDED else NO_IDs_EXCLUDED for i in range(len(children1)): if not sEqual(dts1, children1[i], children2[i], equalMode, excludeChildIDs, dts2, ns2ns1Tbl): return False return True
def createFact(self, conceptQname, attributes=None, text=None, parent=None, afterSibling=None, beforeSibling=None): """Creates new fact, as in formula output instance creation, and validates into object model :param conceptQname: QNames of concept :type conceptQname: QName :param attributes: Tuple of name, value, or tuples of name, value tuples (name,value) or ((name,value)[,(name,value...)]), where name is either QName or clark-notation name string :param text: Text content of fact :type text: str :param parent: lxml element in instance to append as child of :type parent: ModelObject :param beforeSibling: lxml element in instance to insert new concept before :type beforeSibling: ModelObject :param afterSibling: lxml element in instance to insert new concept after :type afterSibling: ModelObject :returns: ModelFact -- New fact object """ if parent is None: parent = self.modelDocument.xmlRootElement newFact = XmlUtil.addChild(parent, conceptQname, attributes=attributes, text=text, afterSibling=afterSibling, beforeSibling=beforeSibling) self.modelDocument.factDiscover(newFact, parentElement=parent) XmlValidate.validate(self, newFact) return newFact
def inlineXbrlDiscover(self, htmlElement): if htmlElement.namespaceURI == XbrlConst.xhtml: # must validate xhtml #load(self.modelXbrl, "http://www.w3.org/2002/08/xhtml/xhtml1-strict.xsd") XmlValidate.xhtmlValidate(self.modelXbrl, htmlElement) # fails on prefixed content for inlineElement in htmlElement.iterdescendants( tag="{http://www.xbrl.org/2008/inlineXBRL}references"): self.schemaLinkbaseRefsDiscover(inlineElement) XmlValidate.validate(self.modelXbrl, inlineElement) # validate instance elements for inlineElement in htmlElement.iterdescendants( tag="{http://www.xbrl.org/2008/inlineXBRL}resources"): self.instanceContentsDiscover(inlineElement) XmlValidate.validate(self.modelXbrl, inlineElement) # validate instance elements tupleElements = [] tuplesByTupleID = {} for modelInlineTuple in htmlElement.iterdescendants( tag="{http://www.xbrl.org/2008/inlineXBRL}tuple"): if isinstance(modelInlineTuple, ModelObject): modelInlineTuple.unorderedTupleFacts = [] if modelInlineTuple.tupleID: tuplesByTupleID[ modelInlineTuple.tupleID] = modelInlineTuple tupleElements.append(modelInlineTuple) # hook up tuples to their container for tupleFact in tupleElements: self.inlineXbrlLocateFactInTuple(tupleFact, tuplesByTupleID) for tag in ("{http://www.xbrl.org/2008/inlineXBRL}nonNumeric", "{http://www.xbrl.org/2008/inlineXBRL}nonFraction", "{http://www.xbrl.org/2008/inlineXBRL}fraction"): for modelInlineFact in htmlElement.iterdescendants(tag=tag): if isinstance(modelInlineFact, ModelObject): self.inlineXbrlLocateFactInTuple(modelInlineFact, tuplesByTupleID) # order tuple facts for tupleFact in tupleElements: tupleFact.modelTupleFacts = [ self.modelXbrl.modelObject(objectIndex) for order, objectIndex in sorted(tupleFact.unorderedTupleFacts) ] # validate particle structure of elements after transformations and established tuple structure for rootModelFact in self.modelXbrl.facts: XmlValidate.validate(self.modelXbrl, rootModelFact, ixFacts=True)
def inlineXbrlDiscover(self, htmlElement): if htmlElement.namespaceURI == XbrlConst.xhtml: # must validate xhtml #load(self.modelXbrl, "http://www.w3.org/2002/08/xhtml/xhtml1-strict.xsd") XmlValidate.xhtmlValidate(self.modelXbrl, htmlElement) # fails on prefixed content for inlineElement in htmlElement.iterdescendants(tag="{http://www.xbrl.org/2008/inlineXBRL}references"): self.schemaLinkbaseRefsDiscover(inlineElement) XmlValidate.validate(self.modelXbrl, inlineElement) # validate instance elements for inlineElement in htmlElement.iterdescendants(tag="{http://www.xbrl.org/2008/inlineXBRL}resources"): self.instanceContentsDiscover(inlineElement) XmlValidate.validate(self.modelXbrl, inlineElement) # validate instance elements tupleElements = [] tuplesByTupleID = {} for modelInlineTuple in htmlElement.iterdescendants(tag="{http://www.xbrl.org/2008/inlineXBRL}tuple"): if isinstance(modelInlineTuple,ModelObject): modelInlineTuple.unorderedTupleFacts = [] if modelInlineTuple.tupleID: tuplesByTupleID[modelInlineTuple.tupleID] = modelInlineTuple tupleElements.append(modelInlineTuple) # hook up tuples to their container for tupleFact in tupleElements: self.inlineXbrlLocateFactInTuple(tupleFact, tuplesByTupleID) for tag in ("{http://www.xbrl.org/2008/inlineXBRL}nonNumeric", "{http://www.xbrl.org/2008/inlineXBRL}nonFraction", "{http://www.xbrl.org/2008/inlineXBRL}fraction"): for modelInlineFact in htmlElement.iterdescendants(tag=tag): if isinstance(modelInlineFact,ModelObject): self.inlineXbrlLocateFactInTuple(modelInlineFact, tuplesByTupleID) # order tuple facts for tupleFact in tupleElements: tupleFact.modelTupleFacts = [ self.modelXbrl.modelObject(objectIndex) for order,objectIndex in sorted(tupleFact.unorderedTupleFacts)] # validate particle structure of elements after transformations and established tuple structure for rootModelFact in self.modelXbrl.facts: XmlValidate.validate(self.modelXbrl, rootModelFact, ixFacts=True)
def backgroundSaveInstance(self, newFilename=None): cntlr = self.modelXbrl.modelManager.cntlr if newFilename: self.modelXbrl.modelManager.showStatus( _("creating new instance {0}").format( os.path.basename(newFilename))) self.modelXbrl.modelManager.cntlr.waitForUiThreadQueue( ) # force status update self.modelXbrl.createInstance( newFilename ) # creates an instance as this modelXbrl's entrypoing instance = self.modelXbrl cntlr.showStatus( _("Saving {0}").format(instance.modelDocument.basename)) cntlr.waitForUiThreadQueue() # force status update newCntx = ModelXbrl.AUTO_LOCATE_ELEMENT newUnit = ModelXbrl.AUTO_LOCATE_ELEMENT # check user keyed changes for bodyCell in self.gridBody.winfo_children(): if isinstance(bodyCell, gridCell) and bodyCell.isChanged: value = bodyCell.value objId = bodyCell.objectId if objId: if objId[0] == "f": factPrototypeIndex = int(objId[1:]) factPrototype = self.factPrototypes[factPrototypeIndex] concept = factPrototype.concept entityIdentScheme = self.newFactItemOptions.entityIdentScheme entityIdentValue = self.newFactItemOptions.entityIdentValue periodType = factPrototype.concept.periodType periodStart = self.newFactItemOptions.startDateDate if periodType == "duration" else None periodEndInstant = self.newFactItemOptions.endDateDate qnameDims = factPrototype.context.qnameDims prevCntx = instance.matchContext( entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, qnameDims, [], []) if prevCntx is not None: cntxId = prevCntx.id else: # need new context newCntx = instance.createContext( entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, concept.qname, qnameDims, [], [], afterSibling=newCntx) cntxId = newCntx.id # new context if concept.isNumeric: if concept.isMonetary: unitMeasure = qname( XbrlConst.iso4217, self.newFactItemOptions.monetaryUnit) unitMeasure.prefix = "iso4217" # want to save with a recommended prefix decimals = self.newFactItemOptions.monetaryDecimals elif concept.isShares: unitMeasure = XbrlConst.qnXbrliShares decimals = self.newFactItemOptions.nonMonetaryDecimals else: unitMeasure = XbrlConst.qnXbrliPure decimals = self.newFactItemOptions.nonMonetaryDecimals prevUnit = instance.matchUnit([unitMeasure], []) if prevUnit is not None: unitId = prevUnit.id else: newUnit = instance.createUnit( [unitMeasure], [], afterSibling=newUnit) unitId = newUnit.id attrs = [("contextRef", cntxId)] if concept.isNumeric: attrs.append(("unitRef", unitId)) attrs.append(("decimals", decimals)) value = Locale.atof(self.modelXbrl.locale, value, str.strip) newFact = instance.createFact(concept.qname, attributes=attrs, text=value) bodyCell.objectId = newFact.objectId( ) # switch cell to now use fact ID if self.factPrototypes[factPrototypeIndex] is not None: self.factPrototypes[factPrototypeIndex].clear() self.factPrototypes[ factPrototypeIndex] = None #dereference fact prototype else: # instance fact, not prototype fact = self.modelXbrl.modelObject(objId) if fact.concept.isNumeric: value = Locale.atof(self.modelXbrl.locale, value, str.strip) if fact.value != value: fact.text = value XmlValidate.validate(instance, fact) bodyCell.isChanged = False # clear change flag instance.saveInstance() cntlr.showStatus(_("Saved {0}").format( instance.modelDocument.basename), clearAfter=3000)
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, *args, **kwargs): # check if big instance and has header with an initial incomplete tree walk (just 2 elements if not _streamingExtensionsCheck: return None # track whether modelXbrl has been validated by this streaming extension modelXbrl._streamingExtensionValidated = False def logSyntaxErrors(parsercontext): for error in parsercontext.error_log: modelXbrl.error("xmlSchema:syntax", _("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"), modelObject=modelXbrl, fileName=os.path.basename(filepath), error=error.message, line=error.line, column=error.column, sourceAction="streaming") #### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ### #### note: iterparse wants a binary file, but file is text mode _file, = modelXbrl.fileSource.file(filepath, binary=True) startedAt = time.time() modelXbrl.profileActivity() ''' this seems twice as slow as iterparse class instInfoTarget(): def __init__(self, element_factory=None, parser=None): self.newTree = True self.streamingAspects = None self.foundInstance = False self.creationSoftwareComment = '' self.currentEltTag = "(before xbrli:xbrl)" self.numRootFacts = 0 def start(self, tag, attrib, nsmap=None): if self.newTree: if tag == "{http://www.xbrl.org/2003/instance}xbrl": self.foundInstance = True self.newTree = False else: # break raise NotInstanceDocumentException() elif not tag.startswith("{http://www.xbrl.org/"): self.numRootFacts += 1 if self.numRootFacts % 1000 == 0: modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0) self.currentEltTag = tag def end(self, tag): pass def data(self, data): pass def comment(self, text): if not self.foundInstance: # accumulate comments before xbrli:xbrl self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text elif not self.creationSoftwareComment: self.creationSoftwareComment = text # or first comment after xbrli:xbrl def pi(self, target, data): if target == "xbrl-streamable-instance": if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl": self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results else: modelXbrl.error("streamingExtensions:headerMisplaced", _("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"), modelObject=modelXbrl, target=target, element=self.currentEltTag) def close(self): if not self.creationSoftwareComment: self.creationSoftwareComment = None return True instInfo = instInfoTarget() infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo) try: etree.parse(_file, parser=infoParser, base_url=filepath) except NotInstanceDocumentException: pass ''' foundErrors = False foundInstance = False streamingAspects = None creationSoftwareComment = None instInfoNumRootFacts = 0 numElts = 0 elt = None instInfoContext = etree.iterparse(_file, events=("start","end"), huge_tree=True) try: for event, elt in instInfoContext: if event == "start": if elt.getparent() is not None: if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl": if not foundInstance: foundInstance = True pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance") if pi is None: break else: streamingAspects = dict(pi.attrib.copy()) if creationSoftwareComment is None: creationSoftwareComment = precedingComment(elt) if not elt.tag.startswith("{http://www.xbrl.org/"): instInfoNumRootFacts += 1 if instInfoNumRootFacts % 1000 == 0: modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0) elif not foundInstance: break elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl": creationSoftwareComment = precedingComment(elt) if precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None: modelXbrl.error("streamingExtensions:headerMisplaced", _("Header is misplaced: %(error)s, must follow xbrli:xbrl element"), modelObject=elt) elif event == "end": elt.clear() numElts += 1 if numElts % 1000 == 0 and elt.getparent() is not None: while elt.getprevious() is not None and elt.getparent() is not None: del elt.getparent()[0] except etree.XMLSyntaxError as err: modelXbrl.error("xmlSchema:syntax", _("Unrecoverable error: %(error)s"), error=err) _file.close() return err _file.seek(0,io.SEEK_SET) # allow reparsing if not foundInstance or streamingAspects is None: del elt _file.close() return None modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt) startedAt = time.time() try: version = Decimal(streamingAspects.get("version")) if int(version) != 1: modelXbrl.error("streamingExtensions:unsupportedVersion", _("Streaming version %(version)s, major version number must be 1"), modelObject=elt, version=version) foundErrors = True except (InvalidOperation, OverflowError): modelXbrl.error("streamingExtensions:versionError", _("Version %(version)s, number must be 1.n"), modelObject=elt, version=streamingAspects.get("version", "(none)")) foundErrors = True for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"): try: bufLimit = Decimal(streamingAspects.get(bufAspect, "INF")) if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0): raise InvalidOperation elif bufAspect == "contextBuffer": contextBufferLimit = bufLimit elif bufAspect == "unitBuffer": unitBufferLimit = bufLimit elif bufAspect == "footnoteBuffer": footnoteBufferLimit = bufLimit except InvalidOperation: modelXbrl.error("streamingExtensions:valueError", _("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"), modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect)) foundErrors = True if _streamingExtensionsValidate: incompatibleValidations = [] _validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem _disclosureSystem = modelXbrl.modelManager.disclosureSystem if _validateDisclosureSystem and _disclosureSystem.validationType == "EFM": incompatibleValidations.append("EFM") if _validateDisclosureSystem and _disclosureSystem.validationType == "GFM": incompatibleValidations.append("GFM") if _validateDisclosureSystem and _disclosureSystem.validationType == "HMRC": incompatibleValidations.append("HMRC") if modelXbrl.modelManager.validateCalcLB: incompatibleValidations.append("calculation LB") if incompatibleValidations: modelXbrl.error("streamingExtensions:incompatibleValidation", _("Streaming instance validation does not support %(incompatibleValidations)s validation"), modelObject=modelXbrl, incompatibleValidations=', '.join(incompatibleValidations)) foundErrors = True if instInfoContext.error_log: foundErrors = True logSyntaxErrors(instInfoContext) del instInfoContext # dereference for pluginMethod in pluginClassMethods("Streaming.BlockStreaming"): _blockingPluginName = pluginMethod(modelXbrl) if _blockingPluginName: # name of blocking plugin is returned modelXbrl.error("streamingExtensions:incompatiblePlugIn", _("Streaming instance not supported by plugin %(blockingPlugin)s"), modelObject=modelXbrl, blockingPlugin=_blockingPluginName) foundErrors = True if foundErrors: _file.close() return None _encoding = XmlUtil.encoding(_file.read(512)) _file.seek(0,io.SEEK_SET) # allow reparsing if _streamingExtensionsValidate: validator = Validate(modelXbrl) instValidator = validator.instValidator contextBuffer = [] contextsToDrop = [] unitBuffer = [] unitsToDrop = [] footnoteBuffer = [] footnoteLinksToDrop = [] _streamingFactsPlugin = any(True for pluginMethod in pluginClassMethods("Streaming.Facts")) _streamingValidateFactsPlugin = (_streamingExtensionsValidate and any(True for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"))) ''' this is very much slower than iterparse class modelLoaderTarget(): def __init__(self, element_factory=None, parser=None): self.newTree = True self.currentMdlObj = None self.beforeInstanceStream = True self.beforeStartStreamingPlugin = True self.numRootFacts = 1 modelXbrl.makeelementParentModelObject = None modelXbrl.isStreamingMode = True self.factsCheckVersion = None self.factsCheckMd5s = Md5Sum() def start(self, tag, attrib, nsmap=None): modelXbrl.makeelementParentModelObject = self.currentMdlObj # pass parent to makeelement for ModelObjectFactory mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap) mdlObj.sourceline = 1 if self.newTree: self.newTree = False self.currentMdlObj = mdlObj modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree()) modelXbrl.modelDocument = modelDocument # needed for incremental validation mdlObj.init(modelDocument) modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement modelDocument.parserLookupName = _parserLookupName modelDocument.parserLookupClass = _parserLookupClass modelDocument.xmlRootElement = mdlObj modelDocument.schemaLocationElements.add(mdlObj) modelDocument.documentEncoding = _encoding modelDocument._creationSoftwareComment = creationSoftwareComment modelXbrl.info("streamingExtensions:streaming", _("Stream processing this instance."), modelObject = modelDocument) else: self.currentMdlObj.append(mdlObj) self.currentMdlObj = mdlObj mdlObj._init() ns = mdlObj.namespaceURI ln = mdlObj.localName if (self.beforeInstanceStream and ( (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or (ns == XbrlConst.xbrli and ln in ("context", "unit")) or (ns not in (XbrlConst.link, XbrlConst.xbrli)))): self.beforeInstanceStream = False if _streamingExtensionsValidate: instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters(modelXbrl.prefixedNamespaces)) else: # need default dimensions ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) elif not self.beforeInstanceStream and self.beforeStartStreamingPlugin: for pluginMethod in pluginClassMethods("Streaming.Start"): pluginMethod(modelXbrl) self.beforeStartStreamingPlugin = False return mdlObj def end(self, tag): modelDocument = modelXbrl.modelDocument mdlObj = self.currentMdlObj parentMdlObj = mdlObj.getparent() self.currentMdlObj = parentMdlObj ns = mdlObj.namespaceURI ln = mdlObj.localName if ns == XbrlConst.xbrli: if ln == "context": if mdlObj.get("sticky"): del mdlObj.attrib["sticky"] XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) else: if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit: # drop before adding as dropped may have same id as added cntx = contextBuffer.pop(0) if _streamingValidateFactsPlugin: contextsToDrop.append(cntx) else: dropContext(modelXbrl, cntx) del parentMdlObj[parentMdlObj.index(cntx)] cntx = None #>>XmlValidate.validate(modelXbrl, mdlObj) #>>modelDocument.contextDiscover(mdlObj) if contextBufferLimit.is_finite(): contextBuffer.append(mdlObj) if _streamingExtensionsValidate: contextsToCheck = (mdlObj,) instValidator.checkContexts(contextsToCheck) if modelXbrl.hasXDT: instValidator.checkContextsDimensions(contextsToCheck) del contextsToCheck # dereference elif ln == "unit": if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit: # drop before adding as dropped may have same id as added unit = unitBuffer.pop(0) if _streamingValidateFactsPlugin: unitsToDrop.append(unit) else: dropUnit(modelXbrl, unit) del parentMdlObj[parentMdlObj.index(unit)] unit = None #>>XmlValidate.validate(modelXbrl, mdlObj) #>>modelDocument.unitDiscover(mdlObj) if unitBufferLimit.is_finite(): unitBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkUnits( (mdlObj,) ) elif ln == "xbrl": # end of document # check remaining batched facts if any if _streamingValidateFactsPlugin: # plugin attempts to process batch of all root facts not yet processed (not just current one) # finish any final batch of facts if len(modelXbrl.facts) > 0: factsToCheck = modelXbrl.facts.copy() factsHaveBeenProcessed = True # can block facts deletion if required data not yet available, such as numeric unit for DpmDB for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"): if not pluginMethod(modelXbrl, factsToCheck): factsHaveBeenProcessed = False if factsHaveBeenProcessed: for fact in factsToCheck: dropFact(modelXbrl, fact, modelXbrl.facts) del parentMdlObj[parentMdlObj.index(fact)] for cntx in contextsToDrop: dropContext(modelXbrl, cntx) del parentMdlObj[parentMdlObj.index(cntx)] for unit in unitsToDrop: dropUnit(modelXbrl, unit) del parentMdlObj[parentMdlObj.index(unit)] for footnoteLink in footnoteLinksToDrop: dropFootnoteLink(modelXbrl, footnoteLink) del parentMdlObj[parentMdlObj.index(footnoteLink)] fact = cntx = unit = footnoteLink = None del contextsToDrop[:] del unitsToDrop[:] del footnoteLinksToDrop[:] del factsToCheck # check remaining footnote refs for footnoteLink in footnoteBuffer: checkFootnoteHrefs(modelXbrl, footnoteLink) for pluginMethod in pluginClassMethods("Streaming.Finish"): pluginMethod(modelXbrl) elif ns == XbrlConst.link: if ln == "footnoteLink": XmlValidate.validate(modelXbrl, mdlObj) footnoteLinks = (mdlObj,) modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True) if footnoteBufferLimit.is_finite(): footnoteBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkLinks(footnoteLinks) if len(footnoteBuffer) > footnoteBufferLimit: # check that hrefObjects for locators were all satisfied # drop before addition as dropped may have same id as added footnoteLink = footnoteBuffer.pop(0) checkFootnoteHrefs(modelXbrl, footnoteLink) if _streamingValidateFactsPlugin: footnoteLinksToDrop.append(footnoteLink) else: dropFootnoteLink(modelXbrl, footnoteLink) del parentMdlObj[parentMdlObj.index(footnoteLink)] footnoteLink = None footnoteLinks = None elif ln in ("schemaRef", "linkbaseRef"): modelDocument.discoverHref(mdlObj) elif not modelXbrl.skipDTS: if ln in ("roleRef", "arcroleRef"): modelDocument.linkbaseDiscover((mdlObj,), inInstance=True) elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl: self.numRootFacts += 1 #>>XmlValidate.validate(modelXbrl, mdlObj) #>>modelDocument.factDiscover(mdlObj, modelXbrl.facts) if self.factsCheckVersion: self.factCheckFact(mdlObj) if _streamingExtensionsValidate or _streamingValidateFactsPlugin: factsToCheck = (mdlObj,) # validate current fact by itself if _streamingExtensionsValidate: instValidator.checkFacts(factsToCheck) if modelXbrl.hasXDT: instValidator.checkFactsDimensions(factsToCheck) if _streamingValidateFactsPlugin: # plugin attempts to process batch of all root facts not yet processed (not just current one) # use batches of 1000 facts if len(modelXbrl.facts) > 1000: factsToCheck = modelXbrl.facts.copy() factsHaveBeenProcessed = True # can block facts deletion if required data not yet available, such as numeric unit for DpmDB for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"): if not pluginMethod(modelXbrl, factsToCheck): factsHaveBeenProcessed = False if factsHaveBeenProcessed: for fact in factsToCheck: dropFact(modelXbrl, fact, modelXbrl.facts) del parentMdlObj[parentMdlObj.index(fact)] for cntx in contextsToDrop: dropContext(modelXbrl, cntx) del parentMdlObj[parentMdlObj.index(cntx)] for unit in unitsToDrop: dropUnit(modelXbrl, unit) del parentMdlObj[parentMdlObj.index(unit)] for footnoteLink in footnoteLinksToDrop: dropFootnoteLink(modelXbrl, footnoteLink) del parentMdlObj[parentMdlObj.index(footnoteLink)] fact = cntx = unit = footnoteLink = None del contextsToDrop[:] del unitsToDrop[:] del footnoteLinksToDrop[:] del factsToCheck # dereference fact or batch of facts else: dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed del parentMdlObj[parentMdlObj.index(mdlObj)] if self.numRootFacts % 1000 == 0: pass #modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, # 100.0 * self.numRootFacts / instInfoNumRootFacts), # minTimeToShow=20.0) gc.collect() sys.stdout.write ("\rAt fact {} of {} mem {}".format(self.numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed)) return mdlObj def data(self, data): self.currentMdlObj.text = data def comment(self, text): pass def pi(self, target, data): if target == "xbrl-facts-check": _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", data) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "version": self.factsCheckVersion = _matchGroups[1] elif _matchGroups[0] == "sum-of-fact-md5s": try: expectedMd5 = Md5Sum(_matchGroups[1]) if self.factsCheckMd5s != expectedMd5: modelXbrl.warning("streamingExtensions:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"), modelObject=modelXbrl, expectedMd5=expectedMd5, actualMd5Sum=self.factsCheckMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) except ValueError: modelXbrl.error("streamingExtensions:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) def close(self): del modelXbrl.makeelementParentModelObject return None def factCheckFact(self, fact): self.factsCheckMd5s += fact.md5sum for _tupleFact in fact.modelTupleFacts: self.factCheckFact(_tupleFact) _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget()) etree.parse(_file, parser=_parser, base_url=filepath) logSyntaxErrors(_parser) ''' # replace modelLoaderTarget with iterparse (as it now supports CustomElementClassLookup) streamingParserContext = etree.iterparse(_file, events=("start","end"), huge_tree=True) from arelle.ModelObjectFactory import setParserElementClassLookup modelXbrl.isStreamingMode = True # must be set before setting element class lookup (_parser, _parserLookupName, _parserLookupClass) = setParserElementClassLookup(streamingParserContext, modelXbrl) foundInstance = False beforeInstanceStream = beforeStartStreamingPlugin = True numRootFacts = 0 factsCheckVersion = None def factCheckFact(fact): modelDocument._factsCheckMd5s += fact.md5sum for _tupleFact in fact.modelTupleFacts: factCheckFact(_tupleFact) for event, mdlObj in streamingParserContext: if event == "start": if mdlObj.tag == "{http://www.xbrl.org/2003/instance}xbrl": modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree()) modelXbrl.modelDocument = modelDocument # needed for incremental validation mdlObj.init(modelDocument) modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement modelDocument.parserLookupName = _parserLookupName modelDocument.parserLookupClass = _parserLookupClass modelDocument.xmlRootElement = mdlObj modelDocument.schemaLocationElements.add(mdlObj) modelDocument.documentEncoding = _encoding modelDocument._creationSoftwareComment = precedingComment(mdlObj) modelDocument._factsCheckMd5s = Md5Sum() modelXbrl.info("streamingExtensions:streaming", _("Stream processing this instance."), modelObject = modelDocument) elif mdlObj.getparent() is not None: mdlObj._init() # requires discovery as part of start elements if mdlObj.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl": if not foundInstance: foundInstance = True pi = precedingProcessingInstruction(mdlObj, "xbrl-facts-check") if pi is not None: factsCheckVersion = pi.attrib.get("version", None) elif not foundInstance: break ns = mdlObj.qname.namespaceURI ln = mdlObj.qname.localName if beforeInstanceStream: if ((ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or (ns == XbrlConst.xbrli and ln in ("context", "unit")) or (ns not in (XbrlConst.link, XbrlConst.xbrli))): beforeInstanceStream = False if _streamingExtensionsValidate: instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters(modelXbrl.prefixedNamespaces)) else: # need default dimensions ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) elif not beforeInstanceStream and beforeStartStreamingPlugin: for pluginMethod in pluginClassMethods("Streaming.Start"): pluginMethod(modelXbrl) beforeStartStreamingPlugin = False elif event == "end": parentMdlObj = mdlObj.getparent() ns = mdlObj.namespaceURI ln = mdlObj.localName if ns == XbrlConst.xbrli: if ln == "context": if mdlObj.get("sticky"): del mdlObj.attrib["sticky"] XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) else: if len(contextBuffer) >= contextBufferLimit: # drop before adding as dropped may have same id as added cntx = contextBuffer.pop(0) if _streamingFactsPlugin or _streamingValidateFactsPlugin: contextsToDrop.append(cntx) else: dropContext(modelXbrl, cntx) #>>del parentMdlObj[parentMdlObj.index(cntx)] cntx = None XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) if contextBufferLimit.is_finite(): contextBuffer.append(mdlObj) if _streamingExtensionsValidate: contextsToCheck = (mdlObj,) instValidator.checkContexts(contextsToCheck) if modelXbrl.hasXDT: instValidator.checkContextsDimensions(contextsToCheck) del contextsToCheck # dereference elif ln == "unit": if len(unitBuffer) >= unitBufferLimit: # drop before additing as dropped may have same id as added unit = unitBuffer.pop(0) if _streamingFactsPlugin or _streamingValidateFactsPlugin: unitsToDrop.append(unit) else: dropUnit(modelXbrl, unit) #>>del parentMdlObj[parentMdlObj.index(unit)] unit = None XmlValidate.validate(modelXbrl, mdlObj) modelDocument.unitDiscover(mdlObj) if unitBufferLimit.is_finite(): unitBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkUnits( (mdlObj,) ) elif ln == "xbrl": # end of document # check remaining batched facts if any if _streamingFactsPlugin or _streamingValidateFactsPlugin: # plugin attempts to process batch of all root facts not yet processed (not just current one) # finish any final batch of facts if len(modelXbrl.facts) > 0: factsToCheck = modelXbrl.facts.copy() # can block facts deletion if required data not yet available, such as numeric unit for DpmDB if _streamingValidateFactsPlugin: for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"): pluginMethod(instValidator, factsToCheck) if _streamingFactsPlugin: for pluginMethod in pluginClassMethods("Streaming.Facts"): pluginMethod(modelXbrl, factsToCheck) for fact in factsToCheck: dropFact(modelXbrl, fact, modelXbrl.facts) #>>del parentMdlObj[parentMdlObj.index(fact)] for cntx in contextsToDrop: dropContext(modelXbrl, cntx) #>>del parentMdlObj[parentMdlObj.index(cntx)] for unit in unitsToDrop: dropUnit(modelXbrl, unit) #>>del parentMdlObj[parentMdlObj.index(unit)] for footnoteLink in footnoteLinksToDrop: dropFootnoteLink(modelXbrl, footnoteLink) #>>del parentMdlObj[parentMdlObj.index(footnoteLink)] fact = cntx = unit = footnoteLink = None del contextsToDrop[:] del unitsToDrop[:] del footnoteLinksToDrop[:] del factsToCheck # check remaining footnote refs for footnoteLink in footnoteBuffer: checkFootnoteHrefs(modelXbrl, footnoteLink) pi = childProcessingInstruction(mdlObj, "xbrl-facts-check", reversed=True) if pi is not None: # attrib is in .text, not attrib, no idea why!!! _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "sum-of-fact-md5s": try: expectedMd5 = Md5Sum(_matchGroups[1]) if modelDocument._factsCheckMd5s != expectedMd5: modelXbrl.warning("streamingExtensions:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"), modelObject=modelXbrl, expectedMd5=expectedMd5, actualMd5Sum=modelDocument._factsCheckMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) except ValueError: modelXbrl.error("streamingExtensions:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) if _streamingValidateFactsPlugin: for pluginMethod in pluginClassMethods("Streaming.ValidateFinish"): pluginMethod(instValidator) if _streamingFactsPlugin: for pluginMethod in pluginClassMethods("Streaming.Finish"): pluginMethod(modelXbrl) elif ns == XbrlConst.link: if ln in ("schemaRef", "linkbaseRef"): modelDocument.discoverHref(mdlObj, urlRewritePluginClass="ModelDocument.InstanceSchemaRefRewriter") elif ln in ("roleRef", "arcroleRef"): modelDocument.linkbaseDiscover((mdlObj,), inInstance=True) elif ln == "footnoteLink": XmlValidate.validate(modelXbrl, mdlObj) footnoteLinks = (mdlObj,) modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True) if footnoteBufferLimit.is_finite(): footnoteBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkLinks(footnoteLinks) if len(footnoteBuffer) > footnoteBufferLimit: # check that hrefObjects for locators were all satisfied # drop before addition as dropped may have same id as added footnoteLink = footnoteBuffer.pop(0) checkFootnoteHrefs(modelXbrl, footnoteLink) if _streamingValidateFactsPlugin: footnoteLinksToDrop.append(footnoteLink) else: dropFootnoteLink(modelXbrl, footnoteLink) #>>del parentMdlObj[parentMdlObj.index(footnoteLink)] footnoteLink = None footnoteLinks = None elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl and isinstance(mdlObj, ModelFact): numRootFacts += 1 XmlValidate.validate(modelXbrl, mdlObj) modelDocument.factDiscover(mdlObj, modelXbrl.facts) if factsCheckVersion: factCheckFact(mdlObj) if _streamingExtensionsValidate or _streamingFactsPlugin or _streamingValidateFactsPlugin: factsToCheck = (mdlObj,) # validate current fact by itself if _streamingExtensionsValidate: instValidator.checkFacts(factsToCheck) if modelXbrl.hasXDT: instValidator.checkFactsDimensions(factsToCheck) if _streamingFactsPlugin or _streamingValidateFactsPlugin: # plugin attempts to process batch of all root facts not yet processed (not just current one) # use batches of 1000 facts if len(modelXbrl.facts) > 1000: factsToCheck = modelXbrl.facts.copy() # can block facts deletion if required data not yet available, such as numeric unit for DpmDB if _streamingValidateFactsPlugin: for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"): pluginMethod(instValidator, factsToCheck) if _streamingFactsPlugin: for pluginMethod in pluginClassMethods("Streaming.Facts"): pluginMethod(modelXbrl, factsToCheck) for fact in factsToCheck: dropFact(modelXbrl, fact, modelXbrl.facts) #>>del parentMdlObj[parentMdlObj.index(fact)] for cntx in contextsToDrop: dropContext(modelXbrl, cntx) #>>del parentMdlObj[parentMdlObj.index(cntx)] for unit in unitsToDrop: dropUnit(modelXbrl, unit) #>>del parentMdlObj[parentMdlObj.index(unit)] for footnoteLink in footnoteLinksToDrop: dropFootnoteLink(modelXbrl, footnoteLink) #>>del parentMdlObj[parentMdlObj.index(footnoteLink)] fact = cntx = unit = footnoteLink = None del contextsToDrop[:] del unitsToDrop[:] del footnoteLinksToDrop[:] del factsToCheck # dereference fact or batch of facts else: dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed #>>del parentMdlObj[parentMdlObj.index(mdlObj)] if numRootFacts % 1000 == 0: pass #modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, # 100.0 * self.numRootFacts / instInfoNumRootFacts), # minTimeToShow=20.0) #gc.collect() #sys.stdout.write ("\rAt fact {} of {} mem {}".format(numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed)) if mdlObj is not None: mdlObj.clear() del _parser, _parserLookupName, _parserLookupClass if _streamingExtensionsValidate and validator is not None: _file.close() del instValidator validator.close() # track that modelXbrl has been validated by this streaming extension modelXbrl._streamingExtensionValidated = True modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt) return modelXbrl.modelDocument
def vEqual(elt1, elt2): if not hasattr(elt1, "xValid"): XmlValidate.validate(elt1.modelXbrl, elt1) if not hasattr(elt2, "xValid"): XmlValidate.validate(elt2.modelXbrl, elt2) return elt1.sValue == elt2.sValue
def end(self, tag): modelDocument = modelXbrl.modelDocument mdlObj = self.currentMdlObj parentMdlObj = mdlObj.getparent() self.currentMdlObj = parentMdlObj ns = mdlObj.namespaceURI ln = mdlObj.localName if ns == XbrlConst.xbrli: if ln == "context": if mdlObj.get("sticky"): del mdlObj.attrib["sticky"] XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) else: if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit: # drop before adding as dropped may have same id as added cntx = contextBuffer.pop(0) dropContext(modelXbrl, cntx) del parentMdlObj[parentMdlObj.index(cntx)] cntx = None XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) if contextBufferLimit.is_finite(): contextBuffer.append(mdlObj) if _streamingExtensionsValidate: contextsToCheck = (mdlObj,) instValidator.checkContexts(contextsToCheck) if modelXbrl.hasXDT: instValidator.checkContextsDimensions(contextsToCheck) del contextsToCheck # dereference elif ln == "unit": if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit: # drop before additing as dropped may have same id as added unit = unitBuffer.pop(0) dropUnit(modelXbrl, unit) del parentMdlObj[parentMdlObj.index(unit)] unit = None XmlValidate.validate(modelXbrl, mdlObj) modelDocument.unitDiscover(mdlObj) if unitBufferLimit.is_finite(): unitBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkUnits( (mdlObj,) ) elif ln == "xbrl": # end of document # check remaining footnote refs for footnoteLink in footnoteBuffer: checkFootnoteHrefs(modelXbrl, footnoteLink) for pluginMethod in pluginClassMethods("Streaming.Finish"): pluginMethod(modelXbrl) elif ns == XbrlConst.link: if ln == "footnoteLink": XmlValidate.validate(modelXbrl, mdlObj) footnoteLinks = (mdlObj,) modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True) if footnoteBufferLimit.is_finite(): footnoteBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkLinks(footnoteLinks) if len(footnoteBuffer) > footnoteBufferLimit: # check that hrefObjects for locators were all satisfied # drop before addition as dropped may have same id as added footnoteLink = footnoteBuffer.pop(0) checkFootnoteHrefs(modelXbrl, footnoteLink) dropFootnoteLink(modelXbrl, footnoteLink) del parentMdlObj[parentMdlObj.index(footnoteLink)] footnoteLink = None footnoteLinks = None elif ln in ("schemaRef", "linkbaseRef"): modelDocument.discoverHref(mdlObj) elif not modelXbrl.skipDTS: if ln in ("roleRef", "arcroleRef"): modelDocument.linkbaseDiscover((mdlObj,), inInstance=True) elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl: self.numRootFacts += 1 XmlValidate.validate(modelXbrl, mdlObj) modelDocument.factDiscover(mdlObj, modelXbrl.facts) if _streamingExtensionsValidate or _streamingValidateFactsPlugin: factsToCheck = (mdlObj,) # validate current fact by itself if _streamingExtensionsValidate: instValidator.checkFacts(factsToCheck) if modelXbrl.hasXDT: instValidator.checkFactsDimensions(factsToCheck) if _streamingValidateFactsPlugin: # plugin attempts to process batch of all root facts not yet processed (not just current one) factsToCheck = modelXbrl.facts.copy() factsHaveBeenProcessed = True # can block facts deletion if required data not yet available, such as numeric unit for DpmDB for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"): if not pluginMethod(modelXbrl, factsToCheck): factsHaveBeenProcessed = False if factsHaveBeenProcessed: for fact in factsToCheck: dropFact(modelXbrl, fact, modelXbrl.facts) del parentMdlObj[parentMdlObj.index(fact)] else: dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed del parentMdlObj[parentMdlObj.index(mdlObj)] del factsToCheck # dereference fact or batch of facts if self.numRootFacts % 1000 == 0: modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, 100.0 * self.numRootFacts / instInfoNumRootFacts), minTimeToShow=20.0) return mdlObj
def end(self, tag): modelDocument = modelXbrl.modelDocument mdlObj = self.currentMdlObj parentMdlObj = mdlObj.getparent() self.currentMdlObj = parentMdlObj ns = mdlObj.namespaceURI ln = mdlObj.localName if ns == XbrlConst.xbrli: if ln == "context": if mdlObj.get("sticky"): del mdlObj.attrib["sticky"] XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) else: if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit: # drop before adding as dropped may have same id as added cntx = contextBuffer.pop(0) dropContext(modelXbrl, cntx) del parentMdlObj[parentMdlObj.index(cntx)] cntx = None XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) if contextBufferLimit.is_finite(): contextBuffer.append(mdlObj) if _streamingExtensionsValidate: contextsToCheck = (mdlObj,) instValidator.checkContexts(contextsToCheck) if modelXbrl.hasXDT: instValidator.checkContextsDimensions(contextsToCheck) del contextsToCheck # dereference elif ln == "unit": if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit: # drop before additing as dropped may have same id as added unit = unitBuffer.pop(0) dropUnit(modelXbrl, unit) del parentMdlObj[parentMdlObj.index(unit)] unit = None XmlValidate.validate(modelXbrl, mdlObj) modelDocument.unitDiscover(mdlObj) if unitBufferLimit.is_finite(): unitBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkUnits( (mdlObj,) ) elif ln == "xbrl": # end of document # check remaining footnote refs for footnoteLink in footnoteBuffer: checkFootnoteHrefs(modelXbrl, footnoteLink) elif ns == XbrlConst.link: if ln == "footnoteLink": XmlValidate.validate(modelXbrl, mdlObj) footnoteLinks = (mdlObj,) modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True) if footnoteBufferLimit.is_finite(): footnoteBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkLinks(footnoteLinks) if len(footnoteBuffer) > footnoteBufferLimit: # check that hrefObjects for locators were all satisfied # drop before addition as dropped may have same id as added footnoteLink = footnoteBuffer.pop(0) checkFootnoteHrefs(modelXbrl, footnoteLink) dropFootnoteLink(modelXbrl, footnoteLink) del parentMdlObj[parentMdlObj.index(footnoteLink)] footnoteLink = None footnoteLinks = None elif ln in ("schemaRef", "linkbaseRef"): modelDocument.discoverHref(mdlObj) elif not modelXbrl.skipDTS: if ln in ("roleRef", "arcroleRef"): modelDocument.linkbaseDiscover((mdlObj,), inInstance=True) elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl: self.numRootFacts += 1 XmlValidate.validate(modelXbrl, mdlObj) modelDocument.factDiscover(mdlObj, modelXbrl.facts) if _streamingExtensionsValidate: factsToCheck = (mdlObj,) instValidator.checkFacts(factsToCheck) if modelXbrl.hasXDT: instValidator.checkFactsDimensions(factsToCheck) del factsToCheck dropFact(modelXbrl, mdlObj, modelXbrl.facts) del parentMdlObj[parentMdlObj.index(mdlObj)] if self.numRootFacts % 1000 == 0: modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, 100.0 * self.numRootFacts / instInfoNumRootFacts), minTimeToShow=20.0) return mdlObj
def vEqual(elt1, elt2): if not hasattr(elt1,"xValid"): XmlValidate.validate(elt1.modelXbrl, elt1) if not hasattr(elt2,"xValid"): XmlValidate.validate(elt2.modelXbrl, elt2) return elt1.sValue == elt2.sValue
def end(self, tag): modelDocument = modelXbrl.modelDocument mdlObj = self.currentMdlObj parentMdlObj = mdlObj.getparent() self.currentMdlObj = parentMdlObj ns = mdlObj.namespaceURI ln = mdlObj.localName if ns == XbrlConst.xbrli: if ln == "context": if mdlObj.get("sticky"): del mdlObj.attrib["sticky"] XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) else: if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit: # drop before adding as dropped may have same id as added cntx = contextBuffer.pop(0) dropContext(modelXbrl, cntx) del parentMdlObj[parentMdlObj.index(cntx)] cntx = None XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) if contextBufferLimit.is_finite(): contextBuffer.append(mdlObj) if _streamingExtensionsValidate: contextsToCheck = (mdlObj,) instValidator.checkContexts(contextsToCheck) if modelXbrl.hasXDT: instValidator.checkContextsDimensions(contextsToCheck) del contextsToCheck # dereference elif ln == "unit": if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit: # drop before additing as dropped may have same id as added unit = unitBuffer.pop(0) dropUnit(modelXbrl, unit) del parentMdlObj[parentMdlObj.index(unit)] unit = None XmlValidate.validate(modelXbrl, mdlObj) modelDocument.unitDiscover(mdlObj) if unitBufferLimit.is_finite(): unitBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkUnits( (mdlObj,) ) elif ln == "xbrl": # end of document # check remaining footnote refs for footnoteLink in footnoteBuffer: checkFootnoteHrefs(modelXbrl, footnoteLink) elif ns == XbrlConst.link: if ln in ("schemaRef", "linkbaseRef"): modelDocument.discoverHref(mdlObj) elif ln in ("roleRef", "arcroleRef"): modelDocument.linkbaseDiscover((mdlObj,), inInstance=True) elif ln == "footnoteLink": XmlValidate.validate(modelXbrl, mdlObj) footnoteLinks = (mdlObj,) modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True) if footnoteBufferLimit.is_finite(): footnoteBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkLinks(footnoteLinks) if len(footnoteBuffer) > footnoteBufferLimit: # check that hrefObjects for locators were all satisfied # drop before addition as dropped may have same id as added footnoteLink = footnoteBuffer.pop(0) checkFootnoteHrefs(modelXbrl, footnoteLink) dropFootnoteLink(modelXbrl, footnoteLink) del parentMdlObj[parentMdlObj.index(footnoteLink)] footnoteLink = None footnoteLinks = None elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl: self.numRootFacts += 1 XmlValidate.validate(modelXbrl, mdlObj) modelDocument.factDiscover(mdlObj, modelXbrl.facts) if _streamingExtensionsValidate: factsToCheck = (mdlObj,) instValidator.checkFacts(factsToCheck) if modelXbrl.hasXDT: instValidator.checkFactsDimensions(factsToCheck) del factsToCheck dropFact(modelXbrl, mdlObj, modelXbrl.facts) del parentMdlObj[parentMdlObj.index(mdlObj)] if self.numRootFacts % 1000 == 0: modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, 100.0 * self.numRootFacts / instInfoNumRootFacts), minTimeToShow=20.0) return mdlObj
def createContext(self, entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, priItem, dims, segOCCs, scenOCCs, afterSibling=None, beforeSibling=None): xbrlElt = self.modelDocument.xmlRootElement if afterSibling == AUTO_LOCATE_ELEMENT: afterSibling = XmlUtil.lastChild( xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context")) cntxId = 'c-{0:02n}'.format(len(self.contexts) + 1) newCntxElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId), afterSibling=afterSibling, beforeSibling=beforeSibling) entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity") XmlUtil.addChild(entityElt, XbrlConst.xbrli, "identifier", attributes=("scheme", entityIdentScheme), text=entityIdentValue) periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period") if periodType == "forever": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever") elif periodType == "instant": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "instant", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) elif periodType == "duration": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "startDate", text=XmlUtil.dateunionValue(periodStart)) XmlUtil.addChild(periodElt, XbrlConst.xbrli, "endDate", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) segmentElt = None scenarioElt = None from arelle.ModelInstanceObject import ModelDimensionValue if dims: # requires primary item to determin ambiguous concepts ''' in theory we have to check full set of dimensions for validity in source or any other context element, but for shortcut will see if each dimension is already reported in an unambiguous valid contextElement ''' from arelle.PrototypeInstanceObject import FactPrototype, ContextPrototype, DimValuePrototype fp = FactPrototype(self, priItem, dims.items()) # force trying a valid prototype's context Elements if not isFactDimensionallyValid( self, fp, setPrototypeContextElements=True): self.info( "arelleLinfo", _("Create context for %(priItem)s, cannot determine valid context elements, no suitable hypercubes" ), modelObject=self, priItem=priItem) fpDims = fp.context.qnameDims for dimQname in sorted(fpDims.keys()): dimValue = fpDims[dimQname] if isinstance(dimValue, DimValuePrototype): dimMemberQname = dimValue.memberQname # None if typed dimension contextEltName = dimValue.contextElement else: # qname for explicit or node for typed dimMemberQname = None contextEltName = None if contextEltName == "segment": if segmentElt is None: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") contextElt = segmentElt elif contextEltName == "scenario": if scenarioElt is None: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") contextElt = scenarioElt else: self.info( "arelleLinfo", _("Create context, %(dimension)s, cannot determine context element, either no all relationship or validation issue" ), modelObject=self, dimension=dimQname), continue dimConcept = self.qnameConcepts[dimQname] dimAttr = ("dimension", XmlUtil.addQnameValue(xbrlElt, dimConcept.qname)) if dimConcept.isTypedDimension: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:typedMember", attributes=dimAttr) if isinstance(dimValue, (ModelDimensionValue, DimValuePrototype)) and dimValue.isTyped: XmlUtil.copyNodes(dimElt, dimValue.typedMember) elif dimMemberQname: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:explicitMember", attributes=dimAttr, text=XmlUtil.addQnameValue( xbrlElt, dimMemberQname)) if segOCCs: if segmentElt is None: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") XmlUtil.copyNodes(segmentElt, segOCCs) if scenOCCs: if scenarioElt is None: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") XmlUtil.copyNodes(scenarioElt, scenOCCs) self.modelDocument.contextDiscover(newCntxElt) XmlValidate.validate(self, newCntxElt) return newCntxElt
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, *args, **kwargs): # check if big instance and has header with an initial incomplete tree walk (just 2 elements if not _streamingExtensionsCheck: return None # track whether modelXbrl has been validated by this streaming extension modelXbrl._streamingExtensionValidated = False def logSyntaxErrors(parsercontext): for error in parsercontext.error_log: modelXbrl.error( "xmlSchema:syntax", _("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element" ), modelObject=modelXbrl, fileName=os.path.basename(filepath), error=error.message, line=error.line, column=error.column, sourceAction="streaming") #### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ### #### note: iterparse wants a binary file, but file is text mode _file, = modelXbrl.fileSource.file(filepath, binary=True) startedAt = time.time() modelXbrl.profileActivity() ''' this seems twice as slow as iterparse class instInfoTarget(): def __init__(self, element_factory=None, parser=None): self.newTree = True self.streamingAspects = None self.foundInstance = False self.creationSoftwareComment = '' self.currentEltTag = "(before xbrli:xbrl)" self.numRootFacts = 0 def start(self, tag, attrib, nsmap=None): if self.newTree: if tag == "{http://www.xbrl.org/2003/instance}xbrl": self.foundInstance = True self.newTree = False else: # break raise NotInstanceDocumentException() elif not tag.startswith("{http://www.xbrl.org/"): self.numRootFacts += 1 if self.numRootFacts % 1000 == 0: modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0) self.currentEltTag = tag def end(self, tag): pass def data(self, data): pass def comment(self, text): if not self.foundInstance: # accumulate comments before xbrli:xbrl self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text elif not self.creationSoftwareComment: self.creationSoftwareComment = text # or first comment after xbrli:xbrl def pi(self, target, data): if target == "xbrl-streamable-instance": if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl": self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results else: modelXbrl.error("streamingExtensions:headerMisplaced", _("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"), modelObject=modelXbrl, target=target, element=self.currentEltTag) def close(self): if not self.creationSoftwareComment: self.creationSoftwareComment = None return True instInfo = instInfoTarget() infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo) try: etree.parse(_file, parser=infoParser, base_url=filepath) except NotInstanceDocumentException: pass ''' foundErrors = False foundInstance = False streamingAspects = None creationSoftwareComment = None instInfoNumRootFacts = 0 numElts = 0 elt = None instInfoContext = etree.iterparse(_file, events=("start", "end"), huge_tree=True) try: for event, elt in instInfoContext: if event == "start": if elt.getparent() is not None: if elt.getparent( ).tag == "{http://www.xbrl.org/2003/instance}xbrl": if not foundInstance: foundInstance = True pi = precedingProcessingInstruction( elt, "xbrl-streamable-instance") if pi is None: break else: streamingAspects = dict(pi.attrib.copy()) if creationSoftwareComment is None: creationSoftwareComment = precedingComment( elt) if not elt.tag.startswith("{http://www.xbrl.org/"): instInfoNumRootFacts += 1 if instInfoNumRootFacts % 1000 == 0: modelXbrl.profileActivity( "... streaming tree check", minTimeToShow=20.0) elif not foundInstance: break elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl": creationSoftwareComment = precedingComment(elt) if precedingProcessingInstruction( elt, "xbrl-streamable-instance") is not None: modelXbrl.error( "streamingExtensions:headerMisplaced", _("Header is misplaced: %(error)s, must follow xbrli:xbrl element" ), modelObject=elt) elif event == "end": elt.clear() numElts += 1 if numElts % 1000 == 0 and elt.getparent() is not None: while elt.getprevious() is not None and elt.getparent( ) is not None: del elt.getparent()[0] except etree.XMLSyntaxError as err: modelXbrl.error("xmlSchema:syntax", _("Unrecoverable error: %(error)s"), error=err) _file.close() return err _file.seek(0, io.SEEK_SET) # allow reparsing if not foundInstance or streamingAspects is None: del elt _file.close() return None modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt) startedAt = time.time() try: version = Decimal(streamingAspects.get("version")) if int(version) != 1: modelXbrl.error( "streamingExtensions:unsupportedVersion", _("Streaming version %(version)s, major version number must be 1" ), modelObject=elt, version=version) foundErrors = True except (InvalidOperation, OverflowError): modelXbrl.error("streamingExtensions:versionError", _("Version %(version)s, number must be 1.n"), modelObject=elt, version=streamingAspects.get("version", "(none)")) foundErrors = True for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"): try: bufLimit = Decimal(streamingAspects.get(bufAspect, "INF")) if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0): raise InvalidOperation elif bufAspect == "contextBuffer": contextBufferLimit = bufLimit elif bufAspect == "unitBuffer": unitBufferLimit = bufLimit elif bufAspect == "footnoteBuffer": footnoteBufferLimit = bufLimit except InvalidOperation: modelXbrl.error( "streamingExtensions:valueError", _("Streaming %(attrib)s %(value)s, number must be a positive integer or INF" ), modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect)) foundErrors = True if _streamingExtensionsValidate: incompatibleValidations = [] _validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem _disclosureSystem = modelXbrl.modelManager.disclosureSystem if _validateDisclosureSystem and _disclosureSystem.validationType == "EFM": incompatibleValidations.append("EFM") if _validateDisclosureSystem and _disclosureSystem.validationType == "GFM": incompatibleValidations.append("GFM") if _validateDisclosureSystem and _disclosureSystem.validationType == "HMRC": incompatibleValidations.append("HMRC") if modelXbrl.modelManager.validateCalcLB: incompatibleValidations.append("calculation LB") if incompatibleValidations: modelXbrl.error( "streamingExtensions:incompatibleValidation", _("Streaming instance validation does not support %(incompatibleValidations)s validation" ), modelObject=modelXbrl, incompatibleValidations=', '.join(incompatibleValidations)) foundErrors = True if instInfoContext.error_log: foundErrors = True logSyntaxErrors(instInfoContext) del instInfoContext # dereference for pluginMethod in pluginClassMethods("Streaming.BlockStreaming"): _blockingPluginName = pluginMethod(modelXbrl) if _blockingPluginName: # name of blocking plugin is returned modelXbrl.error( "streamingExtensions:incompatiblePlugIn", _("Streaming instance not supported by plugin %(blockingPlugin)s" ), modelObject=modelXbrl, blockingPlugin=_blockingPluginName) foundErrors = True if foundErrors: _file.close() return None _encoding = XmlUtil.encoding(_file.read(512)) _file.seek(0, io.SEEK_SET) # allow reparsing if _streamingExtensionsValidate: validator = Validate(modelXbrl) instValidator = validator.instValidator contextBuffer = [] contextsToDrop = [] unitBuffer = [] unitsToDrop = [] footnoteBuffer = [] footnoteLinksToDrop = [] _streamingFactsPlugin = any( True for pluginMethod in pluginClassMethods("Streaming.Facts")) _streamingValidateFactsPlugin = (_streamingExtensionsValidate and any( True for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"))) ''' this is very much slower than iterparse class modelLoaderTarget(): def __init__(self, element_factory=None, parser=None): self.newTree = True self.currentMdlObj = None self.beforeInstanceStream = True self.beforeStartStreamingPlugin = True self.numRootFacts = 1 modelXbrl.makeelementParentModelObject = None modelXbrl.isStreamingMode = True self.factsCheckVersion = None self.factsCheckMd5s = Md5Sum() def start(self, tag, attrib, nsmap=None): modelXbrl.makeelementParentModelObject = self.currentMdlObj # pass parent to makeelement for ModelObjectFactory mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap) mdlObj.sourceline = 1 if self.newTree: self.newTree = False self.currentMdlObj = mdlObj modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree()) modelXbrl.modelDocument = modelDocument # needed for incremental validation mdlObj.init(modelDocument) modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement modelDocument.parserLookupName = _parserLookupName modelDocument.parserLookupClass = _parserLookupClass modelDocument.xmlRootElement = mdlObj modelDocument.schemaLocationElements.add(mdlObj) modelDocument.documentEncoding = _encoding modelDocument._creationSoftwareComment = creationSoftwareComment modelXbrl.info("streamingExtensions:streaming", _("Stream processing this instance."), modelObject = modelDocument) else: self.currentMdlObj.append(mdlObj) self.currentMdlObj = mdlObj mdlObj._init() ns = mdlObj.namespaceURI ln = mdlObj.localName if (self.beforeInstanceStream and ( (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or (ns == XbrlConst.xbrli and ln in ("context", "unit")) or (ns not in (XbrlConst.link, XbrlConst.xbrli)))): self.beforeInstanceStream = False if _streamingExtensionsValidate: instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters(modelXbrl.prefixedNamespaces)) else: # need default dimensions ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) elif not self.beforeInstanceStream and self.beforeStartStreamingPlugin: for pluginMethod in pluginClassMethods("Streaming.Start"): pluginMethod(modelXbrl) self.beforeStartStreamingPlugin = False return mdlObj def end(self, tag): modelDocument = modelXbrl.modelDocument mdlObj = self.currentMdlObj parentMdlObj = mdlObj.getparent() self.currentMdlObj = parentMdlObj ns = mdlObj.namespaceURI ln = mdlObj.localName if ns == XbrlConst.xbrli: if ln == "context": if mdlObj.get("sticky"): del mdlObj.attrib["sticky"] XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) else: if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit: # drop before adding as dropped may have same id as added cntx = contextBuffer.pop(0) if _streamingValidateFactsPlugin: contextsToDrop.append(cntx) else: dropContext(modelXbrl, cntx) del parentMdlObj[parentMdlObj.index(cntx)] cntx = None #>>XmlValidate.validate(modelXbrl, mdlObj) #>>modelDocument.contextDiscover(mdlObj) if contextBufferLimit.is_finite(): contextBuffer.append(mdlObj) if _streamingExtensionsValidate: contextsToCheck = (mdlObj,) instValidator.checkContexts(contextsToCheck) if modelXbrl.hasXDT: instValidator.checkContextsDimensions(contextsToCheck) del contextsToCheck # dereference elif ln == "unit": if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit: # drop before adding as dropped may have same id as added unit = unitBuffer.pop(0) if _streamingValidateFactsPlugin: unitsToDrop.append(unit) else: dropUnit(modelXbrl, unit) del parentMdlObj[parentMdlObj.index(unit)] unit = None #>>XmlValidate.validate(modelXbrl, mdlObj) #>>modelDocument.unitDiscover(mdlObj) if unitBufferLimit.is_finite(): unitBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkUnits( (mdlObj,) ) elif ln == "xbrl": # end of document # check remaining batched facts if any if _streamingValidateFactsPlugin: # plugin attempts to process batch of all root facts not yet processed (not just current one) # finish any final batch of facts if len(modelXbrl.facts) > 0: factsToCheck = modelXbrl.facts.copy() factsHaveBeenProcessed = True # can block facts deletion if required data not yet available, such as numeric unit for DpmDB for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"): if not pluginMethod(modelXbrl, factsToCheck): factsHaveBeenProcessed = False if factsHaveBeenProcessed: for fact in factsToCheck: dropFact(modelXbrl, fact, modelXbrl.facts) del parentMdlObj[parentMdlObj.index(fact)] for cntx in contextsToDrop: dropContext(modelXbrl, cntx) del parentMdlObj[parentMdlObj.index(cntx)] for unit in unitsToDrop: dropUnit(modelXbrl, unit) del parentMdlObj[parentMdlObj.index(unit)] for footnoteLink in footnoteLinksToDrop: dropFootnoteLink(modelXbrl, footnoteLink) del parentMdlObj[parentMdlObj.index(footnoteLink)] fact = cntx = unit = footnoteLink = None del contextsToDrop[:] del unitsToDrop[:] del footnoteLinksToDrop[:] del factsToCheck # check remaining footnote refs for footnoteLink in footnoteBuffer: checkFootnoteHrefs(modelXbrl, footnoteLink) for pluginMethod in pluginClassMethods("Streaming.Finish"): pluginMethod(modelXbrl) elif ns == XbrlConst.link: if ln == "footnoteLink": XmlValidate.validate(modelXbrl, mdlObj) footnoteLinks = (mdlObj,) modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True) if footnoteBufferLimit.is_finite(): footnoteBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkLinks(footnoteLinks) if len(footnoteBuffer) > footnoteBufferLimit: # check that hrefObjects for locators were all satisfied # drop before addition as dropped may have same id as added footnoteLink = footnoteBuffer.pop(0) checkFootnoteHrefs(modelXbrl, footnoteLink) if _streamingValidateFactsPlugin: footnoteLinksToDrop.append(footnoteLink) else: dropFootnoteLink(modelXbrl, footnoteLink) del parentMdlObj[parentMdlObj.index(footnoteLink)] footnoteLink = None footnoteLinks = None elif ln in ("schemaRef", "linkbaseRef"): modelDocument.discoverHref(mdlObj) elif not modelXbrl.skipDTS: if ln in ("roleRef", "arcroleRef"): modelDocument.linkbaseDiscover((mdlObj,), inInstance=True) elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl: self.numRootFacts += 1 #>>XmlValidate.validate(modelXbrl, mdlObj) #>>modelDocument.factDiscover(mdlObj, modelXbrl.facts) if self.factsCheckVersion: self.factCheckFact(mdlObj) if _streamingExtensionsValidate or _streamingValidateFactsPlugin: factsToCheck = (mdlObj,) # validate current fact by itself if _streamingExtensionsValidate: instValidator.checkFacts(factsToCheck) if modelXbrl.hasXDT: instValidator.checkFactsDimensions(factsToCheck) if _streamingValidateFactsPlugin: # plugin attempts to process batch of all root facts not yet processed (not just current one) # use batches of 1000 facts if len(modelXbrl.facts) > 1000: factsToCheck = modelXbrl.facts.copy() factsHaveBeenProcessed = True # can block facts deletion if required data not yet available, such as numeric unit for DpmDB for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"): if not pluginMethod(modelXbrl, factsToCheck): factsHaveBeenProcessed = False if factsHaveBeenProcessed: for fact in factsToCheck: dropFact(modelXbrl, fact, modelXbrl.facts) del parentMdlObj[parentMdlObj.index(fact)] for cntx in contextsToDrop: dropContext(modelXbrl, cntx) del parentMdlObj[parentMdlObj.index(cntx)] for unit in unitsToDrop: dropUnit(modelXbrl, unit) del parentMdlObj[parentMdlObj.index(unit)] for footnoteLink in footnoteLinksToDrop: dropFootnoteLink(modelXbrl, footnoteLink) del parentMdlObj[parentMdlObj.index(footnoteLink)] fact = cntx = unit = footnoteLink = None del contextsToDrop[:] del unitsToDrop[:] del footnoteLinksToDrop[:] del factsToCheck # dereference fact or batch of facts else: dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed del parentMdlObj[parentMdlObj.index(mdlObj)] if self.numRootFacts % 1000 == 0: pass #modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, # 100.0 * self.numRootFacts / instInfoNumRootFacts), # minTimeToShow=20.0) gc.collect() sys.stdout.write ("\rAt fact {} of {} mem {}".format(self.numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed)) return mdlObj def data(self, data): self.currentMdlObj.text = data def comment(self, text): pass def pi(self, target, data): if target == "xbrl-facts-check": _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", data) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "version": self.factsCheckVersion = _matchGroups[1] elif _matchGroups[0] == "sum-of-fact-md5s": try: expectedMd5 = Md5Sum(_matchGroups[1]) if self.factsCheckMd5s != expectedMd5: modelXbrl.warning("streamingExtensions:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"), modelObject=modelXbrl, expectedMd5=expectedMd5, actualMd5Sum=self.factsCheckMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) except ValueError: modelXbrl.error("streamingExtensions:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) def close(self): del modelXbrl.makeelementParentModelObject return None def factCheckFact(self, fact): self.factsCheckMd5s += fact.md5sum for _tupleFact in fact.modelTupleFacts: self.factCheckFact(_tupleFact) _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget()) etree.parse(_file, parser=_parser, base_url=filepath) logSyntaxErrors(_parser) ''' # replace modelLoaderTarget with iterparse (as it now supports CustomElementClassLookup) streamingParserContext = etree.iterparse(_file, events=("start", "end"), huge_tree=True) from arelle.ModelObjectFactory import setParserElementClassLookup modelXbrl.isStreamingMode = True # must be set before setting element class lookup (_parser, _parserLookupName, _parserLookupClass) = setParserElementClassLookup(streamingParserContext, modelXbrl) foundInstance = False beforeInstanceStream = beforeStartStreamingPlugin = True numRootFacts = 0 factsCheckVersion = None def factCheckFact(fact): modelDocument._factsCheckMd5s += fact.md5sum for _tupleFact in fact.modelTupleFacts: factCheckFact(_tupleFact) for event, mdlObj in streamingParserContext: if event == "start": if mdlObj.tag == "{http://www.xbrl.org/2003/instance}xbrl": modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree()) modelXbrl.modelDocument = modelDocument # needed for incremental validation mdlObj.init(modelDocument) modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement modelDocument.parserLookupName = _parserLookupName modelDocument.parserLookupClass = _parserLookupClass modelDocument.xmlRootElement = mdlObj modelDocument.schemaLocationElements.add(mdlObj) modelDocument.documentEncoding = _encoding modelDocument._creationSoftwareComment = precedingComment( mdlObj) modelDocument._factsCheckMd5s = Md5Sum() modelXbrl.info("streamingExtensions:streaming", _("Stream processing this instance."), modelObject=modelDocument) elif mdlObj.getparent() is not None: mdlObj._init() # requires discovery as part of start elements if mdlObj.getparent( ).tag == "{http://www.xbrl.org/2003/instance}xbrl": if not foundInstance: foundInstance = True pi = precedingProcessingInstruction( mdlObj, "xbrl-facts-check") if pi is not None: factsCheckVersion = pi.attrib.get("version", None) elif not foundInstance: break ns = mdlObj.qname.namespaceURI ln = mdlObj.qname.localName if beforeInstanceStream: if ((ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or (ns == XbrlConst.xbrli and ln in ("context", "unit")) or (ns not in (XbrlConst.link, XbrlConst.xbrli))): beforeInstanceStream = False if _streamingExtensionsValidate: instValidator.validate( modelXbrl, modelXbrl.modelManager.formulaOptions. typedParameters(modelXbrl.prefixedNamespaces)) else: # need default dimensions ValidateXbrlDimensions.loadDimensionDefaults( modelXbrl) elif not beforeInstanceStream and beforeStartStreamingPlugin: for pluginMethod in pluginClassMethods("Streaming.Start"): pluginMethod(modelXbrl) beforeStartStreamingPlugin = False elif event == "end": parentMdlObj = mdlObj.getparent() ns = mdlObj.namespaceURI ln = mdlObj.localName if ns == XbrlConst.xbrli: if ln == "context": if mdlObj.get("sticky"): del mdlObj.attrib["sticky"] XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) else: if len(contextBuffer) >= contextBufferLimit: # drop before adding as dropped may have same id as added cntx = contextBuffer.pop(0) if _streamingFactsPlugin or _streamingValidateFactsPlugin: contextsToDrop.append(cntx) else: dropContext(modelXbrl, cntx) #>>del parentMdlObj[parentMdlObj.index(cntx)] cntx = None XmlValidate.validate(modelXbrl, mdlObj) modelDocument.contextDiscover(mdlObj) if contextBufferLimit.is_finite(): contextBuffer.append(mdlObj) if _streamingExtensionsValidate: contextsToCheck = (mdlObj, ) instValidator.checkContexts(contextsToCheck) if modelXbrl.hasXDT: instValidator.checkContextsDimensions( contextsToCheck) del contextsToCheck # dereference elif ln == "unit": if len(unitBuffer) >= unitBufferLimit: # drop before additing as dropped may have same id as added unit = unitBuffer.pop(0) if _streamingFactsPlugin or _streamingValidateFactsPlugin: unitsToDrop.append(unit) else: dropUnit(modelXbrl, unit) #>>del parentMdlObj[parentMdlObj.index(unit)] unit = None XmlValidate.validate(modelXbrl, mdlObj) modelDocument.unitDiscover(mdlObj) if unitBufferLimit.is_finite(): unitBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkUnits((mdlObj, )) elif ln == "xbrl": # end of document # check remaining batched facts if any if _streamingFactsPlugin or _streamingValidateFactsPlugin: # plugin attempts to process batch of all root facts not yet processed (not just current one) # finish any final batch of facts if len(modelXbrl.facts) > 0: factsToCheck = modelXbrl.facts.copy() # can block facts deletion if required data not yet available, such as numeric unit for DpmDB if _streamingValidateFactsPlugin: for pluginMethod in pluginClassMethods( "Streaming.ValidateFacts"): pluginMethod(instValidator, factsToCheck) if _streamingFactsPlugin: for pluginMethod in pluginClassMethods( "Streaming.Facts"): pluginMethod(modelXbrl, factsToCheck) for fact in factsToCheck: dropFact(modelXbrl, fact, modelXbrl.facts) #>>del parentMdlObj[parentMdlObj.index(fact)] for cntx in contextsToDrop: dropContext(modelXbrl, cntx) #>>del parentMdlObj[parentMdlObj.index(cntx)] for unit in unitsToDrop: dropUnit(modelXbrl, unit) #>>del parentMdlObj[parentMdlObj.index(unit)] for footnoteLink in footnoteLinksToDrop: dropFootnoteLink(modelXbrl, footnoteLink) #>>del parentMdlObj[parentMdlObj.index(footnoteLink)] fact = cntx = unit = footnoteLink = None del contextsToDrop[:] del unitsToDrop[:] del footnoteLinksToDrop[:] del factsToCheck # check remaining footnote refs for footnoteLink in footnoteBuffer: checkFootnoteHrefs(modelXbrl, footnoteLink) pi = childProcessingInstruction(mdlObj, "xbrl-facts-check", reversed=True) if pi is not None: # attrib is in .text, not attrib, no idea why!!! _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "sum-of-fact-md5s": try: expectedMd5 = Md5Sum(_matchGroups[1]) if modelDocument._factsCheckMd5s != expectedMd5: modelXbrl.warning( "streamingExtensions:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s" ), modelObject=modelXbrl, expectedMd5=expectedMd5, actualMd5Sum=modelDocument. _factsCheckMd5s) else: modelXbrl.info( "info", _("Successful XBRL facts sum of md5s." ), modelObject=modelXbrl) except ValueError: modelXbrl.error( "streamingExtensions:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s" ), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) if _streamingValidateFactsPlugin: for pluginMethod in pluginClassMethods( "Streaming.ValidateFinish"): pluginMethod(instValidator) if _streamingFactsPlugin: for pluginMethod in pluginClassMethods( "Streaming.Finish"): pluginMethod(modelXbrl) elif ns == XbrlConst.link: if ln in ("schemaRef", "linkbaseRef"): modelDocument.discoverHref( mdlObj, urlRewritePluginClass= "ModelDocument.InstanceSchemaRefRewriter") elif ln in ("roleRef", "arcroleRef"): modelDocument.linkbaseDiscover((mdlObj, ), inInstance=True) elif ln == "footnoteLink": XmlValidate.validate(modelXbrl, mdlObj) footnoteLinks = (mdlObj, ) modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True) if footnoteBufferLimit.is_finite(): footnoteBuffer.append(mdlObj) if _streamingExtensionsValidate: instValidator.checkLinks(footnoteLinks) if len(footnoteBuffer) > footnoteBufferLimit: # check that hrefObjects for locators were all satisfied # drop before addition as dropped may have same id as added footnoteLink = footnoteBuffer.pop(0) checkFootnoteHrefs(modelXbrl, footnoteLink) if _streamingValidateFactsPlugin: footnoteLinksToDrop.append(footnoteLink) else: dropFootnoteLink(modelXbrl, footnoteLink) #>>del parentMdlObj[parentMdlObj.index(footnoteLink)] footnoteLink = None footnoteLinks = None elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl and isinstance( mdlObj, ModelFact): numRootFacts += 1 XmlValidate.validate(modelXbrl, mdlObj) modelDocument.factDiscover(mdlObj, modelXbrl.facts) if factsCheckVersion: factCheckFact(mdlObj) if _streamingExtensionsValidate or _streamingFactsPlugin or _streamingValidateFactsPlugin: factsToCheck = (mdlObj, ) # validate current fact by itself if _streamingExtensionsValidate: instValidator.checkFacts(factsToCheck) if modelXbrl.hasXDT: instValidator.checkFactsDimensions(factsToCheck) if _streamingFactsPlugin or _streamingValidateFactsPlugin: # plugin attempts to process batch of all root facts not yet processed (not just current one) # use batches of 1000 facts if len(modelXbrl.facts) > 1000: factsToCheck = modelXbrl.facts.copy() # can block facts deletion if required data not yet available, such as numeric unit for DpmDB if _streamingValidateFactsPlugin: for pluginMethod in pluginClassMethods( "Streaming.ValidateFacts"): pluginMethod(instValidator, factsToCheck) if _streamingFactsPlugin: for pluginMethod in pluginClassMethods( "Streaming.Facts"): pluginMethod(modelXbrl, factsToCheck) for fact in factsToCheck: dropFact(modelXbrl, fact, modelXbrl.facts) #>>del parentMdlObj[parentMdlObj.index(fact)] for cntx in contextsToDrop: dropContext(modelXbrl, cntx) #>>del parentMdlObj[parentMdlObj.index(cntx)] for unit in unitsToDrop: dropUnit(modelXbrl, unit) #>>del parentMdlObj[parentMdlObj.index(unit)] for footnoteLink in footnoteLinksToDrop: dropFootnoteLink(modelXbrl, footnoteLink) #>>del parentMdlObj[parentMdlObj.index(footnoteLink)] fact = cntx = unit = footnoteLink = None del contextsToDrop[:] del unitsToDrop[:] del footnoteLinksToDrop[:] del factsToCheck # dereference fact or batch of facts else: dropFact( modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed #>>del parentMdlObj[parentMdlObj.index(mdlObj)] if numRootFacts % 1000 == 0: pass #modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, # 100.0 * self.numRootFacts / instInfoNumRootFacts), # minTimeToShow=20.0) #gc.collect() #sys.stdout.write ("\rAt fact {} of {} mem {}".format(numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed)) if mdlObj is not None: mdlObj.clear() del _parser, _parserLookupName, _parserLookupClass if _streamingExtensionsValidate and validator is not None: _file.close() del instValidator validator.close() # track that modelXbrl has been validated by this streaming extension modelXbrl._streamingExtensionValidated = True modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt) return modelXbrl.modelDocument
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath): # check if big instance and has header with an initial incomplete tree walk (just 2 elements def logSyntaxErrors(parsercontext): for error in parsercontext.error_log: modelXbrl.error("xmlSchema:syntax", _("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"), modelObject=modelDocument, fileName=os.path.basename(filepath), error=error.message, line=error.line, column=error.column, sourceAction="streaming") #### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ### #### note: iterparse wants a binary file, but file is text mode _file, = modelXbrl.fileSource.file(filepath, binary=True) startedAt = time.time() modelXbrl.profileActivity() parsercontext = etree.iterparse(_file, events=("start","end"), huge_tree=True) foundInstance = False foundErrors = False streamingAspects = None numRootFacts1 = 0 numElts = 0 elt = None for event, elt in parsercontext: if event == "start": if elt.getparent() is not None: if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl": if not foundInstance: foundInstance = True pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance") if pi is None: break else: streamingAspects = dict(pi.attrib.copy()) if not elt.tag.startswith("{http://www.xbrl.org/"): numRootFacts1 += 1 if numRootFacts1 % 1000 == 0: modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0) elif not foundInstance: break elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl" and precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None: modelXbrl.error("streamingExtensions:headerMisplaced", _("Header is misplaced: %(error)s, must follow xbrli:xbrl element"), modelObject=elt) elif event == "end": elt.clear() numElts += 1 if numElts % 1000 == 0 and elt.getparent() is not None: while elt.getprevious() is not None and elt.getparent() is not None: del elt.getparent()[0] if elt is not None: elt.clear() _file.seek(0,io.SEEK_SET) # allow reparsing if not foundInstance or streamingAspects is None: del elt, parsercontext _file.close() return None modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt) startedAt = time.time() try: version = Decimal(streamingAspects.get("version")) if int(version) != 1: modelXbrl.error("streamingExtensions:unsupportedVersion", _("Streaming version %(version)s, major version number must be 1"), modelObject=elt, version=version) foundErrors = True except (InvalidOperation, OverflowError): modelXbrl.error("streamingExtensions:versionError", _("Version %(version)s, number must be 1.n"), modelObject=elt, version=streamingAspects.get("version", "(none)")) foundErrors = True for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"): try: bufLimit = Decimal(streamingAspects.get(bufAspect, "INF")) if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0): raise InvalidOperation elif bufAspect == "contextBuffer": contextBufferLimit = bufLimit elif bufAspect == "unitBuffer": unitBufferLimit = bufLimit elif bufAspect == "footnoteBuffer": footnoteBufferLimit = bufLimit except InvalidOperation: modelXbrl.error("streamingExtensions:valueError", _("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"), modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect)) foundErrors = True if parsercontext.error_log: foundErrors = True logSyntaxErrors(parsercontext) if foundErrors: _file.close() return None parsercontext = etree.iterparse(_file, events=("start","end"), huge_tree=True) _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl,filepath) eltMdlObjs = {} beforeInstanceStream = True validator = None contextBuffer = [] unitBuffer = [] footnoteBuffer = [] factBuffer = [] numFacts = numRootFacts2 = 1 for event, elt in parsercontext: if event == "start": mdlObj = _parser.makeelement(elt.tag, attrib=elt.attrib, nsmap=elt.nsmap) mdlObj.sourceline = elt.sourceline eltMdlObjs[elt] = mdlObj if elt.getparent() is None: modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, etree.ElementTree(mdlObj)) modelDocument.xmlRootElement = mdlObj modelXbrl.modelDocument = modelDocument # needed for incremental validation mdlObj.init(modelDocument) modelXbrl.info("streamingExtensions:streaming", _("Stream processing this instance."), modelObject = modelDocument) else: eltMdlObjs[elt.getparent()].append(mdlObj) mdlObj._init() ns = mdlObj.namespaceURI ln = mdlObj.localName if (beforeInstanceStream and ( (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or (ns == XbrlConst.xbrli and ln in ("context", "unit")) or (ns not in (XbrlConst.link, XbrlConst.xbrli)))): beforeInstanceStream = False if _streamingExtensionsValidate: validator = Validate(modelXbrl) validator.instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters()) else: # need default dimensions ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) mdlObj = None # deref elif event == "end": mdlObj = eltMdlObjs.pop(elt) if elt.text: # text available after child nodes processed mdlObj.text = elt.text ns = mdlObj.namespaceURI ln = mdlObj.localName parentMdlObj = mdlObj.getparent() if ns == XbrlConst.xbrli: if ln == "context": if mdlObj.get("sticky"): del mdlObj.attrib["sticky"] modelDocument.contextDiscover(mdlObj) else: if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit: # drop before adding as dropped may have same id as added cntx = contextBuffer.pop(0) dropContext(modelXbrl, cntx) del parentMdlObj[parentMdlObj.index(cntx)] cntx = None modelDocument.contextDiscover(mdlObj) if contextBufferLimit.is_finite(): contextBuffer.append(mdlObj) if _streamingExtensionsValidate: contextsToCheck = (mdlObj,) validator.instValidator.checkContexts(contextsToCheck) if modelXbrl.hasXDT: validator.instValidator.checkContextsDimensions(contextsToCheck) del contextsToCheck # dereference elif ln == "unit": if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit: # drop before additing as dropped may have same id as added unit = unitBuffer.pop(0) dropUnit(modelXbrl, unit) del parentMdlObj[parentMdlObj.index(unit)] unit = None modelDocument.unitDiscover(mdlObj) if unitBufferLimit.is_finite(): unitBuffer.append(mdlObj) if _streamingExtensionsValidate: validator.instValidator.checkUnits( (mdlObj,) ) elif ln == "xbrl": # end of document # check remaining footnote refs for footnoteLink in footnoteBuffer: checkFootnoteHrefs(modelXbrl, footnoteLink) elt.clear() elif ns == XbrlConst.link: if ln in ("schemaRef", "linkbaseRef"): modelDocument.discoverHref(mdlObj) elif ln in ("roleRef", "arcroleRef"): modelDocument.linkbaseDiscover((mdlObj,), inInstance=True) elif ln == "footnoteLink": footnoteLinks = (mdlObj,) modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True) if footnoteBufferLimit.is_finite(): footnoteBuffer.append(mdlObj) if _streamingExtensionsValidate: validator.instValidator.checkLinks(footnoteLinks) if len(footnoteBuffer) > footnoteBufferLimit: # check that hrefObjects for locators were all satisfied # drop before addition as dropped may have same id as added footnoteLink = footnoteBuffer.pop(0) checkFootnoteHrefs(modelXbrl, footnoteLink) dropFootnoteLink(modelXbrl, footnoteLink) del parentMdlObj[parentMdlObj.index(footnoteLink)] footnoteLink = None footnoteLinks = None elt.clear() elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl: numRootFacts2 += 1 modelDocument.factDiscover(mdlObj, modelXbrl.facts) XmlValidate.validate(modelXbrl, mdlObj) if _streamingExtensionsValidate: factsToCheck = (mdlObj,) validator.instValidator.checkFacts(factsToCheck) if modelXbrl.hasXDT: validator.instValidator.checkFactsDimensions(factsToCheck) del factsToCheck dropFact(modelXbrl, mdlObj, modelXbrl.facts) del parentMdlObj[parentMdlObj.index(mdlObj)] if numRootFacts2 % 1000 == 0: modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(numRootFacts2, numRootFacts1, 100.0 * numRootFacts2 / numRootFacts1), minTimeToShow=20.0) # get rid of root element from iterparse's tree elt.clear() while elt.getprevious() is not None: # cleans up any prior siblings del elt.getparent()[0] mdlObj = None # deref logSyntaxErrors(parsercontext) del parsercontext if validator is not None: validator.close() _file.close() modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt) return modelDocument
def instanceDiscover(self, xbrlElement): self.schemaLinkbaseRefsDiscover(xbrlElement) self.linkbaseDiscover(xbrlElement,inInstance=True) # for role/arcroleRefs and footnoteLinks self.instanceContentsDiscover(xbrlElement) XmlValidate.validate(self.modelXbrl, xbrlElement) # validate instance elements
def produceOutputFact(xpCtx, formula, result): priorErrorCount = len(xpCtx.modelXbrl.errors) # assemble context conceptQname = aspectValue(xpCtx, formula, Aspect.CONCEPT, "xbrlfe:missingConceptRule") if isinstance(conceptQname, VariableBindingError): xpCtx.modelXbrl.error(conceptQname.err, _("Formula %(xlinkLabel)s concept: %(concept)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, concept=conceptQname.msg) modelConcept = None else: modelConcept = xpCtx.modelXbrl.qnameConcepts[conceptQname] if modelConcept is None or not modelConcept.isItem: xpCtx.modelXbrl.error("xbrlfe:missingConceptRule", _("Formula %(xlinkLabel)s concept %(concept)s is not an item"), modelObject=formula, xlinkLabel=formula.xlinkLabel, concept=conceptQname) # entity entityIdentScheme = aspectValue(xpCtx, formula, Aspect.SCHEME, "xbrlfe:missingEntityIdentifierRule") if isinstance(entityIdentScheme, VariableBindingError): xpCtx.modelXbrl.error(str(entityIdentScheme), _("Formula %(xlinkLabel)s entity identifier scheme: %(scheme)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, scheme=entityIdentScheme.msg) entityIdentValue = None else: entityIdentValue = aspectValue(xpCtx, formula, Aspect.VALUE, "xbrlfe:missingEntityIdentifierRule") if isinstance(entityIdentValue, VariableBindingError): xpCtx.modelXbrl.error(str(entityIdentScheme), _("Formula %(xlinkLabel)s entity identifier value: %(entityIdentifier)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, entityIdentifier=entityIdentValue.msg) # period periodType = aspectValue(xpCtx, formula, Aspect.PERIOD_TYPE, "xbrlfe:missingPeriodRule") periodStart = None periodEndInstant = None if isinstance(periodType, VariableBindingError): xpCtx.modelXbrl.error(str(periodType), _("Formula %(xlinkLabel)s period type: %(periodType)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, periodType=periodType.msg) elif periodType == "instant": periodEndInstant = aspectValue(xpCtx, formula, Aspect.INSTANT, "xbrlfe:missingPeriodRule") if isinstance(periodEndInstant, VariableBindingError): xpCtx.modelXbrl.error(str(periodEndInstant), _("Formula %(xlinkLabel)s period end: %(period)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, period=periodEndInstant.msg) elif periodType == "duration": periodStart = aspectValue(xpCtx, formula, Aspect.START, "xbrlfe:missingPeriodRule") if isinstance(periodStart, VariableBindingError): xpCtx.modelXbrl.error(str(periodStart), _("Formula %(xlinkLabel)s period start: %(period)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, period=periodStart.msg) periodEndInstant = aspectValue(xpCtx, formula, Aspect.END, "xbrlfe:missingPeriodRule") if isinstance(periodEndInstant, VariableBindingError): xpCtx.modelXbrl.error(str(periodEndInstant), _("Formula %(xlinkLabel)s period end: %(period)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, period=periodEndInstant.msg) # unit if modelConcept is not None and modelConcept.isNumeric: unitSource = aspectValue(xpCtx, formula, Aspect.UNIT_MEASURES, None) multDivBy = aspectValue(xpCtx, formula, Aspect.MULTIPLY_BY, "xbrlfe:missingUnitRule") if isinstance(multDivBy, VariableBindingError): xpCtx.modelXbrl.error(str(multDivBy) if isinstance(multDivBy, VariableBindingError) else "xbrlfe:missingUnitRule", _("Formula %(xlinkLabel)s unit: %(unit)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, unit=multDivBy.msg) multiplyBy = (); divideBy = () # prevent errors later if bad else: divMultBy = aspectValue(xpCtx, formula, Aspect.DIVIDE_BY, "xbrlfe:missingUnitRule") if isinstance(divMultBy, VariableBindingError): xpCtx.modelXbrl.error(str(multDivBy) if isinstance(divMultBy, VariableBindingError) else "xbrlfe:missingUnitRule", _("Formula %(xlinkLabel)s unit: %(unit)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, unit=divMultBy.msg) multiplyBy = (); divideBy = () # prevent errors later if bad else: multiplyBy = unitSource[0] + multDivBy[0] + divMultBy[1] divideBy = unitSource[1] + multDivBy[1] + divMultBy[0] # remove cancelling mult/div units lookForCommonUnits = True while lookForCommonUnits: lookForCommonUnits = False for commonUnit in multiplyBy: if commonUnit in divideBy: multiplyBy.remove(commonUnit) divideBy.remove(commonUnit) lookForCommonUnits = True break if len(multiplyBy) == 0: # if no units add pure multiplyBy.append(XbrlConst.qnXbrliPure) # dimensions segOCCs = [] scenOCCs = [] if formula.aspectModel == "dimensional": dimAspects = {} dimQnames = aspectValue(xpCtx, formula, Aspect.DIMENSIONS, None) if dimQnames: for dimQname in dimQnames: dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname] dimErr = "xbrlfe:missing{0}DimensionRule".format("typed" if dimConcept is not None and dimConcept.isTypedDimension else "explicit") dimValue = aspectValue(xpCtx, formula, dimQname, dimErr) if isinstance(dimValue, VariableBindingError): xpCtx.modelXbrl.error(dimErr, _("Formula %(xlinkLabel)s dimension %(dimension)s: %(value)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, dimension=dimQname, value=dimValue.msg) elif dimValue is not None and xpCtx.modelXbrl.qnameDimensionDefaults.get(dimQname) != dimValue: dimAspects[dimQname] = dimValue segOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SEGMENT, None) scenOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SCENARIO, None) else: dimAspects = None # non-dimensional segOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SEGMENT, None) scenOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SCENARIO, None) if priorErrorCount < len(xpCtx.modelXbrl.errors): return None # had errors, don't produce output fact # does context exist in out instance document outputInstanceQname = formula.outputInstanceQname outputXbrlInstance = xpCtx.inScopeVars[outputInstanceQname] xbrlElt = outputXbrlInstance.modelDocument.xmlRootElement # in source instance document # add context prevCntx = outputXbrlInstance.matchContext( entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, dimAspects, segOCCs, scenOCCs) if prevCntx is not None: cntxId = prevCntx.id newCntxElt = prevCntx else: cntxId = 'c-{0:02n}'.format( len(outputXbrlInstance.contexts) + 1) newCntxElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId), afterSibling=xpCtx.outputLastContext.get(outputInstanceQname)) xpCtx.outputLastContext[outputInstanceQname] = newCntxElt entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity") XmlUtil.addChild(entityElt, XbrlConst.xbrli, "identifier", attributes=("scheme", entityIdentScheme), text=entityIdentValue) periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period") if periodType == "forever": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever") elif periodType == "instant": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "instant", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) elif periodType == "duration": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "startDate", text=XmlUtil.dateunionValue(periodStart)) XmlUtil.addChild(periodElt, XbrlConst.xbrli, "endDate", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) segmentElt = None scenarioElt = None from arelle.ModelInstanceObject import ModelDimensionValue if dimAspects: for dimQname in sorted(dimAspects.keys()): dimValue = dimAspects[dimQname] if isinstance(dimValue, ModelDimensionValue): if dimValue.isExplicit: dimMemberQname = dimValue.memberQname contextEltName = dimValue.contextElement else: # qname for explicit or node for typed dimMemberQname = dimValue contextEltName = xpCtx.modelXbrl.qnameDimensionContextElement.get(dimQname) if contextEltName == "segment": if segmentElt is None: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") contextElt = segmentElt elif contextEltName == "scenario": if scenarioElt is None: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") contextElt = scenarioElt else: continue dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname] dimAttr = ("dimension", XmlUtil.addQnameValue(xbrlElt, dimConcept.qname)) if dimConcept.isTypedDimension: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:typedMember", attributes=dimAttr) if isinstance(dimValue, ModelDimensionValue) and dimValue.isTyped: XmlUtil.copyChildren(dimElt, dimValue) elif dimMemberQname: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:explicitMember", attributes=dimAttr, text=XmlUtil.addQnameValue(xbrlElt, dimMemberQname)) if segOCCs: if segmentElt is None: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") XmlUtil.copyNodes(segmentElt, segOCCs) if scenOCCs: if scenarioElt is None: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") XmlUtil.copyNodes(scenarioElt, scenOCCs) outputXbrlInstance.modelDocument.contextDiscover(newCntxElt) XmlValidate.validate(outputXbrlInstance, newCntxElt) # does unit exist # add unit if modelConcept.isNumeric: prevUnit = outputXbrlInstance.matchUnit(multiplyBy, divideBy) if prevUnit is not None: unitId = prevUnit.id newUnitElt = prevUnit else: unitId = 'u-{0:02n}'.format( len(outputXbrlInstance.units) + 1) newUnitElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId), afterSibling=xpCtx.outputLastUnit.get(outputInstanceQname)) xpCtx.outputLastUnit[outputInstanceQname] = newUnitElt if len(divideBy) == 0: for multiply in multiplyBy: XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) else: divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide") numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator") denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator") for multiply in multiplyBy: XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) for divide in divideBy: XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, divide)) outputXbrlInstance.modelDocument.unitDiscover(newUnitElt) XmlValidate.validate(outputXbrlInstance, newUnitElt) # add fact attrs = [("contextRef", cntxId)] precision = None decimals = None if modelConcept.isNumeric: attrs.append(("unitRef", unitId)) value = formula.evaluate(xpCtx) valueSeqLen = len(value) if valueSeqLen > 1: xpCtx.modelXbrl.error("xbrlfe:nonSingletonOutputValue", _("Formula %(xlinkLabel)s value is a sequence of length %(valueSequenceLength)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, valueSequenceLength=valueSeqLen) else: if valueSeqLen == 0: #xsi:nil if no value attrs.append((XbrlConst.qnXsiNil, "true")) v = None else: # add precision/decimals for non-fraction numerics if modelConcept.isNumeric and not modelConcept.isFraction: if formula.hasDecimals: decimals = formula.evaluateRule(xpCtx, Aspect.DECIMALS) attrs.append(("decimals", decimals)) else: if formula.hasPrecision: precision = formula.evaluateRule(xpCtx, Aspect.PRECISION) else: precision = 0 attrs.append(("precision", precision)) x = value[0] if isinstance(x,float): from math import (log10, isnan, isinf, fabs) if (isnan(x) or (precision and (isinf(precision) or precision == 0)) or (decimals and isinf(decimals))): v = xsString(xpCtx, x) elif decimals is not None: v = "%.*f" % ( int(decimals), x) elif precision is not None and precision != 0: a = fabs(x) log = log10(a) if a != 0 else 0 v = "%.*f" % ( int(precision) - int(log) - (1 if a >= 1 else 0), x) else: # no implicit precision yet v = xsString(xpCtx, x) elif isinstance(x,QName): v = XmlUtil.addQnameValue(xbrlElt, x) elif isinstance(x,datetime.datetime): v = XmlUtil.dateunionValue(x) else: v = xsString(xpCtx, x) newFact = XmlUtil.addChild(xbrlElt, conceptQname, attributes=attrs, text=v, afterSibling=xpCtx.outputLastFact.get(outputInstanceQname)) xpCtx.outputLastFact[outputInstanceQname] = newFact outputXbrlInstance.modelDocument.factDiscover(newFact, outputXbrlInstance.facts) XmlValidate.validate(outputXbrlInstance, newFact) return newFact
def backgroundSaveInstance(self, newFilename=None): cntlr = self.modelXbrl.modelManager.cntlr if newFilename and self.modelXbrl.modelDocument.type != ModelDocument.Type.INSTANCE: self.modelXbrl.modelManager.showStatus(_("creating new instance {0}").format(os.path.basename(newFilename))) self.modelXbrl.modelManager.cntlr.waitForUiThreadQueue() # force status update self.modelXbrl.createInstance(newFilename) # creates an instance as this modelXbrl's entrypoing instance = self.modelXbrl cntlr.showStatus(_("Saving {0}").format(instance.modelDocument.basename)) cntlr.waitForUiThreadQueue() # force status update newCntx = ModelXbrl.AUTO_LOCATE_ELEMENT newUnit = ModelXbrl.AUTO_LOCATE_ELEMENT # check user keyed changes for bodyCell in self.gridBody.winfo_children(): if isinstance(bodyCell, gridCell) and bodyCell.isChanged: value = bodyCell.value objId = bodyCell.objectId if objId: if objId[0] == "f": factPrototypeIndex = int(objId[1:]) factPrototype = self.factPrototypes[factPrototypeIndex] concept = factPrototype.concept entityIdentScheme = self.newFactItemOptions.entityIdentScheme entityIdentValue = self.newFactItemOptions.entityIdentValue periodType = factPrototype.concept.periodType periodStart = self.newFactItemOptions.startDateDate if periodType == "duration" else None periodEndInstant = self.newFactItemOptions.endDateDate qnameDims = factPrototype.context.qnameDims prevCntx = instance.matchContext( entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, qnameDims, [], []) if prevCntx is not None: cntxId = prevCntx.id else: # need new context newCntx = instance.createContext(entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, concept.qname, qnameDims, [], [], afterSibling=newCntx) cntxId = newCntx.id # new context if concept.isNumeric: if concept.isMonetary: unitMeasure = qname(XbrlConst.iso4217, self.newFactItemOptions.monetaryUnit) unitMeasure.prefix = "iso4217" # want to save with a recommended prefix decimals = self.newFactItemOptions.monetaryDecimals elif concept.isShares: unitMeasure = XbrlConst.qnXbrliShares decimals = self.newFactItemOptions.nonMonetaryDecimals else: unitMeasure = XbrlConst.qnXbrliPure decimals = self.newFactItemOptions.nonMonetaryDecimals prevUnit = instance.matchUnit([unitMeasure],[]) if prevUnit is not None: unitId = prevUnit.id else: newUnit = instance.createUnit([unitMeasure],[], afterSibling=newUnit) unitId = newUnit.id attrs = [("contextRef", cntxId)] if concept.isNumeric: attrs.append(("unitRef", unitId)) attrs.append(("decimals", decimals)) value = Locale.atof(self.modelXbrl.locale, value, str.strip) newFact = instance.createFact(concept.qname, attributes=attrs, text=value) bodyCell.objectId = newFact.objectId() # switch cell to now use fact ID if self.factPrototypes[factPrototypeIndex] is not None: self.factPrototypes[factPrototypeIndex].clear() self.factPrototypes[factPrototypeIndex] = None #dereference fact prototype else: # instance fact, not prototype fact = self.modelXbrl.modelObject(objId) if fact.concept.isNumeric: value = Locale.atof(self.modelXbrl.locale, value, str.strip) if fact.value != value: if fact.concept.isNumeric and fact.isNil != (not value): fact.isNil = not value if value: # had been nil, now it needs decimals fact.decimals = (self.newFactItemOptions.monetaryDecimals if fact.concept.isMonetary else self.newFactItemOptions.nonMonetaryDecimals) fact.text = value XmlValidate.validate(instance, fact) bodyCell.isChanged = False # clear change flag instance.saveInstance(newFilename) # may override prior filename for instance from main menu cntlr.showStatus(_("Saved {0}").format(instance.modelDocument.basename), clearAfter=3000)
def createContext(self, entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, priItem, dims, segOCCs, scenOCCs, afterSibling=None, beforeSibling=None): xbrlElt = self.modelDocument.xmlRootElement if afterSibling == AUTO_LOCATE_ELEMENT: afterSibling = XmlUtil.lastChild(xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context")) cntxId = 'c-{0:02n}'.format( len(self.contexts) + 1) newCntxElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId), afterSibling=afterSibling, beforeSibling=beforeSibling) entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity") XmlUtil.addChild(entityElt, XbrlConst.xbrli, "identifier", attributes=("scheme", entityIdentScheme), text=entityIdentValue) periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period") if periodType == "forever": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever") elif periodType == "instant": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "instant", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) elif periodType == "duration": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "startDate", text=XmlUtil.dateunionValue(periodStart)) XmlUtil.addChild(periodElt, XbrlConst.xbrli, "endDate", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) segmentElt = None scenarioElt = None from arelle.ModelInstanceObject import ModelDimensionValue if dims: # requires primary item to determin ambiguous concepts ''' in theory we have to check full set of dimensions for validity in source or any other context element, but for shortcut will see if each dimension is already reported in an unambiguous valid contextElement ''' from arelle.PrototypeInstanceObject import FactPrototype, ContextPrototype, DimValuePrototype fp = FactPrototype(self, priItem, dims.items()) # force trying a valid prototype's context Elements if not isFactDimensionallyValid(self, fp, setPrototypeContextElements=True): self.info("arelleLinfo", _("Create context for %(priItem)s, cannot determine valid context elements, no suitable hypercubes"), modelObject=self, priItem=priItem) fpDims = fp.context.qnameDims for dimQname in sorted(fpDims.keys()): dimValue = fpDims[dimQname] if isinstance(dimValue, DimValuePrototype): dimMemberQname = dimValue.memberQname # None if typed dimension contextEltName = dimValue.contextElement else: # qname for explicit or node for typed dimMemberQname = None contextEltName = None if contextEltName == "segment": if segmentElt is None: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") contextElt = segmentElt elif contextEltName == "scenario": if scenarioElt is None: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") contextElt = scenarioElt else: self.info("arelleLinfo", _("Create context, %(dimension)s, cannot determine context element, either no all relationship or validation issue"), modelObject=self, dimension=dimQname), continue dimConcept = self.qnameConcepts[dimQname] dimAttr = ("dimension", XmlUtil.addQnameValue(xbrlElt, dimConcept.qname)) if dimConcept.isTypedDimension: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:typedMember", attributes=dimAttr) if isinstance(dimValue, (ModelDimensionValue, DimValuePrototype)) and dimValue.isTyped: XmlUtil.copyNodes(dimElt, dimValue.typedMember) elif dimMemberQname: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:explicitMember", attributes=dimAttr, text=XmlUtil.addQnameValue(xbrlElt, dimMemberQname)) if segOCCs: if segmentElt is None: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") XmlUtil.copyNodes(segmentElt, segOCCs) if scenOCCs: if scenarioElt is None: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") XmlUtil.copyNodes(scenarioElt, scenOCCs) self.modelDocument.contextDiscover(newCntxElt) XmlValidate.validate(self, newCntxElt) return newCntxElt