def createFact(self, conceptQname, attributes=None, text=None, parent=None, afterSibling=None, beforeSibling=None):
if parent is None: parent = self.modelDocument.xmlRootElement
newFact = XmlUtil.addChild(parent, conceptQname, attributes=attributes, text=text,
afterSibling=afterSibling, beforeSibling=beforeSibling)
self.modelDocument.factDiscover(newFact, parentElement=parent)
XmlValidate.validate(self, newFact)
return newFact
def contextDiscover(self, cntxElement):
id = cntxElement.getAttribute("id")
self.modelXbrl.contexts[id] = modelContext = ModelObject.createContext(
self, cntxElement)
for container in (("segment", modelContext.segDimValues,
modelContext.segNonDimValues),
("scenario", modelContext.scenDimValues,
modelContext.scenNonDimValues)):
containerName, containerDimValues, containerNonDimValues = container
for containerElement in XmlUtil.descendants(
cntxElement, XbrlConst.xbrli, containerName):
for sElt in containerElement.childNodes:
if sElt.nodeType == 1:
if sElt.namespaceURI == XbrlConst.xbrldi and sElt.localName in (
"explicitMember", "typedMember"):
XmlValidate.validate(self.modelXbrl, sElt)
modelDimValue = ModelObject.createDimensionValue(
self, sElt)
dimension = modelDimValue.dimension
if dimension and dimension not in containerDimValues:
containerDimValues[dimension] = modelDimValue
else:
modelContext.errorDimValues.append(
modelDimValue)
modelContext.qnameDims[
modelDimValue.
dimensionQname] = modelDimValue # both seg and scen
else:
containerNonDimValues.append(sElt)
def attributeDict(modelXbrl, elt, exclusions=set(), equalMode=S_EQUAL, excludeIDs=NO_IDs_EXCLUDED, ns2ns1Tbl=None, keyByTag=False, distinguishNaNs=False):
if not hasattr(elt,"xValid"):
XmlValidate.validate(modelXbrl, elt)
attrs = {}
# TBD: replace with validated attributes
for modelAttribute in elt.xAttributes.values():
attrTag = modelAttribute.attrTag
ns, sep, localName = attrTag.partition('}')
attrNsURI = ns[1:] if sep else None
if ns2ns1Tbl and attrNsURI in ns2ns1Tbl:
attrNsURI = ns2ns1Tbl[attrNsURI]
if (attrTag not in exclusions and
(attrNsURI is None or attrNsURI not in exclusions)):
if keyByTag:
qname = attrTag
elif attrNsURI is not None:
qname = QName(None, attrNsURI, localName)
else:
qname = QName(None, None, attrTag)
try:
if excludeIDs and modelAttribute.xValid == XmlValidate.VALID_ID:
continue
if modelAttribute.xValid != XmlValidate.UNKNOWN:
value = modelAttribute.sValue if equalMode <= S_EQUAL2 else modelAttribute.xValue
else: # unable to validate, no schema definition, use string value of attribute
value = modelAttribute.text
if distinguishNaNs and isinstance(value,float) and math.isnan(value):
value = (value,elt)
attrs[qname] = value
except KeyError:
pass # what should be done if attribute failed to have psvi value
return attrs
def doc(xc, p, contextItem, args):
if len(args) != 1: raise XPathContext.FunctionNumArgs()
uri = stringArg(xc, args, 0, "xs:string", emptyFallback=None)
if uri is None:
return ()
if xc.progHeader is None or xc.progHeader.element is None:
raise XPathContext.XPathException(
p, 'err:FODC0005',
_('Function xf:doc no formula resource element for {0}').format(
uri))
if not UrlUtil.isValid(uri):
raise XPathContext.XPathException(
p, 'err:FODC0005',
_('Function xf:doc $uri is not valid {0}').format(uri))
normalizedUri = xc.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
uri,
xc.progHeader.element.modelDocument.baseForElement(
xc.progHeader.element))
if normalizedUri in xc.modelXbrl.urlDocs:
return xc.modelXbrl.urlDocs[normalizedUri].xmlDocument
modelDocument = ModelDocument.load(xc.modelXbrl, normalizedUri)
if modelDocument is None:
raise XPathContext.XPathException(
p, 'err:FODC0005',
_('Function xf:doc $uri not successfully loaded {0}').format(uri))
# assure that document is validated
XmlValidate.validate(xc.modelXbrl, modelDocument.xmlRootElement)
return modelDocument.xmlDocument
def sEqual(dts1, elt1, elt2, equalMode=S_EQUAL, excludeIDs=NO_IDs_EXCLUDED, dts2=None, ns2ns1Tbl=None):
if dts2 is None: dts2 = dts1
if elt1.localName != elt2.localName:
return False
if ns2ns1Tbl and elt2.namespaceURI in ns2ns1Tbl:
if elt1.namespaceURI != ns2ns1Tbl[elt2.namespaceURI]:
return False
elif elt1.namespaceURI != elt2.namespaceURI:
return False
if not hasattr(elt1,u"xValid"):
XmlValidate.validate(dts1, elt1)
if not hasattr(elt2,u"xValid"):
XmlValidate.validate(dts2, elt2)
children1 = childElements(elt1)
children2 = childElements(elt2)
if len(children1) != len(children2):
return False
if (not xEqual(elt1, elt2,
# must use stringValue for nested contents of mixed content
# ... this is now in xValue for mixed content
# VALIDATE_BY_STRING_VALUE if len(children1) and elt1.xValid == VALID else
equalMode
) or
attributeDict(dts1, elt1, (), equalMode, excludeIDs) !=
attributeDict(dts2, elt2, (), equalMode, excludeIDs, ns2ns1Tbl)):
return False
excludeChildIDs = excludeIDs if excludeIDs != TOP_IDs_EXCLUDED else NO_IDs_EXCLUDED
for i in xrange( len(children1) ):
if not sEqual(dts1, children1[i], children2[i], equalMode, excludeChildIDs, dts2, ns2ns1Tbl):
return False
return True
def sEqual(dts1,
elt1,
elt2,
equalMode=S_EQUAL,
excludeIDs=NO_IDs_EXCLUDED,
dts2=None,
ns2ns1Tbl=None):
if dts2 is None: dts2 = dts1
if elt1.localName != elt2.localName:
return False
if ns2ns1Tbl and elt2.namespaceURI in ns2ns1Tbl:
if elt1.namespaceURI != ns2ns1Tbl[elt2.namespaceURI]:
return False
elif elt1.namespaceURI != elt2.namespaceURI:
return False
if not hasattr(elt1, "xValid"):
XmlValidate.validate(dts1, elt1)
if not hasattr(elt2, "xValid"):
XmlValidate.validate(dts2, elt2)
if (not xEqual(elt1, elt2, equalMode)
or attributeDict(dts1, elt1, (), equalMode, excludeIDs) !=
attributeDict(dts2, elt2, (), equalMode, excludeIDs, ns2ns1Tbl)):
return False
children1 = childElements(elt1)
children2 = childElements(elt2)
if len(children1) != len(children2):
return False
excludeChildIDs = excludeIDs if excludeIDs != TOP_IDs_EXCLUDED else NO_IDs_EXCLUDED
for i in range(len(children1)):
if not sEqual(dts1, children1[i], children2[i], equalMode,
excludeChildIDs, dts2, ns2ns1Tbl):
return False
return True
def equalityHash(elt, equalMode=S_EQUAL, excludeIDs=NO_IDs_EXCLUDED):
if isinstance(elt, ModelObject):
try:
if equalMode == S_EQUAL:
return elt._hashSEqual
else:
return elt._hashXpathEqual
except AttributeError:
dts = elt.modelXbrl
if not hasattr(elt,"xValid"):
XmlValidate.validate(dts, elt)
hashableValue = elt.sValue if equalMode == S_EQUAL else elt.xValue
if isinstance(hashableValue,float) and math.isnan(hashableValue):
hashableValue = (hashableValue,elt) # ensure this NaN only compares to itself and no other NaN
_hash = hash((elt.elementQname,
hashableValue,
tuple(attributeDict(dts, elt, (), equalMode, excludeIDs, distinguishNaNs=True).items()),
tuple(equalityHash(child,equalMode,excludeIDs) for child in childElements(elt))
))
if equalMode == S_EQUAL:
elt._hashSEqual = _hash
else:
elt._hashXpathEqual = _hash
return _hash
elif isinstance(elt, (tuple,list,set)):
return hash( tuple(equalityHash(i) for i in elt) )
else:
return hash(None)
def sEqual(dts1, elt1, elt2, equalMode=S_EQUAL, excludeIDs=NO_IDs_EXCLUDED, dts2=None, ns2ns1Tbl=None):
if dts2 is None: dts2 = dts1
if elt1.localName != elt2.localName:
return False
if ns2ns1Tbl and elt2.namespaceURI in ns2ns1Tbl:
if elt1.namespaceURI != ns2ns1Tbl[elt2.namespaceURI]:
return False
elif elt1.namespaceURI != elt2.namespaceURI:
return False
if not hasattr(elt1,"xValid"):
XmlValidate.validate(dts1, elt1)
if not hasattr(elt2,"xValid"):
XmlValidate.validate(dts2, elt2)
if (not xEqual(elt1, elt2, equalMode) or
attributeDict(dts1, elt1, (), equalMode, excludeIDs) !=
attributeDict(dts2, elt2, (), equalMode, excludeIDs, ns2ns1Tbl)):
return False
children1 = childElements(elt1)
children2 = childElements(elt2)
if len(children1) != len(children2):
return False
excludeChildIDs = excludeIDs if excludeIDs != TOP_IDs_EXCLUDED else NO_IDs_EXCLUDED
for i in range( len(children1) ):
if not sEqual(dts1, children1[i], children2[i], equalMode, excludeChildIDs, dts2, ns2ns1Tbl):
return False
return True
def attributeDict(modelXbrl, elt, exclusions=set(), equalMode=S_EQUAL, excludeIDs=NO_IDs_EXCLUDED, ns2ns1Tbl=None, keyByTag=False, distinguishNaNs=False):
if not hasattr(elt,"xValid"):
XmlValidate.validate(modelXbrl, elt)
attrs = {}
# TBD: replace with validated attributes
for modelAttribute in elt.xAttributes.values():
attrTag = modelAttribute.attrTag
ns, sep, localName = attrTag.partition('}')
attrNsURI = ns[1:] if sep else None
if ns2ns1Tbl and attrNsURI in ns2ns1Tbl:
attrNsURI = ns2ns1Tbl[attrNsURI]
if (attrTag not in exclusions and
(attrNsURI is None or attrNsURI not in exclusions)):
if keyByTag:
qname = attrTag
elif attrNsURI is not None:
qname = QName(None, attrNsURI, localName)
else:
qname = QName(None, None, attrTag)
try:
if excludeIDs and modelAttribute.xValid == XmlValidate.VALID_ID:
continue
value = modelAttribute.sValue if equalMode <= S_EQUAL2 else modelAttribute.xValue
if distinguishNaNs and isinstance(value,float) and math.isnan(value):
value = (value,elt)
attrs[qname] = value
except KeyError:
pass # what should be done if attribute failed to have psvi value
return attrs
def equalityHash(elt, equalMode=S_EQUAL, excludeIDs=NO_IDs_EXCLUDED):
if isinstance(elt, ModelObject):
try:
if equalMode == S_EQUAL:
return elt._hashSEqual
else:
return elt._hashXpathEqual
except AttributeError:
dts = elt.modelXbrl
if not hasattr(elt, "xValid"):
XmlValidate.validate(dts, elt)
hashableValue = elt.sValue if equalMode == S_EQUAL else elt.xValue
if isinstance(hashableValue, float) and math.isnan(hashableValue):
hashableValue = (
hashableValue, elt
) # ensure this NaN only compares to itself and no other NaN
_hash = hash((elt.elementQname, hashableValue,
tuple(
attributeDict(dts,
elt, (),
equalMode,
excludeIDs,
distinguishNaNs=True).items()),
tuple(
equalityHash(child, equalMode, excludeIDs)
for child in childElements(elt))))
if equalMode == S_EQUAL:
elt._hashSEqual = _hash
else:
elt._hashXpathEqual = _hash
return _hash
elif isinstance(elt, (tuple, list, set)):
return hash(tuple(equalityHash(i) for i in elt))
else:
return hash(None)
def createUnit(self, multiplyBy, divideBy, afterSibling=None, beforeSibling=None):
"""Creates new unit, by measures, as in formula usage, if any
:param multiplyBy: List of multiply-by measure QNames (or top level measures if no divideBy)
:type multiplyBy: [QName]
:param divideBy: List of multiply-by measure QNames (or empty list if no divideBy)
:type divideBy: [QName]
:param beforeSibling: lxml element in instance to insert new concept before
:type beforeSibling: ModelObject
:param afterSibling: lxml element in instance to insert new concept after
:type afterSibling: ModelObject
:returns: ModelUnit -- New unit object
"""
xbrlElt = self.modelDocument.xmlRootElement
if afterSibling == AUTO_LOCATE_ELEMENT:
afterSibling = XmlUtil.lastChild(xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context", "unit"))
unitId = 'u-{0:02n}'.format( len(self.units) + 1)
newUnitElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId),
afterSibling=afterSibling, beforeSibling=beforeSibling)
if len(divideBy) == 0:
for multiply in multiplyBy:
XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply))
else:
divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide")
numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator")
denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator")
for multiply in multiplyBy:
XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply))
for divide in divideBy:
XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, divide))
self.modelDocument.unitDiscover(newUnitElt)
XmlValidate.validate(self, newUnitElt)
return newUnitElt
def xfxc_element(xc, p, contextItem, args):
if not 2 <= len(args) <= 4: raise XPathContext.FunctionNumArgs()
qn = qnameArg(xc, p, args, 0, 'QName', emptyFallback=None)
attrArg = args[1] if isinstance(args[1], (list, tuple)) else (args[1], )
# attributes have to be pairs
if attrArg:
if len(attrArg) & 1 or any(
not isinstance(attrArg[i], (QName, _STR_BASE))
for i in range(0, len(attrArg), 2)):
raise XPathContext.FunctionArgType(
1,
"((xs:qname|xs:string),xs:anyAtomicValue)",
errCode="xfxce:AttributesNotNameValuePairs")
else:
attrParam = [
(attrArg[i], attrArg[i + 1]
) # need name-value pairs for XmlUtil function
for i in range(0, len(attrArg), 2)
]
else:
attrParam = None
value = atomicArg(xc, p, args, 2, "xs:anyAtomicType", emptyFallback='')
if not value: # be sure '' is None so no text node is created
value = None
if len(args) < 4:
childElements = None
else:
childElements = xc.flattenSequence(args[3])
# scratchpad instance document emulates fn:doc( ) to hold XML nodes
scratchpadXmlDocUrl = "http://www.xbrl.org/2012/function/creation/xml_scratchpad.xml"
if scratchpadXmlDocUrl in xc.modelXbrl.urlDocs:
modelDocument = xc.modelXbrl.urlDocs[scratchpadXmlDocUrl]
else:
# create scratchpad xml document
# this will get the fake instance document in the list of modelXbrl docs so that it is garbage collected
from arelle import ModelDocument
modelDocument = ModelDocument.create(
xc.modelXbrl,
ModelDocument.Type.UnknownXML,
scratchpadXmlDocUrl,
initialXml=
"<xfc:dummy xmlns:xfc='http://www.xbrl.org/2012/function/creation'/>"
)
newElement = XmlUtil.addChild(modelDocument.xmlRootElement,
qn,
attributes=attrParam,
text=value)
if childElements:
for element in childElements:
if isinstance(element, etree.ElementBase):
newElement.append(element)
# node myst be validated for use in instance creation (typed dimension references)
XmlValidate.validate(xc.modelXbrl, newElement)
return newElement
def instanceDiscover(self, xbrlElement):
self.schemaLinkbaseRefsDiscover(xbrlElement)
self.linkbaseDiscover(
xbrlElement,
inInstance=True) # for role/arcroleRefs and footnoteLinks
self.instanceContentsDiscover(xbrlElement)
XmlValidate.validate(self.modelXbrl,
xbrlElement) # validate instance elements
def usedOns(self):
try:
return self._usedOns
except AttributeError:
XmlValidate.validate(self.modelXbrl, self)
self._usedOns = set(usedOn.xValue
for usedOn in self.iterdescendants("{http://www.xbrl.org/2003/linkbase}usedOn")
if isinstance(usedOn,ModelObject))
return self._usedOns
def xEqual(elt1, elt2, equalMode=S_EQUAL):
if not hasattr(elt1,"xValid"):
XmlValidate.validate(elt1.modelXbrl, elt1)
if not hasattr(elt2,"xValid"):
XmlValidate.validate(elt2.modelXbrl, elt2)
if equalMode == S_EQUAL or (equalMode == S_EQUAL2 and not isinstance(elt1.sValue, QName)):
return elt1.sValue == elt2.sValue
else:
return elt1.xValue == elt2.xValue
def xEqual(elt1, elt2, equalMode=S_EQUAL):
if not hasattr(elt1,"xValid"):
XmlValidate.validate(elt1.modelXbrl, elt1)
if not hasattr(elt2,"xValid"):
XmlValidate.validate(elt2.modelXbrl, elt2)
if equalMode == S_EQUAL or (equalMode == S_EQUAL2 and not isinstance(elt1.sValue, QName)):
return elt1.sValue == elt2.sValue
else:
return elt1.xValue == elt2.xValue
def usedOns(self):
try:
return self._usedOns
except AttributeError:
XmlValidate.validate(self.modelXbrl, self)
self._usedOns = set(
usedOn.xValue for usedOn in self.iterdescendants(
"{http://www.xbrl.org/2003/linkbase}usedOn")
if isinstance(usedOn, ModelObject))
return self._usedOns
def checkAttribute(elt, isIxElt, attrTag, attrValue):
ixEltAttrDefs = ixAttrDefined.get(elt.namespaceURI, EMPTYDICT).get(elt.localName, ())
if attrTag.startswith("{"):
ns, sep, localName = attrTag[1:].partition("}")
else:
ns = None
localName = attrTag
if ns is not None and ns not in XbrlConst.ixbrlAll and attrTag not in ixEltAttrDefs:
if isIxElt:
allowedNs = allowedNonIxAttrNS.get(elt.localName, None)
if allowedNs != "##other" and ns != allowedNs:
modelXbrl.error(ixMsgCode("qualifiedAttributeNotExpected", elt),
_("Inline XBRL element %(element)s has qualified attribute %(name)s"),
modelObject=elt, element=str(elt.elementQname), name=attrTag)
if ns == XbrlConst.xbrli and elt.localName in {
"fraction", "nonFraction", "nonNumeric", "references", "relationship", "tuple"}:
modelXbrl.error(ixMsgCode("qualifiedAttributeDisallowed", elt),
_("Inline XBRL element %(element)s has disallowed attribute %(name)s"),
modelObject=elt, element=str(elt.elementQname), name=attrTag)
else:
if ns in XbrlConst.ixbrlAll:
modelXbrl.error(ixMsgCode("inlineAttributeMisplaced", elt, name="other"),
_("Inline XBRL attributes are not allowed on html elements: ix:%(name)s"),
modelObject=elt, name=localName)
elif ns not in {XbrlConst.xml, XbrlConst.xsi, XbrlConst.xhtml}:
modelXbrl.error(ixMsgCode("extensionAttributeMisplaced", ns=_ixNS),
_("Extension attributes are not allowed on html elements: %(tag)s"),
modelObject=elt, tag=attrTag)
elif isIxElt:
try:
_xsdType = ixAttrType[elt.namespaceURI][localName]
if isinstance(_xsdType, dict):
baseXsdType = _xsdType["type"]
facets = _xsdType
else:
baseXsdType = _xsdType
facets = None
XmlValidate.validateValue(modelXbrl, elt, attrTag, baseXsdType, attrValue, facets=facets)
if not (attrTag in ixEltAttrDefs or
(localName in ixEltAttrDefs and (not ns or ns in XbrlConst.ixbrlAll))):
raise KeyError
disallowedXbrliAttrs = ({"scheme", "periodType", "balance", "contextRef", "unitRef", "precision", "decimals"} -
{"fraction": {"contextRef", "unitRef"},
"nonFraction": {"contextRef", "unitRef", "decimals", "precision"},
"nonNumeric": {"contextRef"}}.get(elt.localName, set()))
disallowedAttrs = set(a for a in disallowedXbrliAttrs if elt.get(a) is not None)
if disallowedAttrs:
modelXbrl.error(ixMsgCode("inlineElementAttributes",elt),
_("Inline XBRL element %(element)s has disallowed attributes %(attributes)s"),
modelObject=elt, element=elt.elementQname, attributes=", ".join(disallowedAttrs))
except KeyError:
modelXbrl.error(ixMsgCode("attributeNotExpected",elt),
_("Attribute %(attribute)s is not expected on element ix:%(element)s"),
modelObject=elt, attribute=attrTag, element=elt.localName)
def checkAttribute(elt, isIxElt, attrTag, attrValue):
ixEltAttrDefs = ixAttrDefined.get(elt.namespaceURI, EMPTYDICT).get(elt.localName, ())
if attrTag.startswith("{"):
ns, sep, localName = attrTag[1:].partition("}")
else:
ns = None
localName = attrTag
if ns is not None and ns not in XbrlConst.ixbrlAll and attrTag not in ixEltAttrDefs:
if isIxElt:
allowedNs = allowedNonIxAttrNS.get(elt.localName, None)
if allowedNs != "##other" and ns != allowedNs:
modelXbrl.error(ixMsgCode("qualifiedAttributeNotExpected", elt),
_("Inline XBRL element %(element)s has qualified attribute %(name)s"),
modelObject=elt, element=str(elt.elementQname), name=attrTag)
if ns == XbrlConst.xbrli and elt.localName in {
"fraction", "nonFraction", "nonNumeric", "references", "relationship", "tuple"}:
modelXbrl.error(ixMsgCode("qualifiedAttributeDisallowed", elt),
_("Inline XBRL element %(element)s has disallowed attribute %(name)s"),
modelObject=elt, element=str(elt.elementQname), name=attrTag)
else:
if ns in XbrlConst.ixbrlAll:
modelXbrl.error(ixMsgCode("inlineAttributeMisplaced", elt, name="other"),
_("Inline XBRL attributes are not allowed on html elements: ix:%(name)s"),
modelObject=elt, name=localName)
elif ns not in {XbrlConst.xml, XbrlConst.xsi, XbrlConst.xhtml}:
modelXbrl.error(ixMsgCode("extensionAttributeMisplaced", ns=_ixNS),
_("Extension attributes are not allowed on html elements: %(tag)s"),
modelObject=elt, tag=attrTag)
elif isIxElt:
try:
_xsdType = ixAttrType[elt.namespaceURI][localName]
if isinstance(_xsdType, dict):
baseXsdType = _xsdType["type"]
facets = _xsdType
else:
baseXsdType = _xsdType
facets = None
XmlValidate.validateValue(modelXbrl, elt, attrTag, baseXsdType, attrValue, facets=facets)
if not (attrTag in ixEltAttrDefs or
(localName in ixEltAttrDefs and (not ns or ns in XbrlConst.ixbrlAll))):
raise KeyError
disallowedXbrliAttrs = ({"scheme", "periodType", "balance", "contextRef", "unitRef", "precision", "decimals"} -
{"fraction": {"contextRef", "unitRef"},
"nonFraction": {"contextRef", "unitRef", "decimals", "precision"},
"nonNumeric": {"contextRef"}}.get(elt.localName, set()))
disallowedAttrs = set(a for a in disallowedXbrliAttrs if elt.get(a) is not None)
if disallowedAttrs:
modelXbrl.error(ixMsgCode("inlineElementAttributes",elt),
_("Inline XBRL element %(element)s has disallowed attributes %(attributes)s"),
modelObject=elt, element=elt.elementQname, attributes=", ".join(disallowedAttrs))
except KeyError:
modelXbrl.error(ixMsgCode("attributeNotExpected",elt),
_("Attribute %(attribute)s is not expected on element ix:%(element)s"),
modelObject=elt, attribute=attrTag, element=elt.localName)
def xEqual(elt1, elt2, equalMode=S_EQUAL):
if not hasattr(elt1,u"xValid"):
XmlValidate.validate(elt1.modelXbrl, elt1)
if not hasattr(elt2,u"xValid"):
XmlValidate.validate(elt2.modelXbrl, elt2)
if equalMode == VALIDATE_BY_STRING_VALUE:
return elt1.stringValue == elt2.stringValue
elif equalMode == S_EQUAL or (equalMode == S_EQUAL2 and not isinstance(elt1.sValue, QName)):
return elt1.sValue == elt2.sValue
else:
return elt1.xValue == elt2.xValue
def typedValue(dts, element, attrQname=None):
try:
if attrQname:
node = element.getAttributeNodeNS(attrQname.namespaceURI,attrQname.localName)
else:
node = element
if node.xValid == XmlValidate.VALID:
return node.xValue
except AttributeError:
if dts:
XmlValidate.validate(dts, element, recurse=False, attrQname=attrQname)
return typedValue(None, element, attrQname=attrQname)
return None
def typedValue(dts, element, attrQname=None):
try:
if attrQname: # PSVI attribute value
modelAttribute = element.xAttributes[attrQname.clarkNotation]
if modelAttribute.xValid >= XmlValidate.VALID:
return modelAttribute.xValue
else: # PSVI element value (of text)
if element.xValid >= XmlValidate.VALID:
return element.xValue
except (AttributeError, KeyError):
if dts:
XmlValidate.validate(dts, element, recurse=False, attrQname=attrQname)
return typedValue(None, element, attrQname=attrQname)
return None
def typedValue(dts, element, attrQname=None):
try:
if attrQname: # PSVI attribute value
modelAttribute = element.xAttributes[attrQname.clarkNotation]
if modelAttribute.xValid >= XmlValidate.VALID:
return modelAttribute.xValue
else: # PSVI element value (of text)
if element.xValid >= XmlValidate.VALID:
return element.xValue
except (AttributeError, KeyError):
if dts:
XmlValidate.validate(dts, element, recurse=False, attrQname=attrQname)
return typedValue(None, element, attrQname=attrQname)
return None
def xfxc_element(xc, p, contextItem, args):
if not 2 <= len(args) <= 4: raise XPathContext.FunctionNumArgs()
qn = qnameArg(xc, p, args, 0, 'QName', emptyFallback=None)
attrArg = args[1] if isinstance(args[1],(list,tuple)) else (args[1],)
# attributes have to be pairs
if attrArg:
if len(attrArg) & 1 or any(not isinstance(attrArg[i], (QName, _STR_BASE))
for i in range(0, len(attrArg),2)):
raise XPathContext.FunctionArgType(1,"((xs:qname|xs:string),xs:anyAtomicValue)", errCode="xfxce:AttributesNotNameValuePairs")
else:
attrParam = [(attrArg[i],attrArg[i+1]) # need name-value pairs for XmlUtil function
for i in range(0, len(attrArg),2)]
else:
attrParam = None
value = atomicArg(xc, p, args, 2, "xs:anyAtomicType", emptyFallback='')
if not value: # be sure '' is None so no text node is created
value = None
if len(args) < 4:
childElements = None
else:
childElements = xc.flattenSequence(args[3])
# scratchpad instance document emulates fn:doc( ) to hold XML nodes
scratchpadXmlDocUrl = "http://www.xbrl.org/2012/function/creation/xml_scratchpad.xml"
if scratchpadXmlDocUrl in xc.modelXbrl.urlDocs:
modelDocument = xc.modelXbrl.urlDocs[scratchpadXmlDocUrl]
else:
# create scratchpad xml document
# this will get the fake instance document in the list of modelXbrl docs so that it is garbage collected
from arelle import ModelDocument
modelDocument = ModelDocument.create(xc.modelXbrl,
ModelDocument.Type.UnknownXML,
scratchpadXmlDocUrl,
initialXml="<xfc:dummy xmlns:xfc='http://www.xbrl.org/2012/function/creation'/>")
newElement = XmlUtil.addChild(modelDocument.xmlRootElement,
qn,
attributes=attrParam,
text=value)
if childElements:
for element in childElements:
if isinstance(element, etree.ElementBase):
newElement.append(element)
# node myst be validated for use in instance creation (typed dimension references)
XmlValidate.validate(xc.modelXbrl, newElement)
return newElement
def constrainingFacets(self, facetValues=None):
facetValues = facetValues if facetValues else {}
for facetElt in XmlUtil.schemaFacets(
self, ("{http://www.w3.org/2001/XMLSchema}length",
"{http://www.w3.org/2001/XMLSchema}minLength",
"{http://www.w3.org/2001/XMLSchema}maxLength",
"{http://www.w3.org/2001/XMLSchema}pattern",
"{http://www.w3.org/2001/XMLSchema}whiteSpace",
"{http://www.w3.org/2001/XMLSchema}maxInclusive",
"{http://www.w3.org/2001/XMLSchema}maxExclusive",
"{http://www.w3.org/2001/XMLSchema}minExclusive",
"{http://www.w3.org/2001/XMLSchema}totalDigits",
"{http://www.w3.org/2001/XMLSchema}fractionDigits")):
facetValue = XmlValidate.validateFacet(self, facetElt)
facetName = facetElt.localName
if facetName not in facetValues and facetValue is not None: # facetValue can be zero but not None
facetValues[facetName] = facetValue
if "enumeration" not in facetValues:
for facetElt in XmlUtil.schemaFacets(
self, ("{http://www.w3.org/2001/XMLSchema}enumeration", )):
facetValues.setdefault("enumeration",
set()).add(facetElt.get("value"))
typeDerivedFrom = self.typeDerivedFrom
if isinstance(typeDerivedFrom, ModelType):
typeDerivedFrom.constrainingFacets(facetValues)
return facetValues
def createFact(self,
conceptQname,
attributes=None,
text=None,
parent=None,
afterSibling=None,
beforeSibling=None):
if parent is None: parent = self.modelDocument.xmlRootElement
newFact = XmlUtil.addChild(parent,
conceptQname,
attributes=attributes,
text=text,
afterSibling=afterSibling,
beforeSibling=beforeSibling)
self.modelDocument.factDiscover(newFact, parentElement=parent)
XmlValidate.validate(self, newFact)
return newFact
def typedValue(dts, element, attrQname=None):
try:
if attrQname:
node = element.getAttributeNodeNS(attrQname.namespaceURI,
attrQname.localName)
else:
node = element
if node.xValid == XmlValidate.VALID:
return node.xValue
except AttributeError:
if dts:
XmlValidate.validate(dts,
element,
recurse=False,
attrQname=attrQname)
return typedValue(None, element, attrQname=attrQname)
return None
def checkAttribute(elt, isIxElt, attrTag, attrValue):
if attrTag.startswith("{"):
ns, sep, localName = attrTag[1:].partition("}")
if isIxElt:
allowedNs = nonIxAttrNS.get(elt.localName, None)
if allowedNs != "##other" and ns != allowedNs:
modelXbrl.error("ix:qualifiedAttributeNotExpected",
_("Inline XBRL element %(element)s: has qualified attribute %(name)s"),
modelObject=elt, element=str(elt.elementQname), name=attrTag)
else:
if ns in XbrlConst.ixbrlAll:
modelXbrl.error("ix:inlineAttributeMisplaced",
_("Inline XBRL attributes are not allowed on html elements: ix:%(name)s"),
modelObject=elt, name=localName)
elif ns not in {XbrlConst.xml, XbrlConst.xsi, XbrlConst.xhtml}:
modelXbrl.error("ix:extensionAttributeMisplaced",
_("Extension attributes are not allowed on html elements: %(tag)s"),
modelObject=elt, tag=attrTag)
elif isIxElt:
try:
_xsdType = ixAttrType[elt.namespaceURI][attrTag]
if isinstance(_xsdType, dict):
baseXsdType = _xsdType["type"]
facets = _xsdType
else:
baseXsdType = _xsdType
facets = None
XmlValidate.validateValue(modelXbrl, elt, attrTag, baseXsdType, attrValue, facets=facets)
disallowedXbrliAttrs = ({"scheme", "periodType", "balance", "contextRef", "unitRef", "precision", "decimals"} -
{"fraction": {"contextRef", "unitRef"},
"nonFraction": {"contextRef", "unitRef", "decimals", "precision"},
"nonNumeric": {"contextRef"}}.get(elt.localName, set()))
disallowedAttrs = [a for a in disallowedXbrliAttrs if elt.get(a) is not None]
if disallowedAttrs:
modelXbrl.error("ix:inlineElementAttributes",
_("Inline XBRL element %(element)s has disallowed attributes %(attributes)s"),
modelObject=elt, element=elt.elementQname, attributes=", ".join(disallowedAttrs))
except KeyError:
modelXbrl.error("ix:attributeNotExpected",
_("Attribute %(attribute)s is not expected on element element ix:%(element)s"),
modelObject=elt, attribute=attrTag, element=elt.localName)
def contextDiscover(self, cntxElement):
id = cntxElement.getAttribute("id")
self.modelXbrl.contexts[id] = modelContext = ModelObject.createContext(self,cntxElement)
for container in (("segment", modelContext.segDimValues, modelContext.segNonDimValues),
("scenario", modelContext.scenDimValues, modelContext.scenNonDimValues)):
containerName, containerDimValues, containerNonDimValues = container
for containerElement in XmlUtil.descendants(cntxElement, XbrlConst.xbrli, containerName):
for sElt in containerElement.childNodes:
if sElt.nodeType == 1:
if sElt.namespaceURI == XbrlConst.xbrldi and sElt.localName in ("explicitMember","typedMember"):
XmlValidate.validate(self.modelXbrl, sElt)
modelDimValue = ModelObject.createDimensionValue(self,sElt)
dimension = modelDimValue.dimension
if dimension and dimension not in containerDimValues:
containerDimValues[dimension] = modelDimValue
else:
modelContext.errorDimValues.append(modelDimValue)
modelContext.qnameDims[modelDimValue.dimensionQname] = modelDimValue # both seg and scen
else:
containerNonDimValues.append(sElt)
def endElementNS(self, name, qname):
thisQname = QName(None, *name)
if self.qnameStack and self.qnameStack[0].elementQname == thisQname:
elt = self.qnameStack.pop(0)
if elt.namespaceURI == XbrlConst.xbrli:
if elt.localName == "unit":
elt._measures = (sorted(elt._measures[0]), sorted(elt._measures[1]))
if elt.id in self.unitRefedFacts:
for fact in self.unitRefedFacts[elt.id]:
fact._unit = elt
del self.unitRefedFacts[elt.id]
elif elt.localName == "context":
if elt.id in self.contextRefedFacts:
for fact in self.contextRefedFacts[elt.id]:
fact._context = elt
del self.contextRefedFacts[elt.id]
self.currentNamespaceURI = None
self.currentLocalName = None
XmlValidate.validate(self.modelXbrl, elt, recurse=False)
pass
def doc(xc, p, contextItem, args):
if len(args) != 1: raise XPathContext.FunctionNumArgs()
uri = stringArg(xc, args, 0, "xs:string", emptyFallback=None)
if uri is None:
return ()
if xc.progHeader is None or xc.progHeader.element is None:
raise XPathContext.XPathException(p, 'err:FODC0005', _('Function xf:doc no formula resource element for {0}').format(uri))
if not UrlUtil.isValid(uri):
raise XPathContext.XPathException(p, 'err:FODC0005', _('Function xf:doc $uri is not valid {0}').format(uri))
normalizedUri = xc.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
uri,
xc.progHeader.element.modelDocument.baseForElement(xc.progHeader.element))
if normalizedUri in xc.modelXbrl.urlDocs:
return xc.modelXbrl.urlDocs[normalizedUri].xmlDocument
modelDocument = ModelDocument.load(xc.modelXbrl, normalizedUri)
if modelDocument is None:
raise XPathContext.XPathException(p, 'err:FODC0005', _('Function xf:doc $uri not successfully loaded {0}').format(uri))
# assure that document is validated
XmlValidate.validate(xc.modelXbrl, modelDocument.xmlRootElement)
return modelDocument.xmlDocument
def checkAttribute(elt, isIxElt, attrTag, attrValue):
if attrTag.startswith("{"):
ns, sep, localName = attrTag[1:].partition("}")
if isIxElt:
if ns not in (XbrlConst.xml, XbrlConst.xsi):
modelXbrl.error("ix:qualifiedAttributeNotExpected",
_("Inline XBRL element %(element)s: has qualified attribute %(name)s"),
modelObject=elt, element=str(elt.elementQname), name=attrTag)
else:
if ns in XbrlConst.ixbrlAll:
modelXbrl.error("ix:inlineAttributeMisplaced",
_("Inline XBRL attributes are not allowed on html elements: ix:%(name)s"),
modelObject=elt, name=localName)
elif ns not in {XbrlConst.xml, XbrlConst.xsi, XbrlConst.xhtml}:
modelXbrl.error("ix:extensionAttributeMisplaced",
_("Extension attributes are not allowed on html elements: %(tag)s"),
modelObject=elt, tag=attrTag)
elif isIxElt:
try:
_xsdType = ixAttrType[elt.namespaceURI][attrTag]
if isinstance(_xsdType, dict):
baseXsdType = _xsdType["type"]
facets = _xsdType
else:
baseXsdType = _xsdType
facets = None
XmlValidate.validateValue(modelXbrl, elt, attrTag, baseXsdType, attrValue, facets=facets)
disallowedXbrliAttrs = ({"scheme", "periodType", "balance", "contextRef", "unitRef", "precision", "decimals"} -
{"fraction": {"contextRef", "unitRef"},
"nonFraction": {"contextRef", "unitRef", "decimals", "precision"},
"nonNumeric": {"contextRef"}}.get(elt.localName, set()))
disallowedAttrs = [a for a in disallowedXbrliAttrs if elt.get(a) is not None]
if disallowedAttrs:
modelXbrl.error("ix:inlineElementAttributes",
_("Inline XBRL element %(element)s has disallowed attributes %(attributes)s"),
modelObject=elt, element=elt.elementQname, attributes=", ".join(disallowedAttrs))
except KeyError:
modelXbrl.error("ix:attributeNotExpected",
_("Attribute %(attribute)s is not expected on element element ix:%(element)s"),
modelObject=elt, attribute=attrTag, element=elt.localName)
def createUnit(self,
multiplyBy,
divideBy,
afterSibling=None,
beforeSibling=None):
xbrlElt = self.modelDocument.xmlRootElement
if afterSibling == AUTO_LOCATE_ELEMENT:
afterSibling = XmlUtil.lastChild(
xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType",
"arcroleType", "context", "unit"))
unitId = 'u-{0:02n}'.format(len(self.units) + 1)
newUnitElt = XmlUtil.addChild(xbrlElt,
XbrlConst.xbrli,
"unit",
attributes=("id", unitId),
afterSibling=afterSibling,
beforeSibling=beforeSibling)
if len(divideBy) == 0:
for multiply in multiplyBy:
XmlUtil.addChild(newUnitElt,
XbrlConst.xbrli,
"measure",
text=XmlUtil.addQnameValue(xbrlElt, multiply))
else:
divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide")
numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator")
denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli,
"unitDenominator")
for multiply in multiplyBy:
XmlUtil.addChild(numElt,
XbrlConst.xbrli,
"measure",
text=XmlUtil.addQnameValue(xbrlElt, multiply))
for divide in divideBy:
XmlUtil.addChild(denElt,
XbrlConst.xbrli,
"measure",
text=XmlUtil.addQnameValue(xbrlElt, divide))
self.modelDocument.unitDiscover(newUnitElt)
XmlValidate.validate(self, newUnitElt)
return newUnitElt
def constrainingFacets(self, facetValues=None):
facetValues = facetValues if facetValues else {}
for facetElt in XmlUtil.schemaFacets(self, (
"{http://www.w3.org/2001/XMLSchema}length", "{http://www.w3.org/2001/XMLSchema}minLength",
"{http://www.w3.org/2001/XMLSchema}maxLength",
"{http://www.w3.org/2001/XMLSchema}pattern", "{http://www.w3.org/2001/XMLSchema}whiteSpace",
"{http://www.w3.org/2001/XMLSchema}maxInclusive", "{http://www.w3.org/2001/XMLSchema}maxExclusive", "{http://www.w3.org/2001/XMLSchema}minExclusive",
"{http://www.w3.org/2001/XMLSchema}totalDigits", "{http://www.w3.org/2001/XMLSchema}fractionDigits")):
XmlValidate.validateFacet(self, facetElt)
facetName = facetElt.localName
if facetName not in facetValues:
facetValue = XmlValidate.validateFacet(self, facetElt)
if facetValue:
facetValues[facetName] = facetValue
if "enumeration" not in facetValues:
for facetElt in XmlUtil.schemaFacets(self, ("{http://www.w3.org/2001/XMLSchema}enumeration",)):
facetValues.setdefault("enumeration",set()).add(facetElt.get("value"))
typeDerivedFrom = self.typeDerivedFrom
if typeDerivedFrom is not None:
typeDerivedFrom.constrainingFacets(facetValues)
return facetValues
def createUnit(self, multiplyBy, divideBy, afterSibling=None, beforeSibling=None):
xbrlElt = self.modelDocument.xmlRootElement
if afterSibling == AUTO_LOCATE_ELEMENT:
afterSibling = XmlUtil.lastChild(xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context", "unit"))
unitId = 'u-{0:02n}'.format( len(self.units) + 1)
newUnitElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId),
afterSibling=afterSibling, beforeSibling=beforeSibling)
if len(divideBy) == 0:
for multiply in multiplyBy:
XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply))
else:
divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide")
numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator")
denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator")
for multiply in multiplyBy:
XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply))
for divide in divideBy:
XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, divide))
self.modelDocument.unitDiscover(newUnitElt)
XmlValidate.validate(self, newUnitElt)
return newUnitElt
def sEqual(dts1,
elt1,
elt2,
equalMode=S_EQUAL,
excludeIDs=NO_IDs_EXCLUDED,
dts2=None,
ns2ns1Tbl=None):
if dts2 is None: dts2 = dts1
if elt1.localName != elt2.localName:
return False
if ns2ns1Tbl and elt2.namespaceURI in ns2ns1Tbl:
if elt1.namespaceURI != ns2ns1Tbl[elt2.namespaceURI]:
return False
elif elt1.namespaceURI != elt2.namespaceURI:
return False
if not hasattr(elt1, "xValid"):
XmlValidate.validate(dts1, elt1)
if not hasattr(elt2, "xValid"):
XmlValidate.validate(dts2, elt2)
children1 = childElements(elt1)
children2 = childElements(elt2)
if len(children1) != len(children2):
return False
if (not xEqual(
elt1,
elt2,
# must use stringValue for nested contents of mixed content
# ... this is now in xValue for mixed content
# VALIDATE_BY_STRING_VALUE if len(children1) and elt1.xValid == VALID else
equalMode) or attributeDict(dts1, elt1, (), equalMode, excludeIDs)
!= attributeDict(dts2, elt2,
(), equalMode, excludeIDs, ns2ns1Tbl)):
return False
excludeChildIDs = excludeIDs if excludeIDs != TOP_IDs_EXCLUDED else NO_IDs_EXCLUDED
for i in range(len(children1)):
if not sEqual(dts1, children1[i], children2[i], equalMode,
excludeChildIDs, dts2, ns2ns1Tbl):
return False
return True
def createFact(self, conceptQname, attributes=None, text=None, parent=None, afterSibling=None, beforeSibling=None):
"""Creates new fact, as in formula output instance creation, and validates into object model
:param conceptQname: QNames of concept
:type conceptQname: QName
:param attributes: Tuple of name, value, or tuples of name, value tuples (name,value) or ((name,value)[,(name,value...)]), where name is either QName or clark-notation name string
:param text: Text content of fact
:type text: str
:param parent: lxml element in instance to append as child of
:type parent: ModelObject
:param beforeSibling: lxml element in instance to insert new concept before
:type beforeSibling: ModelObject
:param afterSibling: lxml element in instance to insert new concept after
:type afterSibling: ModelObject
:returns: ModelFact -- New fact object
"""
if parent is None: parent = self.modelDocument.xmlRootElement
newFact = XmlUtil.addChild(parent, conceptQname, attributes=attributes, text=text,
afterSibling=afterSibling, beforeSibling=beforeSibling)
self.modelDocument.factDiscover(newFact, parentElement=parent)
XmlValidate.validate(self, newFact)
return newFact
def inlineXbrlDiscover(self, htmlElement):
if htmlElement.namespaceURI == XbrlConst.xhtml: # must validate xhtml
#load(self.modelXbrl, "http://www.w3.org/2002/08/xhtml/xhtml1-strict.xsd")
XmlValidate.xhtmlValidate(self.modelXbrl,
htmlElement) # fails on prefixed content
for inlineElement in htmlElement.iterdescendants(
tag="{http://www.xbrl.org/2008/inlineXBRL}references"):
self.schemaLinkbaseRefsDiscover(inlineElement)
XmlValidate.validate(self.modelXbrl,
inlineElement) # validate instance elements
for inlineElement in htmlElement.iterdescendants(
tag="{http://www.xbrl.org/2008/inlineXBRL}resources"):
self.instanceContentsDiscover(inlineElement)
XmlValidate.validate(self.modelXbrl,
inlineElement) # validate instance elements
tupleElements = []
tuplesByTupleID = {}
for modelInlineTuple in htmlElement.iterdescendants(
tag="{http://www.xbrl.org/2008/inlineXBRL}tuple"):
if isinstance(modelInlineTuple, ModelObject):
modelInlineTuple.unorderedTupleFacts = []
if modelInlineTuple.tupleID:
tuplesByTupleID[
modelInlineTuple.tupleID] = modelInlineTuple
tupleElements.append(modelInlineTuple)
# hook up tuples to their container
for tupleFact in tupleElements:
self.inlineXbrlLocateFactInTuple(tupleFact, tuplesByTupleID)
for tag in ("{http://www.xbrl.org/2008/inlineXBRL}nonNumeric",
"{http://www.xbrl.org/2008/inlineXBRL}nonFraction",
"{http://www.xbrl.org/2008/inlineXBRL}fraction"):
for modelInlineFact in htmlElement.iterdescendants(tag=tag):
if isinstance(modelInlineFact, ModelObject):
self.inlineXbrlLocateFactInTuple(modelInlineFact,
tuplesByTupleID)
# order tuple facts
for tupleFact in tupleElements:
tupleFact.modelTupleFacts = [
self.modelXbrl.modelObject(objectIndex)
for order, objectIndex in sorted(tupleFact.unorderedTupleFacts)
]
# validate particle structure of elements after transformations and established tuple structure
for rootModelFact in self.modelXbrl.facts:
XmlValidate.validate(self.modelXbrl, rootModelFact, ixFacts=True)
def inlineXbrlDiscover(self, htmlElement):
if htmlElement.namespaceURI == XbrlConst.xhtml: # must validate xhtml
#load(self.modelXbrl, "http://www.w3.org/2002/08/xhtml/xhtml1-strict.xsd")
XmlValidate.xhtmlValidate(self.modelXbrl, htmlElement) # fails on prefixed content
for inlineElement in htmlElement.iterdescendants(tag="{http://www.xbrl.org/2008/inlineXBRL}references"):
self.schemaLinkbaseRefsDiscover(inlineElement)
XmlValidate.validate(self.modelXbrl, inlineElement) # validate instance elements
for inlineElement in htmlElement.iterdescendants(tag="{http://www.xbrl.org/2008/inlineXBRL}resources"):
self.instanceContentsDiscover(inlineElement)
XmlValidate.validate(self.modelXbrl, inlineElement) # validate instance elements
tupleElements = []
tuplesByTupleID = {}
for modelInlineTuple in htmlElement.iterdescendants(tag="{http://www.xbrl.org/2008/inlineXBRL}tuple"):
if isinstance(modelInlineTuple,ModelObject):
modelInlineTuple.unorderedTupleFacts = []
if modelInlineTuple.tupleID:
tuplesByTupleID[modelInlineTuple.tupleID] = modelInlineTuple
tupleElements.append(modelInlineTuple)
# hook up tuples to their container
for tupleFact in tupleElements:
self.inlineXbrlLocateFactInTuple(tupleFact, tuplesByTupleID)
for tag in ("{http://www.xbrl.org/2008/inlineXBRL}nonNumeric", "{http://www.xbrl.org/2008/inlineXBRL}nonFraction", "{http://www.xbrl.org/2008/inlineXBRL}fraction"):
for modelInlineFact in htmlElement.iterdescendants(tag=tag):
if isinstance(modelInlineFact,ModelObject):
self.inlineXbrlLocateFactInTuple(modelInlineFact, tuplesByTupleID)
# order tuple facts
for tupleFact in tupleElements:
tupleFact.modelTupleFacts = [
self.modelXbrl.modelObject(objectIndex)
for order,objectIndex in sorted(tupleFact.unorderedTupleFacts)]
# validate particle structure of elements after transformations and established tuple structure
for rootModelFact in self.modelXbrl.facts:
XmlValidate.validate(self.modelXbrl, rootModelFact, ixFacts=True)
def backgroundSaveInstance(self, newFilename=None):
cntlr = self.modelXbrl.modelManager.cntlr
if newFilename:
self.modelXbrl.modelManager.showStatus(
_("creating new instance {0}").format(
os.path.basename(newFilename)))
self.modelXbrl.modelManager.cntlr.waitForUiThreadQueue(
) # force status update
self.modelXbrl.createInstance(
newFilename
) # creates an instance as this modelXbrl's entrypoing
instance = self.modelXbrl
cntlr.showStatus(
_("Saving {0}").format(instance.modelDocument.basename))
cntlr.waitForUiThreadQueue() # force status update
newCntx = ModelXbrl.AUTO_LOCATE_ELEMENT
newUnit = ModelXbrl.AUTO_LOCATE_ELEMENT
# check user keyed changes
for bodyCell in self.gridBody.winfo_children():
if isinstance(bodyCell, gridCell) and bodyCell.isChanged:
value = bodyCell.value
objId = bodyCell.objectId
if objId:
if objId[0] == "f":
factPrototypeIndex = int(objId[1:])
factPrototype = self.factPrototypes[factPrototypeIndex]
concept = factPrototype.concept
entityIdentScheme = self.newFactItemOptions.entityIdentScheme
entityIdentValue = self.newFactItemOptions.entityIdentValue
periodType = factPrototype.concept.periodType
periodStart = self.newFactItemOptions.startDateDate if periodType == "duration" else None
periodEndInstant = self.newFactItemOptions.endDateDate
qnameDims = factPrototype.context.qnameDims
prevCntx = instance.matchContext(
entityIdentScheme, entityIdentValue, periodType,
periodStart, periodEndInstant, qnameDims, [], [])
if prevCntx is not None:
cntxId = prevCntx.id
else: # need new context
newCntx = instance.createContext(
entityIdentScheme,
entityIdentValue,
periodType,
periodStart,
periodEndInstant,
concept.qname,
qnameDims, [], [],
afterSibling=newCntx)
cntxId = newCntx.id
# new context
if concept.isNumeric:
if concept.isMonetary:
unitMeasure = qname(
XbrlConst.iso4217,
self.newFactItemOptions.monetaryUnit)
unitMeasure.prefix = "iso4217" # want to save with a recommended prefix
decimals = self.newFactItemOptions.monetaryDecimals
elif concept.isShares:
unitMeasure = XbrlConst.qnXbrliShares
decimals = self.newFactItemOptions.nonMonetaryDecimals
else:
unitMeasure = XbrlConst.qnXbrliPure
decimals = self.newFactItemOptions.nonMonetaryDecimals
prevUnit = instance.matchUnit([unitMeasure], [])
if prevUnit is not None:
unitId = prevUnit.id
else:
newUnit = instance.createUnit(
[unitMeasure], [], afterSibling=newUnit)
unitId = newUnit.id
attrs = [("contextRef", cntxId)]
if concept.isNumeric:
attrs.append(("unitRef", unitId))
attrs.append(("decimals", decimals))
value = Locale.atof(self.modelXbrl.locale, value,
str.strip)
newFact = instance.createFact(concept.qname,
attributes=attrs,
text=value)
bodyCell.objectId = newFact.objectId(
) # switch cell to now use fact ID
if self.factPrototypes[factPrototypeIndex] is not None:
self.factPrototypes[factPrototypeIndex].clear()
self.factPrototypes[
factPrototypeIndex] = None #dereference fact prototype
else: # instance fact, not prototype
fact = self.modelXbrl.modelObject(objId)
if fact.concept.isNumeric:
value = Locale.atof(self.modelXbrl.locale, value,
str.strip)
if fact.value != value:
fact.text = value
XmlValidate.validate(instance, fact)
bodyCell.isChanged = False # clear change flag
instance.saveInstance()
cntlr.showStatus(_("Saved {0}").format(
instance.modelDocument.basename),
clearAfter=3000)
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, *args, **kwargs):
# check if big instance and has header with an initial incomplete tree walk (just 2 elements
if not _streamingExtensionsCheck:
return None
# track whether modelXbrl has been validated by this streaming extension
modelXbrl._streamingExtensionValidated = False
def logSyntaxErrors(parsercontext):
for error in parsercontext.error_log:
modelXbrl.error("xmlSchema:syntax",
_("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"),
modelObject=modelXbrl, fileName=os.path.basename(filepath),
error=error.message, line=error.line, column=error.column, sourceAction="streaming")
#### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
#### note: iterparse wants a binary file, but file is text mode
_file, = modelXbrl.fileSource.file(filepath, binary=True)
startedAt = time.time()
modelXbrl.profileActivity()
''' this seems twice as slow as iterparse
class instInfoTarget():
def __init__(self, element_factory=None, parser=None):
self.newTree = True
self.streamingAspects = None
self.foundInstance = False
self.creationSoftwareComment = ''
self.currentEltTag = "(before xbrli:xbrl)"
self.numRootFacts = 0
def start(self, tag, attrib, nsmap=None):
if self.newTree:
if tag == "{http://www.xbrl.org/2003/instance}xbrl":
self.foundInstance = True
self.newTree = False
else: # break
raise NotInstanceDocumentException()
elif not tag.startswith("{http://www.xbrl.org/"):
self.numRootFacts += 1
if self.numRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
self.currentEltTag = tag
def end(self, tag):
pass
def data(self, data):
pass
def comment(self, text):
if not self.foundInstance: # accumulate comments before xbrli:xbrl
self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text
elif not self.creationSoftwareComment:
self.creationSoftwareComment = text # or first comment after xbrli:xbrl
def pi(self, target, data):
if target == "xbrl-streamable-instance":
if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl":
self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results
else:
modelXbrl.error("streamingExtensions:headerMisplaced",
_("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"),
modelObject=modelXbrl, target=target, element=self.currentEltTag)
def close(self):
if not self.creationSoftwareComment:
self.creationSoftwareComment = None
return True
instInfo = instInfoTarget()
infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo)
try:
etree.parse(_file, parser=infoParser, base_url=filepath)
except NotInstanceDocumentException:
pass
'''
foundErrors = False
foundInstance = False
streamingAspects = None
creationSoftwareComment = None
instInfoNumRootFacts = 0
numElts = 0
elt = None
instInfoContext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
try:
for event, elt in instInfoContext:
if event == "start":
if elt.getparent() is not None:
if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
if not foundInstance:
foundInstance = True
pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance")
if pi is None:
break
else:
streamingAspects = dict(pi.attrib.copy())
if creationSoftwareComment is None:
creationSoftwareComment = precedingComment(elt)
if not elt.tag.startswith("{http://www.xbrl.org/"):
instInfoNumRootFacts += 1
if instInfoNumRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
elif not foundInstance:
break
elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl":
creationSoftwareComment = precedingComment(elt)
if precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None:
modelXbrl.error("streamingExtensions:headerMisplaced",
_("Header is misplaced: %(error)s, must follow xbrli:xbrl element"),
modelObject=elt)
elif event == "end":
elt.clear()
numElts += 1
if numElts % 1000 == 0 and elt.getparent() is not None:
while elt.getprevious() is not None and elt.getparent() is not None:
del elt.getparent()[0]
except etree.XMLSyntaxError as err:
modelXbrl.error("xmlSchema:syntax",
_("Unrecoverable error: %(error)s"),
error=err)
_file.close()
return err
_file.seek(0,io.SEEK_SET) # allow reparsing
if not foundInstance or streamingAspects is None:
del elt
_file.close()
return None
modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
startedAt = time.time()
try:
version = Decimal(streamingAspects.get("version"))
if int(version) != 1:
modelXbrl.error("streamingExtensions:unsupportedVersion",
_("Streaming version %(version)s, major version number must be 1"),
modelObject=elt, version=version)
foundErrors = True
except (InvalidOperation, OverflowError):
modelXbrl.error("streamingExtensions:versionError",
_("Version %(version)s, number must be 1.n"),
modelObject=elt, version=streamingAspects.get("version", "(none)"))
foundErrors = True
for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
try:
bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
raise InvalidOperation
elif bufAspect == "contextBuffer":
contextBufferLimit = bufLimit
elif bufAspect == "unitBuffer":
unitBufferLimit = bufLimit
elif bufAspect == "footnoteBuffer":
footnoteBufferLimit = bufLimit
except InvalidOperation:
modelXbrl.error("streamingExtensions:valueError",
_("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"),
modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect))
foundErrors = True
if _streamingExtensionsValidate:
incompatibleValidations = []
_validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
_disclosureSystem = modelXbrl.modelManager.disclosureSystem
if _validateDisclosureSystem and _disclosureSystem.validationType == "EFM":
incompatibleValidations.append("EFM")
if _validateDisclosureSystem and _disclosureSystem.validationType == "GFM":
incompatibleValidations.append("GFM")
if _validateDisclosureSystem and _disclosureSystem.validationType == "HMRC":
incompatibleValidations.append("HMRC")
if modelXbrl.modelManager.validateCalcLB:
incompatibleValidations.append("calculation LB")
if incompatibleValidations:
modelXbrl.error("streamingExtensions:incompatibleValidation",
_("Streaming instance validation does not support %(incompatibleValidations)s validation"),
modelObject=modelXbrl, incompatibleValidations=', '.join(incompatibleValidations))
foundErrors = True
if instInfoContext.error_log:
foundErrors = True
logSyntaxErrors(instInfoContext)
del instInfoContext # dereference
for pluginMethod in pluginClassMethods("Streaming.BlockStreaming"):
_blockingPluginName = pluginMethod(modelXbrl)
if _blockingPluginName: # name of blocking plugin is returned
modelXbrl.error("streamingExtensions:incompatiblePlugIn",
_("Streaming instance not supported by plugin %(blockingPlugin)s"),
modelObject=modelXbrl, blockingPlugin=_blockingPluginName)
foundErrors = True
if foundErrors:
_file.close()
return None
_encoding = XmlUtil.encoding(_file.read(512))
_file.seek(0,io.SEEK_SET) # allow reparsing
if _streamingExtensionsValidate:
validator = Validate(modelXbrl)
instValidator = validator.instValidator
contextBuffer = []
contextsToDrop = []
unitBuffer = []
unitsToDrop = []
footnoteBuffer = []
footnoteLinksToDrop = []
_streamingFactsPlugin = any(True for pluginMethod in pluginClassMethods("Streaming.Facts"))
_streamingValidateFactsPlugin = (_streamingExtensionsValidate and
any(True for pluginMethod in pluginClassMethods("Streaming.ValidateFacts")))
''' this is very much slower than iterparse
class modelLoaderTarget():
def __init__(self, element_factory=None, parser=None):
self.newTree = True
self.currentMdlObj = None
self.beforeInstanceStream = True
self.beforeStartStreamingPlugin = True
self.numRootFacts = 1
modelXbrl.makeelementParentModelObject = None
modelXbrl.isStreamingMode = True
self.factsCheckVersion = None
self.factsCheckMd5s = Md5Sum()
def start(self, tag, attrib, nsmap=None):
modelXbrl.makeelementParentModelObject = self.currentMdlObj # pass parent to makeelement for ModelObjectFactory
mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
mdlObj.sourceline = 1
if self.newTree:
self.newTree = False
self.currentMdlObj = mdlObj
modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
modelXbrl.modelDocument = modelDocument # needed for incremental validation
mdlObj.init(modelDocument)
modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement
modelDocument.parserLookupName = _parserLookupName
modelDocument.parserLookupClass = _parserLookupClass
modelDocument.xmlRootElement = mdlObj
modelDocument.schemaLocationElements.add(mdlObj)
modelDocument.documentEncoding = _encoding
modelDocument._creationSoftwareComment = creationSoftwareComment
modelXbrl.info("streamingExtensions:streaming",
_("Stream processing this instance."),
modelObject = modelDocument)
else:
self.currentMdlObj.append(mdlObj)
self.currentMdlObj = mdlObj
mdlObj._init()
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if (self.beforeInstanceStream and (
(ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
(ns == XbrlConst.xbrli and ln in ("context", "unit")) or
(ns not in (XbrlConst.link, XbrlConst.xbrli)))):
self.beforeInstanceStream = False
if _streamingExtensionsValidate:
instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters(modelXbrl.prefixedNamespaces))
else: # need default dimensions
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
elif not self.beforeInstanceStream and self.beforeStartStreamingPlugin:
for pluginMethod in pluginClassMethods("Streaming.Start"):
pluginMethod(modelXbrl)
self.beforeStartStreamingPlugin = False
return mdlObj
def end(self, tag):
modelDocument = modelXbrl.modelDocument
mdlObj = self.currentMdlObj
parentMdlObj = mdlObj.getparent()
self.currentMdlObj = parentMdlObj
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
else:
if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
if _streamingValidateFactsPlugin:
contextsToDrop.append(cntx)
else:
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
#>>XmlValidate.validate(modelXbrl, mdlObj)
#>>modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj,)
instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
instValidator.checkContextsDimensions(contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
# drop before adding as dropped may have same id as added
unit = unitBuffer.pop(0)
if _streamingValidateFactsPlugin:
unitsToDrop.append(unit)
else:
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
unit = None
#>>XmlValidate.validate(modelXbrl, mdlObj)
#>>modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkUnits( (mdlObj,) )
elif ln == "xbrl": # end of document
# check remaining batched facts if any
if _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
# finish any final batch of facts
if len(modelXbrl.facts) > 0:
factsToCheck = modelXbrl.facts.copy()
factsHaveBeenProcessed = True
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
if not pluginMethod(modelXbrl, factsToCheck):
factsHaveBeenProcessed = False
if factsHaveBeenProcessed:
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(fact)]
for cntx in contextsToDrop:
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
for unit in unitsToDrop:
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
for footnoteLink in footnoteLinksToDrop:
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
fact = cntx = unit = footnoteLink = None
del contextsToDrop[:]
del unitsToDrop[:]
del footnoteLinksToDrop[:]
del factsToCheck
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
for pluginMethod in pluginClassMethods("Streaming.Finish"):
pluginMethod(modelXbrl)
elif ns == XbrlConst.link:
if ln == "footnoteLink":
XmlValidate.validate(modelXbrl, mdlObj)
footnoteLinks = (mdlObj,)
modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
if _streamingValidateFactsPlugin:
footnoteLinksToDrop.append(footnoteLink)
else:
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elif ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(mdlObj)
elif not modelXbrl.skipDTS:
if ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
self.numRootFacts += 1
#>>XmlValidate.validate(modelXbrl, mdlObj)
#>>modelDocument.factDiscover(mdlObj, modelXbrl.facts)
if self.factsCheckVersion:
self.factCheckFact(mdlObj)
if _streamingExtensionsValidate or _streamingValidateFactsPlugin:
factsToCheck = (mdlObj,) # validate current fact by itself
if _streamingExtensionsValidate:
instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
instValidator.checkFactsDimensions(factsToCheck)
if _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
# use batches of 1000 facts
if len(modelXbrl.facts) > 1000:
factsToCheck = modelXbrl.facts.copy()
factsHaveBeenProcessed = True
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
if not pluginMethod(modelXbrl, factsToCheck):
factsHaveBeenProcessed = False
if factsHaveBeenProcessed:
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(fact)]
for cntx in contextsToDrop:
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
for unit in unitsToDrop:
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
for footnoteLink in footnoteLinksToDrop:
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
fact = cntx = unit = footnoteLink = None
del contextsToDrop[:]
del unitsToDrop[:]
del footnoteLinksToDrop[:]
del factsToCheck # dereference fact or batch of facts
else:
dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed
del parentMdlObj[parentMdlObj.index(mdlObj)]
if self.numRootFacts % 1000 == 0:
pass
#modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts,
# 100.0 * self.numRootFacts / instInfoNumRootFacts),
# minTimeToShow=20.0)
gc.collect()
sys.stdout.write ("\rAt fact {} of {} mem {}".format(self.numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed))
return mdlObj
def data(self, data):
self.currentMdlObj.text = data
def comment(self, text):
pass
def pi(self, target, data):
if target == "xbrl-facts-check":
_match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", data)
if _match:
_matchGroups = _match.groups()
if len(_matchGroups) == 2:
if _matchGroups[0] == "version":
self.factsCheckVersion = _matchGroups[1]
elif _matchGroups[0] == "sum-of-fact-md5s":
try:
expectedMd5 = Md5Sum(_matchGroups[1])
if self.factsCheckMd5s != expectedMd5:
modelXbrl.warning("streamingExtensions:xbrlFactsCheckWarning",
_("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"),
modelObject=modelXbrl, expectedMd5=expectedMd5, actualMd5Sum=self.factsCheckMd5s)
else:
modelXbrl.info("info",
_("Successful XBRL facts sum of md5s."),
modelObject=modelXbrl)
except ValueError:
modelXbrl.error("streamingExtensions:xbrlFactsCheckError",
_("Invalid sum-of-md5s %(sumOfMd5)s"),
modelObject=modelXbrl, sumOfMd5=_matchGroups[1])
def close(self):
del modelXbrl.makeelementParentModelObject
return None
def factCheckFact(self, fact):
self.factsCheckMd5s += fact.md5sum
for _tupleFact in fact.modelTupleFacts:
self.factCheckFact(_tupleFact)
_parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget())
etree.parse(_file, parser=_parser, base_url=filepath)
logSyntaxErrors(_parser)
'''
# replace modelLoaderTarget with iterparse (as it now supports CustomElementClassLookup)
streamingParserContext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
from arelle.ModelObjectFactory import setParserElementClassLookup
modelXbrl.isStreamingMode = True # must be set before setting element class lookup
(_parser, _parserLookupName, _parserLookupClass) = setParserElementClassLookup(streamingParserContext, modelXbrl)
foundInstance = False
beforeInstanceStream = beforeStartStreamingPlugin = True
numRootFacts = 0
factsCheckVersion = None
def factCheckFact(fact):
modelDocument._factsCheckMd5s += fact.md5sum
for _tupleFact in fact.modelTupleFacts:
factCheckFact(_tupleFact)
for event, mdlObj in streamingParserContext:
if event == "start":
if mdlObj.tag == "{http://www.xbrl.org/2003/instance}xbrl":
modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
modelXbrl.modelDocument = modelDocument # needed for incremental validation
mdlObj.init(modelDocument)
modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement
modelDocument.parserLookupName = _parserLookupName
modelDocument.parserLookupClass = _parserLookupClass
modelDocument.xmlRootElement = mdlObj
modelDocument.schemaLocationElements.add(mdlObj)
modelDocument.documentEncoding = _encoding
modelDocument._creationSoftwareComment = precedingComment(mdlObj)
modelDocument._factsCheckMd5s = Md5Sum()
modelXbrl.info("streamingExtensions:streaming",
_("Stream processing this instance."),
modelObject = modelDocument)
elif mdlObj.getparent() is not None:
mdlObj._init() # requires discovery as part of start elements
if mdlObj.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
if not foundInstance:
foundInstance = True
pi = precedingProcessingInstruction(mdlObj, "xbrl-facts-check")
if pi is not None:
factsCheckVersion = pi.attrib.get("version", None)
elif not foundInstance:
break
ns = mdlObj.qname.namespaceURI
ln = mdlObj.qname.localName
if beforeInstanceStream:
if ((ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
(ns == XbrlConst.xbrli and ln in ("context", "unit")) or
(ns not in (XbrlConst.link, XbrlConst.xbrli))):
beforeInstanceStream = False
if _streamingExtensionsValidate:
instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters(modelXbrl.prefixedNamespaces))
else: # need default dimensions
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
elif not beforeInstanceStream and beforeStartStreamingPlugin:
for pluginMethod in pluginClassMethods("Streaming.Start"):
pluginMethod(modelXbrl)
beforeStartStreamingPlugin = False
elif event == "end":
parentMdlObj = mdlObj.getparent()
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
else:
if len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
if _streamingFactsPlugin or _streamingValidateFactsPlugin:
contextsToDrop.append(cntx)
else:
dropContext(modelXbrl, cntx)
#>>del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj,)
instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
instValidator.checkContextsDimensions(contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if len(unitBuffer) >= unitBufferLimit:
# drop before additing as dropped may have same id as added
unit = unitBuffer.pop(0)
if _streamingFactsPlugin or _streamingValidateFactsPlugin:
unitsToDrop.append(unit)
else:
dropUnit(modelXbrl, unit)
#>>del parentMdlObj[parentMdlObj.index(unit)]
unit = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkUnits( (mdlObj,) )
elif ln == "xbrl": # end of document
# check remaining batched facts if any
if _streamingFactsPlugin or _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
# finish any final batch of facts
if len(modelXbrl.facts) > 0:
factsToCheck = modelXbrl.facts.copy()
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
if _streamingValidateFactsPlugin:
for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
pluginMethod(instValidator, factsToCheck)
if _streamingFactsPlugin:
for pluginMethod in pluginClassMethods("Streaming.Facts"):
pluginMethod(modelXbrl, factsToCheck)
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
#>>del parentMdlObj[parentMdlObj.index(fact)]
for cntx in contextsToDrop:
dropContext(modelXbrl, cntx)
#>>del parentMdlObj[parentMdlObj.index(cntx)]
for unit in unitsToDrop:
dropUnit(modelXbrl, unit)
#>>del parentMdlObj[parentMdlObj.index(unit)]
for footnoteLink in footnoteLinksToDrop:
dropFootnoteLink(modelXbrl, footnoteLink)
#>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
fact = cntx = unit = footnoteLink = None
del contextsToDrop[:]
del unitsToDrop[:]
del footnoteLinksToDrop[:]
del factsToCheck
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
pi = childProcessingInstruction(mdlObj, "xbrl-facts-check", reversed=True)
if pi is not None: # attrib is in .text, not attrib, no idea why!!!
_match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text)
if _match:
_matchGroups = _match.groups()
if len(_matchGroups) == 2:
if _matchGroups[0] == "sum-of-fact-md5s":
try:
expectedMd5 = Md5Sum(_matchGroups[1])
if modelDocument._factsCheckMd5s != expectedMd5:
modelXbrl.warning("streamingExtensions:xbrlFactsCheckWarning",
_("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"),
modelObject=modelXbrl, expectedMd5=expectedMd5, actualMd5Sum=modelDocument._factsCheckMd5s)
else:
modelXbrl.info("info",
_("Successful XBRL facts sum of md5s."),
modelObject=modelXbrl)
except ValueError:
modelXbrl.error("streamingExtensions:xbrlFactsCheckError",
_("Invalid sum-of-md5s %(sumOfMd5)s"),
modelObject=modelXbrl, sumOfMd5=_matchGroups[1])
if _streamingValidateFactsPlugin:
for pluginMethod in pluginClassMethods("Streaming.ValidateFinish"):
pluginMethod(instValidator)
if _streamingFactsPlugin:
for pluginMethod in pluginClassMethods("Streaming.Finish"):
pluginMethod(modelXbrl)
elif ns == XbrlConst.link:
if ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(mdlObj, urlRewritePluginClass="ModelDocument.InstanceSchemaRefRewriter")
elif ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
elif ln == "footnoteLink":
XmlValidate.validate(modelXbrl, mdlObj)
footnoteLinks = (mdlObj,)
modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
if _streamingValidateFactsPlugin:
footnoteLinksToDrop.append(footnoteLink)
else:
dropFootnoteLink(modelXbrl, footnoteLink)
#>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl and isinstance(mdlObj, ModelFact):
numRootFacts += 1
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.factDiscover(mdlObj, modelXbrl.facts)
if factsCheckVersion:
factCheckFact(mdlObj)
if _streamingExtensionsValidate or _streamingFactsPlugin or _streamingValidateFactsPlugin:
factsToCheck = (mdlObj,) # validate current fact by itself
if _streamingExtensionsValidate:
instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
instValidator.checkFactsDimensions(factsToCheck)
if _streamingFactsPlugin or _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
# use batches of 1000 facts
if len(modelXbrl.facts) > 1000:
factsToCheck = modelXbrl.facts.copy()
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
if _streamingValidateFactsPlugin:
for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
pluginMethod(instValidator, factsToCheck)
if _streamingFactsPlugin:
for pluginMethod in pluginClassMethods("Streaming.Facts"):
pluginMethod(modelXbrl, factsToCheck)
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
#>>del parentMdlObj[parentMdlObj.index(fact)]
for cntx in contextsToDrop:
dropContext(modelXbrl, cntx)
#>>del parentMdlObj[parentMdlObj.index(cntx)]
for unit in unitsToDrop:
dropUnit(modelXbrl, unit)
#>>del parentMdlObj[parentMdlObj.index(unit)]
for footnoteLink in footnoteLinksToDrop:
dropFootnoteLink(modelXbrl, footnoteLink)
#>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
fact = cntx = unit = footnoteLink = None
del contextsToDrop[:]
del unitsToDrop[:]
del footnoteLinksToDrop[:]
del factsToCheck # dereference fact or batch of facts
else:
dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed
#>>del parentMdlObj[parentMdlObj.index(mdlObj)]
if numRootFacts % 1000 == 0:
pass
#modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts,
# 100.0 * self.numRootFacts / instInfoNumRootFacts),
# minTimeToShow=20.0)
#gc.collect()
#sys.stdout.write ("\rAt fact {} of {} mem {}".format(numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed))
if mdlObj is not None:
mdlObj.clear()
del _parser, _parserLookupName, _parserLookupClass
if _streamingExtensionsValidate and validator is not None:
_file.close()
del instValidator
validator.close()
# track that modelXbrl has been validated by this streaming extension
modelXbrl._streamingExtensionValidated = True
modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
return modelXbrl.modelDocument
def vEqual(elt1, elt2):
if not hasattr(elt1, "xValid"):
XmlValidate.validate(elt1.modelXbrl, elt1)
if not hasattr(elt2, "xValid"):
XmlValidate.validate(elt2.modelXbrl, elt2)
return elt1.sValue == elt2.sValue
def end(self, tag):
modelDocument = modelXbrl.modelDocument
mdlObj = self.currentMdlObj
parentMdlObj = mdlObj.getparent()
self.currentMdlObj = parentMdlObj
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
else:
if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj,)
instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
instValidator.checkContextsDimensions(contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
# drop before additing as dropped may have same id as added
unit = unitBuffer.pop(0)
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
unit = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkUnits( (mdlObj,) )
elif ln == "xbrl": # end of document
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
for pluginMethod in pluginClassMethods("Streaming.Finish"):
pluginMethod(modelXbrl)
elif ns == XbrlConst.link:
if ln == "footnoteLink":
XmlValidate.validate(modelXbrl, mdlObj)
footnoteLinks = (mdlObj,)
modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elif ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(mdlObj)
elif not modelXbrl.skipDTS:
if ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
self.numRootFacts += 1
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.factDiscover(mdlObj, modelXbrl.facts)
if _streamingExtensionsValidate or _streamingValidateFactsPlugin:
factsToCheck = (mdlObj,) # validate current fact by itself
if _streamingExtensionsValidate:
instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
instValidator.checkFactsDimensions(factsToCheck)
if _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
factsToCheck = modelXbrl.facts.copy()
factsHaveBeenProcessed = True
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
if not pluginMethod(modelXbrl, factsToCheck):
factsHaveBeenProcessed = False
if factsHaveBeenProcessed:
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(fact)]
else:
dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed
del parentMdlObj[parentMdlObj.index(mdlObj)]
del factsToCheck # dereference fact or batch of facts
if self.numRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts,
100.0 * self.numRootFacts / instInfoNumRootFacts),
minTimeToShow=20.0)
return mdlObj
def end(self, tag):
modelDocument = modelXbrl.modelDocument
mdlObj = self.currentMdlObj
parentMdlObj = mdlObj.getparent()
self.currentMdlObj = parentMdlObj
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
else:
if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj,)
instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
instValidator.checkContextsDimensions(contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
# drop before additing as dropped may have same id as added
unit = unitBuffer.pop(0)
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
unit = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkUnits( (mdlObj,) )
elif ln == "xbrl": # end of document
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
elif ns == XbrlConst.link:
if ln == "footnoteLink":
XmlValidate.validate(modelXbrl, mdlObj)
footnoteLinks = (mdlObj,)
modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elif ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(mdlObj)
elif not modelXbrl.skipDTS:
if ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
self.numRootFacts += 1
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.factDiscover(mdlObj, modelXbrl.facts)
if _streamingExtensionsValidate:
factsToCheck = (mdlObj,)
instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
instValidator.checkFactsDimensions(factsToCheck)
del factsToCheck
dropFact(modelXbrl, mdlObj, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(mdlObj)]
if self.numRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts,
100.0 * self.numRootFacts / instInfoNumRootFacts),
minTimeToShow=20.0)
return mdlObj
def vEqual(elt1, elt2):
if not hasattr(elt1,"xValid"):
XmlValidate.validate(elt1.modelXbrl, elt1)
if not hasattr(elt2,"xValid"):
XmlValidate.validate(elt2.modelXbrl, elt2)
return elt1.sValue == elt2.sValue
def end(self, tag):
modelDocument = modelXbrl.modelDocument
mdlObj = self.currentMdlObj
parentMdlObj = mdlObj.getparent()
self.currentMdlObj = parentMdlObj
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
else:
if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj,)
instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
instValidator.checkContextsDimensions(contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
# drop before additing as dropped may have same id as added
unit = unitBuffer.pop(0)
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
unit = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkUnits( (mdlObj,) )
elif ln == "xbrl": # end of document
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
elif ns == XbrlConst.link:
if ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(mdlObj)
elif ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
elif ln == "footnoteLink":
XmlValidate.validate(modelXbrl, mdlObj)
footnoteLinks = (mdlObj,)
modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
self.numRootFacts += 1
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.factDiscover(mdlObj, modelXbrl.facts)
if _streamingExtensionsValidate:
factsToCheck = (mdlObj,)
instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
instValidator.checkFactsDimensions(factsToCheck)
del factsToCheck
dropFact(modelXbrl, mdlObj, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(mdlObj)]
if self.numRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts,
100.0 * self.numRootFacts / instInfoNumRootFacts),
minTimeToShow=20.0)
return mdlObj
def createContext(self,
entityIdentScheme,
entityIdentValue,
periodType,
periodStart,
periodEndInstant,
priItem,
dims,
segOCCs,
scenOCCs,
afterSibling=None,
beforeSibling=None):
xbrlElt = self.modelDocument.xmlRootElement
if afterSibling == AUTO_LOCATE_ELEMENT:
afterSibling = XmlUtil.lastChild(
xbrlElt, XbrlConst.xbrli,
("schemaLocation", "roleType", "arcroleType", "context"))
cntxId = 'c-{0:02n}'.format(len(self.contexts) + 1)
newCntxElt = XmlUtil.addChild(xbrlElt,
XbrlConst.xbrli,
"context",
attributes=("id", cntxId),
afterSibling=afterSibling,
beforeSibling=beforeSibling)
entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity")
XmlUtil.addChild(entityElt,
XbrlConst.xbrli,
"identifier",
attributes=("scheme", entityIdentScheme),
text=entityIdentValue)
periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period")
if periodType == "forever":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever")
elif periodType == "instant":
XmlUtil.addChild(periodElt,
XbrlConst.xbrli,
"instant",
text=XmlUtil.dateunionValue(periodEndInstant,
subtractOneDay=True))
elif periodType == "duration":
XmlUtil.addChild(periodElt,
XbrlConst.xbrli,
"startDate",
text=XmlUtil.dateunionValue(periodStart))
XmlUtil.addChild(periodElt,
XbrlConst.xbrli,
"endDate",
text=XmlUtil.dateunionValue(periodEndInstant,
subtractOneDay=True))
segmentElt = None
scenarioElt = None
from arelle.ModelInstanceObject import ModelDimensionValue
if dims: # requires primary item to determin ambiguous concepts
''' in theory we have to check full set of dimensions for validity in source or any other
context element, but for shortcut will see if each dimension is already reported in an
unambiguous valid contextElement
'''
from arelle.PrototypeInstanceObject import FactPrototype, ContextPrototype, DimValuePrototype
fp = FactPrototype(self, priItem, dims.items())
# force trying a valid prototype's context Elements
if not isFactDimensionallyValid(
self, fp, setPrototypeContextElements=True):
self.info(
"arelleLinfo",
_("Create context for %(priItem)s, cannot determine valid context elements, no suitable hypercubes"
),
modelObject=self,
priItem=priItem)
fpDims = fp.context.qnameDims
for dimQname in sorted(fpDims.keys()):
dimValue = fpDims[dimQname]
if isinstance(dimValue, DimValuePrototype):
dimMemberQname = dimValue.memberQname # None if typed dimension
contextEltName = dimValue.contextElement
else: # qname for explicit or node for typed
dimMemberQname = None
contextEltName = None
if contextEltName == "segment":
if segmentElt is None:
segmentElt = XmlUtil.addChild(entityElt,
XbrlConst.xbrli,
"segment")
contextElt = segmentElt
elif contextEltName == "scenario":
if scenarioElt is None:
scenarioElt = XmlUtil.addChild(newCntxElt,
XbrlConst.xbrli,
"scenario")
contextElt = scenarioElt
else:
self.info(
"arelleLinfo",
_("Create context, %(dimension)s, cannot determine context element, either no all relationship or validation issue"
),
modelObject=self,
dimension=dimQname),
continue
dimConcept = self.qnameConcepts[dimQname]
dimAttr = ("dimension",
XmlUtil.addQnameValue(xbrlElt, dimConcept.qname))
if dimConcept.isTypedDimension:
dimElt = XmlUtil.addChild(contextElt,
XbrlConst.xbrldi,
"xbrldi:typedMember",
attributes=dimAttr)
if isinstance(dimValue,
(ModelDimensionValue,
DimValuePrototype)) and dimValue.isTyped:
XmlUtil.copyNodes(dimElt, dimValue.typedMember)
elif dimMemberQname:
dimElt = XmlUtil.addChild(contextElt,
XbrlConst.xbrldi,
"xbrldi:explicitMember",
attributes=dimAttr,
text=XmlUtil.addQnameValue(
xbrlElt, dimMemberQname))
if segOCCs:
if segmentElt is None:
segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli,
"segment")
XmlUtil.copyNodes(segmentElt, segOCCs)
if scenOCCs:
if scenarioElt is None:
scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli,
"scenario")
XmlUtil.copyNodes(scenarioElt, scenOCCs)
self.modelDocument.contextDiscover(newCntxElt)
XmlValidate.validate(self, newCntxElt)
return newCntxElt
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, *args, **kwargs):
# check if big instance and has header with an initial incomplete tree walk (just 2 elements
if not _streamingExtensionsCheck:
return None
# track whether modelXbrl has been validated by this streaming extension
modelXbrl._streamingExtensionValidated = False
def logSyntaxErrors(parsercontext):
for error in parsercontext.error_log:
modelXbrl.error(
"xmlSchema:syntax",
_("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"
),
modelObject=modelXbrl,
fileName=os.path.basename(filepath),
error=error.message,
line=error.line,
column=error.column,
sourceAction="streaming")
#### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
#### note: iterparse wants a binary file, but file is text mode
_file, = modelXbrl.fileSource.file(filepath, binary=True)
startedAt = time.time()
modelXbrl.profileActivity()
''' this seems twice as slow as iterparse
class instInfoTarget():
def __init__(self, element_factory=None, parser=None):
self.newTree = True
self.streamingAspects = None
self.foundInstance = False
self.creationSoftwareComment = ''
self.currentEltTag = "(before xbrli:xbrl)"
self.numRootFacts = 0
def start(self, tag, attrib, nsmap=None):
if self.newTree:
if tag == "{http://www.xbrl.org/2003/instance}xbrl":
self.foundInstance = True
self.newTree = False
else: # break
raise NotInstanceDocumentException()
elif not tag.startswith("{http://www.xbrl.org/"):
self.numRootFacts += 1
if self.numRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
self.currentEltTag = tag
def end(self, tag):
pass
def data(self, data):
pass
def comment(self, text):
if not self.foundInstance: # accumulate comments before xbrli:xbrl
self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text
elif not self.creationSoftwareComment:
self.creationSoftwareComment = text # or first comment after xbrli:xbrl
def pi(self, target, data):
if target == "xbrl-streamable-instance":
if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl":
self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results
else:
modelXbrl.error("streamingExtensions:headerMisplaced",
_("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"),
modelObject=modelXbrl, target=target, element=self.currentEltTag)
def close(self):
if not self.creationSoftwareComment:
self.creationSoftwareComment = None
return True
instInfo = instInfoTarget()
infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo)
try:
etree.parse(_file, parser=infoParser, base_url=filepath)
except NotInstanceDocumentException:
pass
'''
foundErrors = False
foundInstance = False
streamingAspects = None
creationSoftwareComment = None
instInfoNumRootFacts = 0
numElts = 0
elt = None
instInfoContext = etree.iterparse(_file,
events=("start", "end"),
huge_tree=True)
try:
for event, elt in instInfoContext:
if event == "start":
if elt.getparent() is not None:
if elt.getparent(
).tag == "{http://www.xbrl.org/2003/instance}xbrl":
if not foundInstance:
foundInstance = True
pi = precedingProcessingInstruction(
elt, "xbrl-streamable-instance")
if pi is None:
break
else:
streamingAspects = dict(pi.attrib.copy())
if creationSoftwareComment is None:
creationSoftwareComment = precedingComment(
elt)
if not elt.tag.startswith("{http://www.xbrl.org/"):
instInfoNumRootFacts += 1
if instInfoNumRootFacts % 1000 == 0:
modelXbrl.profileActivity(
"... streaming tree check",
minTimeToShow=20.0)
elif not foundInstance:
break
elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl":
creationSoftwareComment = precedingComment(elt)
if precedingProcessingInstruction(
elt, "xbrl-streamable-instance") is not None:
modelXbrl.error(
"streamingExtensions:headerMisplaced",
_("Header is misplaced: %(error)s, must follow xbrli:xbrl element"
),
modelObject=elt)
elif event == "end":
elt.clear()
numElts += 1
if numElts % 1000 == 0 and elt.getparent() is not None:
while elt.getprevious() is not None and elt.getparent(
) is not None:
del elt.getparent()[0]
except etree.XMLSyntaxError as err:
modelXbrl.error("xmlSchema:syntax",
_("Unrecoverable error: %(error)s"),
error=err)
_file.close()
return err
_file.seek(0, io.SEEK_SET) # allow reparsing
if not foundInstance or streamingAspects is None:
del elt
_file.close()
return None
modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
startedAt = time.time()
try:
version = Decimal(streamingAspects.get("version"))
if int(version) != 1:
modelXbrl.error(
"streamingExtensions:unsupportedVersion",
_("Streaming version %(version)s, major version number must be 1"
),
modelObject=elt,
version=version)
foundErrors = True
except (InvalidOperation, OverflowError):
modelXbrl.error("streamingExtensions:versionError",
_("Version %(version)s, number must be 1.n"),
modelObject=elt,
version=streamingAspects.get("version", "(none)"))
foundErrors = True
for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
try:
bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
raise InvalidOperation
elif bufAspect == "contextBuffer":
contextBufferLimit = bufLimit
elif bufAspect == "unitBuffer":
unitBufferLimit = bufLimit
elif bufAspect == "footnoteBuffer":
footnoteBufferLimit = bufLimit
except InvalidOperation:
modelXbrl.error(
"streamingExtensions:valueError",
_("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"
),
modelObject=elt,
attrib=bufAspect,
value=streamingAspects.get(bufAspect))
foundErrors = True
if _streamingExtensionsValidate:
incompatibleValidations = []
_validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
_disclosureSystem = modelXbrl.modelManager.disclosureSystem
if _validateDisclosureSystem and _disclosureSystem.validationType == "EFM":
incompatibleValidations.append("EFM")
if _validateDisclosureSystem and _disclosureSystem.validationType == "GFM":
incompatibleValidations.append("GFM")
if _validateDisclosureSystem and _disclosureSystem.validationType == "HMRC":
incompatibleValidations.append("HMRC")
if modelXbrl.modelManager.validateCalcLB:
incompatibleValidations.append("calculation LB")
if incompatibleValidations:
modelXbrl.error(
"streamingExtensions:incompatibleValidation",
_("Streaming instance validation does not support %(incompatibleValidations)s validation"
),
modelObject=modelXbrl,
incompatibleValidations=', '.join(incompatibleValidations))
foundErrors = True
if instInfoContext.error_log:
foundErrors = True
logSyntaxErrors(instInfoContext)
del instInfoContext # dereference
for pluginMethod in pluginClassMethods("Streaming.BlockStreaming"):
_blockingPluginName = pluginMethod(modelXbrl)
if _blockingPluginName: # name of blocking plugin is returned
modelXbrl.error(
"streamingExtensions:incompatiblePlugIn",
_("Streaming instance not supported by plugin %(blockingPlugin)s"
),
modelObject=modelXbrl,
blockingPlugin=_blockingPluginName)
foundErrors = True
if foundErrors:
_file.close()
return None
_encoding = XmlUtil.encoding(_file.read(512))
_file.seek(0, io.SEEK_SET) # allow reparsing
if _streamingExtensionsValidate:
validator = Validate(modelXbrl)
instValidator = validator.instValidator
contextBuffer = []
contextsToDrop = []
unitBuffer = []
unitsToDrop = []
footnoteBuffer = []
footnoteLinksToDrop = []
_streamingFactsPlugin = any(
True for pluginMethod in pluginClassMethods("Streaming.Facts"))
_streamingValidateFactsPlugin = (_streamingExtensionsValidate and any(
True
for pluginMethod in pluginClassMethods("Streaming.ValidateFacts")))
''' this is very much slower than iterparse
class modelLoaderTarget():
def __init__(self, element_factory=None, parser=None):
self.newTree = True
self.currentMdlObj = None
self.beforeInstanceStream = True
self.beforeStartStreamingPlugin = True
self.numRootFacts = 1
modelXbrl.makeelementParentModelObject = None
modelXbrl.isStreamingMode = True
self.factsCheckVersion = None
self.factsCheckMd5s = Md5Sum()
def start(self, tag, attrib, nsmap=None):
modelXbrl.makeelementParentModelObject = self.currentMdlObj # pass parent to makeelement for ModelObjectFactory
mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
mdlObj.sourceline = 1
if self.newTree:
self.newTree = False
self.currentMdlObj = mdlObj
modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
modelXbrl.modelDocument = modelDocument # needed for incremental validation
mdlObj.init(modelDocument)
modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement
modelDocument.parserLookupName = _parserLookupName
modelDocument.parserLookupClass = _parserLookupClass
modelDocument.xmlRootElement = mdlObj
modelDocument.schemaLocationElements.add(mdlObj)
modelDocument.documentEncoding = _encoding
modelDocument._creationSoftwareComment = creationSoftwareComment
modelXbrl.info("streamingExtensions:streaming",
_("Stream processing this instance."),
modelObject = modelDocument)
else:
self.currentMdlObj.append(mdlObj)
self.currentMdlObj = mdlObj
mdlObj._init()
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if (self.beforeInstanceStream and (
(ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
(ns == XbrlConst.xbrli and ln in ("context", "unit")) or
(ns not in (XbrlConst.link, XbrlConst.xbrli)))):
self.beforeInstanceStream = False
if _streamingExtensionsValidate:
instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters(modelXbrl.prefixedNamespaces))
else: # need default dimensions
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
elif not self.beforeInstanceStream and self.beforeStartStreamingPlugin:
for pluginMethod in pluginClassMethods("Streaming.Start"):
pluginMethod(modelXbrl)
self.beforeStartStreamingPlugin = False
return mdlObj
def end(self, tag):
modelDocument = modelXbrl.modelDocument
mdlObj = self.currentMdlObj
parentMdlObj = mdlObj.getparent()
self.currentMdlObj = parentMdlObj
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
else:
if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
if _streamingValidateFactsPlugin:
contextsToDrop.append(cntx)
else:
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
#>>XmlValidate.validate(modelXbrl, mdlObj)
#>>modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj,)
instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
instValidator.checkContextsDimensions(contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
# drop before adding as dropped may have same id as added
unit = unitBuffer.pop(0)
if _streamingValidateFactsPlugin:
unitsToDrop.append(unit)
else:
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
unit = None
#>>XmlValidate.validate(modelXbrl, mdlObj)
#>>modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkUnits( (mdlObj,) )
elif ln == "xbrl": # end of document
# check remaining batched facts if any
if _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
# finish any final batch of facts
if len(modelXbrl.facts) > 0:
factsToCheck = modelXbrl.facts.copy()
factsHaveBeenProcessed = True
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
if not pluginMethod(modelXbrl, factsToCheck):
factsHaveBeenProcessed = False
if factsHaveBeenProcessed:
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(fact)]
for cntx in contextsToDrop:
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
for unit in unitsToDrop:
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
for footnoteLink in footnoteLinksToDrop:
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
fact = cntx = unit = footnoteLink = None
del contextsToDrop[:]
del unitsToDrop[:]
del footnoteLinksToDrop[:]
del factsToCheck
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
for pluginMethod in pluginClassMethods("Streaming.Finish"):
pluginMethod(modelXbrl)
elif ns == XbrlConst.link:
if ln == "footnoteLink":
XmlValidate.validate(modelXbrl, mdlObj)
footnoteLinks = (mdlObj,)
modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
if _streamingValidateFactsPlugin:
footnoteLinksToDrop.append(footnoteLink)
else:
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elif ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(mdlObj)
elif not modelXbrl.skipDTS:
if ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
self.numRootFacts += 1
#>>XmlValidate.validate(modelXbrl, mdlObj)
#>>modelDocument.factDiscover(mdlObj, modelXbrl.facts)
if self.factsCheckVersion:
self.factCheckFact(mdlObj)
if _streamingExtensionsValidate or _streamingValidateFactsPlugin:
factsToCheck = (mdlObj,) # validate current fact by itself
if _streamingExtensionsValidate:
instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
instValidator.checkFactsDimensions(factsToCheck)
if _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
# use batches of 1000 facts
if len(modelXbrl.facts) > 1000:
factsToCheck = modelXbrl.facts.copy()
factsHaveBeenProcessed = True
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
if not pluginMethod(modelXbrl, factsToCheck):
factsHaveBeenProcessed = False
if factsHaveBeenProcessed:
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(fact)]
for cntx in contextsToDrop:
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
for unit in unitsToDrop:
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
for footnoteLink in footnoteLinksToDrop:
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
fact = cntx = unit = footnoteLink = None
del contextsToDrop[:]
del unitsToDrop[:]
del footnoteLinksToDrop[:]
del factsToCheck # dereference fact or batch of facts
else:
dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed
del parentMdlObj[parentMdlObj.index(mdlObj)]
if self.numRootFacts % 1000 == 0:
pass
#modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts,
# 100.0 * self.numRootFacts / instInfoNumRootFacts),
# minTimeToShow=20.0)
gc.collect()
sys.stdout.write ("\rAt fact {} of {} mem {}".format(self.numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed))
return mdlObj
def data(self, data):
self.currentMdlObj.text = data
def comment(self, text):
pass
def pi(self, target, data):
if target == "xbrl-facts-check":
_match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", data)
if _match:
_matchGroups = _match.groups()
if len(_matchGroups) == 2:
if _matchGroups[0] == "version":
self.factsCheckVersion = _matchGroups[1]
elif _matchGroups[0] == "sum-of-fact-md5s":
try:
expectedMd5 = Md5Sum(_matchGroups[1])
if self.factsCheckMd5s != expectedMd5:
modelXbrl.warning("streamingExtensions:xbrlFactsCheckWarning",
_("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"),
modelObject=modelXbrl, expectedMd5=expectedMd5, actualMd5Sum=self.factsCheckMd5s)
else:
modelXbrl.info("info",
_("Successful XBRL facts sum of md5s."),
modelObject=modelXbrl)
except ValueError:
modelXbrl.error("streamingExtensions:xbrlFactsCheckError",
_("Invalid sum-of-md5s %(sumOfMd5)s"),
modelObject=modelXbrl, sumOfMd5=_matchGroups[1])
def close(self):
del modelXbrl.makeelementParentModelObject
return None
def factCheckFact(self, fact):
self.factsCheckMd5s += fact.md5sum
for _tupleFact in fact.modelTupleFacts:
self.factCheckFact(_tupleFact)
_parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget())
etree.parse(_file, parser=_parser, base_url=filepath)
logSyntaxErrors(_parser)
'''
# replace modelLoaderTarget with iterparse (as it now supports CustomElementClassLookup)
streamingParserContext = etree.iterparse(_file,
events=("start", "end"),
huge_tree=True)
from arelle.ModelObjectFactory import setParserElementClassLookup
modelXbrl.isStreamingMode = True # must be set before setting element class lookup
(_parser, _parserLookupName,
_parserLookupClass) = setParserElementClassLookup(streamingParserContext,
modelXbrl)
foundInstance = False
beforeInstanceStream = beforeStartStreamingPlugin = True
numRootFacts = 0
factsCheckVersion = None
def factCheckFact(fact):
modelDocument._factsCheckMd5s += fact.md5sum
for _tupleFact in fact.modelTupleFacts:
factCheckFact(_tupleFact)
for event, mdlObj in streamingParserContext:
if event == "start":
if mdlObj.tag == "{http://www.xbrl.org/2003/instance}xbrl":
modelDocument = ModelDocument(modelXbrl, Type.INSTANCE,
mappedUri, filepath,
mdlObj.getroottree())
modelXbrl.modelDocument = modelDocument # needed for incremental validation
mdlObj.init(modelDocument)
modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement
modelDocument.parserLookupName = _parserLookupName
modelDocument.parserLookupClass = _parserLookupClass
modelDocument.xmlRootElement = mdlObj
modelDocument.schemaLocationElements.add(mdlObj)
modelDocument.documentEncoding = _encoding
modelDocument._creationSoftwareComment = precedingComment(
mdlObj)
modelDocument._factsCheckMd5s = Md5Sum()
modelXbrl.info("streamingExtensions:streaming",
_("Stream processing this instance."),
modelObject=modelDocument)
elif mdlObj.getparent() is not None:
mdlObj._init() # requires discovery as part of start elements
if mdlObj.getparent(
).tag == "{http://www.xbrl.org/2003/instance}xbrl":
if not foundInstance:
foundInstance = True
pi = precedingProcessingInstruction(
mdlObj, "xbrl-facts-check")
if pi is not None:
factsCheckVersion = pi.attrib.get("version", None)
elif not foundInstance:
break
ns = mdlObj.qname.namespaceURI
ln = mdlObj.qname.localName
if beforeInstanceStream:
if ((ns == XbrlConst.link
and ln not in ("schemaRef", "linkbaseRef")) or
(ns == XbrlConst.xbrli and ln in ("context", "unit"))
or (ns not in (XbrlConst.link, XbrlConst.xbrli))):
beforeInstanceStream = False
if _streamingExtensionsValidate:
instValidator.validate(
modelXbrl,
modelXbrl.modelManager.formulaOptions.
typedParameters(modelXbrl.prefixedNamespaces))
else: # need default dimensions
ValidateXbrlDimensions.loadDimensionDefaults(
modelXbrl)
elif not beforeInstanceStream and beforeStartStreamingPlugin:
for pluginMethod in pluginClassMethods("Streaming.Start"):
pluginMethod(modelXbrl)
beforeStartStreamingPlugin = False
elif event == "end":
parentMdlObj = mdlObj.getparent()
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
else:
if len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
if _streamingFactsPlugin or _streamingValidateFactsPlugin:
contextsToDrop.append(cntx)
else:
dropContext(modelXbrl, cntx)
#>>del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj, )
instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
instValidator.checkContextsDimensions(
contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if len(unitBuffer) >= unitBufferLimit:
# drop before additing as dropped may have same id as added
unit = unitBuffer.pop(0)
if _streamingFactsPlugin or _streamingValidateFactsPlugin:
unitsToDrop.append(unit)
else:
dropUnit(modelXbrl, unit)
#>>del parentMdlObj[parentMdlObj.index(unit)]
unit = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkUnits((mdlObj, ))
elif ln == "xbrl": # end of document
# check remaining batched facts if any
if _streamingFactsPlugin or _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
# finish any final batch of facts
if len(modelXbrl.facts) > 0:
factsToCheck = modelXbrl.facts.copy()
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
if _streamingValidateFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.ValidateFacts"):
pluginMethod(instValidator, factsToCheck)
if _streamingFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.Facts"):
pluginMethod(modelXbrl, factsToCheck)
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
#>>del parentMdlObj[parentMdlObj.index(fact)]
for cntx in contextsToDrop:
dropContext(modelXbrl, cntx)
#>>del parentMdlObj[parentMdlObj.index(cntx)]
for unit in unitsToDrop:
dropUnit(modelXbrl, unit)
#>>del parentMdlObj[parentMdlObj.index(unit)]
for footnoteLink in footnoteLinksToDrop:
dropFootnoteLink(modelXbrl, footnoteLink)
#>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
fact = cntx = unit = footnoteLink = None
del contextsToDrop[:]
del unitsToDrop[:]
del footnoteLinksToDrop[:]
del factsToCheck
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
pi = childProcessingInstruction(mdlObj,
"xbrl-facts-check",
reversed=True)
if pi is not None: # attrib is in .text, not attrib, no idea why!!!
_match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']",
pi.text)
if _match:
_matchGroups = _match.groups()
if len(_matchGroups) == 2:
if _matchGroups[0] == "sum-of-fact-md5s":
try:
expectedMd5 = Md5Sum(_matchGroups[1])
if modelDocument._factsCheckMd5s != expectedMd5:
modelXbrl.warning(
"streamingExtensions:xbrlFactsCheckWarning",
_("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"
),
modelObject=modelXbrl,
expectedMd5=expectedMd5,
actualMd5Sum=modelDocument.
_factsCheckMd5s)
else:
modelXbrl.info(
"info",
_("Successful XBRL facts sum of md5s."
),
modelObject=modelXbrl)
except ValueError:
modelXbrl.error(
"streamingExtensions:xbrlFactsCheckError",
_("Invalid sum-of-md5s %(sumOfMd5)s"
),
modelObject=modelXbrl,
sumOfMd5=_matchGroups[1])
if _streamingValidateFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.ValidateFinish"):
pluginMethod(instValidator)
if _streamingFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.Finish"):
pluginMethod(modelXbrl)
elif ns == XbrlConst.link:
if ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(
mdlObj,
urlRewritePluginClass=
"ModelDocument.InstanceSchemaRefRewriter")
elif ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj, ), inInstance=True)
elif ln == "footnoteLink":
XmlValidate.validate(modelXbrl, mdlObj)
footnoteLinks = (mdlObj, )
modelDocument.linkbaseDiscover(footnoteLinks,
inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
if _streamingValidateFactsPlugin:
footnoteLinksToDrop.append(footnoteLink)
else:
dropFootnoteLink(modelXbrl, footnoteLink)
#>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl and isinstance(
mdlObj, ModelFact):
numRootFacts += 1
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.factDiscover(mdlObj, modelXbrl.facts)
if factsCheckVersion:
factCheckFact(mdlObj)
if _streamingExtensionsValidate or _streamingFactsPlugin or _streamingValidateFactsPlugin:
factsToCheck = (mdlObj,
) # validate current fact by itself
if _streamingExtensionsValidate:
instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
instValidator.checkFactsDimensions(factsToCheck)
if _streamingFactsPlugin or _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
# use batches of 1000 facts
if len(modelXbrl.facts) > 1000:
factsToCheck = modelXbrl.facts.copy()
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
if _streamingValidateFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.ValidateFacts"):
pluginMethod(instValidator, factsToCheck)
if _streamingFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.Facts"):
pluginMethod(modelXbrl, factsToCheck)
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
#>>del parentMdlObj[parentMdlObj.index(fact)]
for cntx in contextsToDrop:
dropContext(modelXbrl, cntx)
#>>del parentMdlObj[parentMdlObj.index(cntx)]
for unit in unitsToDrop:
dropUnit(modelXbrl, unit)
#>>del parentMdlObj[parentMdlObj.index(unit)]
for footnoteLink in footnoteLinksToDrop:
dropFootnoteLink(modelXbrl, footnoteLink)
#>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
fact = cntx = unit = footnoteLink = None
del contextsToDrop[:]
del unitsToDrop[:]
del footnoteLinksToDrop[:]
del factsToCheck # dereference fact or batch of facts
else:
dropFact(
modelXbrl, mdlObj,
modelXbrl.facts) # single fact has been processed
#>>del parentMdlObj[parentMdlObj.index(mdlObj)]
if numRootFacts % 1000 == 0:
pass
#modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts,
# 100.0 * self.numRootFacts / instInfoNumRootFacts),
# minTimeToShow=20.0)
#gc.collect()
#sys.stdout.write ("\rAt fact {} of {} mem {}".format(numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed))
if mdlObj is not None:
mdlObj.clear()
del _parser, _parserLookupName, _parserLookupClass
if _streamingExtensionsValidate and validator is not None:
_file.close()
del instValidator
validator.close()
# track that modelXbrl has been validated by this streaming extension
modelXbrl._streamingExtensionValidated = True
modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
return modelXbrl.modelDocument
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath):
# check if big instance and has header with an initial incomplete tree walk (just 2 elements
def logSyntaxErrors(parsercontext):
for error in parsercontext.error_log:
modelXbrl.error("xmlSchema:syntax",
_("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"),
modelObject=modelDocument, fileName=os.path.basename(filepath),
error=error.message, line=error.line, column=error.column, sourceAction="streaming")
#### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
#### note: iterparse wants a binary file, but file is text mode
_file, = modelXbrl.fileSource.file(filepath, binary=True)
startedAt = time.time()
modelXbrl.profileActivity()
parsercontext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
foundInstance = False
foundErrors = False
streamingAspects = None
numRootFacts1 = 0
numElts = 0
elt = None
for event, elt in parsercontext:
if event == "start":
if elt.getparent() is not None:
if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
if not foundInstance:
foundInstance = True
pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance")
if pi is None:
break
else:
streamingAspects = dict(pi.attrib.copy())
if not elt.tag.startswith("{http://www.xbrl.org/"):
numRootFacts1 += 1
if numRootFacts1 % 1000 == 0:
modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
elif not foundInstance:
break
elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl" and precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None:
modelXbrl.error("streamingExtensions:headerMisplaced",
_("Header is misplaced: %(error)s, must follow xbrli:xbrl element"),
modelObject=elt)
elif event == "end":
elt.clear()
numElts += 1
if numElts % 1000 == 0 and elt.getparent() is not None:
while elt.getprevious() is not None and elt.getparent() is not None:
del elt.getparent()[0]
if elt is not None:
elt.clear()
_file.seek(0,io.SEEK_SET) # allow reparsing
if not foundInstance or streamingAspects is None:
del elt, parsercontext
_file.close()
return None
modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
startedAt = time.time()
try:
version = Decimal(streamingAspects.get("version"))
if int(version) != 1:
modelXbrl.error("streamingExtensions:unsupportedVersion",
_("Streaming version %(version)s, major version number must be 1"),
modelObject=elt, version=version)
foundErrors = True
except (InvalidOperation, OverflowError):
modelXbrl.error("streamingExtensions:versionError",
_("Version %(version)s, number must be 1.n"),
modelObject=elt, version=streamingAspects.get("version", "(none)"))
foundErrors = True
for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
try:
bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
raise InvalidOperation
elif bufAspect == "contextBuffer":
contextBufferLimit = bufLimit
elif bufAspect == "unitBuffer":
unitBufferLimit = bufLimit
elif bufAspect == "footnoteBuffer":
footnoteBufferLimit = bufLimit
except InvalidOperation:
modelXbrl.error("streamingExtensions:valueError",
_("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"),
modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect))
foundErrors = True
if parsercontext.error_log:
foundErrors = True
logSyntaxErrors(parsercontext)
if foundErrors:
_file.close()
return None
parsercontext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
_parser, _parserLookupName, _parserLookupClass = parser(modelXbrl,filepath)
eltMdlObjs = {}
beforeInstanceStream = True
validator = None
contextBuffer = []
unitBuffer = []
footnoteBuffer = []
factBuffer = []
numFacts = numRootFacts2 = 1
for event, elt in parsercontext:
if event == "start":
mdlObj = _parser.makeelement(elt.tag, attrib=elt.attrib, nsmap=elt.nsmap)
mdlObj.sourceline = elt.sourceline
eltMdlObjs[elt] = mdlObj
if elt.getparent() is None:
modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, etree.ElementTree(mdlObj))
modelDocument.xmlRootElement = mdlObj
modelXbrl.modelDocument = modelDocument # needed for incremental validation
mdlObj.init(modelDocument)
modelXbrl.info("streamingExtensions:streaming",
_("Stream processing this instance."),
modelObject = modelDocument)
else:
eltMdlObjs[elt.getparent()].append(mdlObj)
mdlObj._init()
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if (beforeInstanceStream and (
(ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
(ns == XbrlConst.xbrli and ln in ("context", "unit")) or
(ns not in (XbrlConst.link, XbrlConst.xbrli)))):
beforeInstanceStream = False
if _streamingExtensionsValidate:
validator = Validate(modelXbrl)
validator.instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters())
else: # need default dimensions
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
mdlObj = None # deref
elif event == "end":
mdlObj = eltMdlObjs.pop(elt)
if elt.text: # text available after child nodes processed
mdlObj.text = elt.text
ns = mdlObj.namespaceURI
ln = mdlObj.localName
parentMdlObj = mdlObj.getparent()
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
modelDocument.contextDiscover(mdlObj)
else:
if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj,)
validator.instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
validator.instValidator.checkContextsDimensions(contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
# drop before additing as dropped may have same id as added
unit = unitBuffer.pop(0)
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
unit = None
modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
validator.instValidator.checkUnits( (mdlObj,) )
elif ln == "xbrl": # end of document
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
elt.clear()
elif ns == XbrlConst.link:
if ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(mdlObj)
elif ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
elif ln == "footnoteLink":
footnoteLinks = (mdlObj,)
modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
validator.instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elt.clear()
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
numRootFacts2 += 1
modelDocument.factDiscover(mdlObj, modelXbrl.facts)
XmlValidate.validate(modelXbrl, mdlObj)
if _streamingExtensionsValidate:
factsToCheck = (mdlObj,)
validator.instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
validator.instValidator.checkFactsDimensions(factsToCheck)
del factsToCheck
dropFact(modelXbrl, mdlObj, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(mdlObj)]
if numRootFacts2 % 1000 == 0:
modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(numRootFacts2, numRootFacts1, 100.0 * numRootFacts2 / numRootFacts1),
minTimeToShow=20.0)
# get rid of root element from iterparse's tree
elt.clear()
while elt.getprevious() is not None: # cleans up any prior siblings
del elt.getparent()[0]
mdlObj = None # deref
logSyntaxErrors(parsercontext)
del parsercontext
if validator is not None:
validator.close()
_file.close()
modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
return modelDocument
def instanceDiscover(self, xbrlElement):
self.schemaLinkbaseRefsDiscover(xbrlElement)
self.linkbaseDiscover(xbrlElement,inInstance=True) # for role/arcroleRefs and footnoteLinks
self.instanceContentsDiscover(xbrlElement)
XmlValidate.validate(self.modelXbrl, xbrlElement) # validate instance elements
def produceOutputFact(xpCtx, formula, result):
priorErrorCount = len(xpCtx.modelXbrl.errors)
# assemble context
conceptQname = aspectValue(xpCtx, formula, Aspect.CONCEPT, "xbrlfe:missingConceptRule")
if isinstance(conceptQname, VariableBindingError):
xpCtx.modelXbrl.error(conceptQname.err,
_("Formula %(xlinkLabel)s concept: %(concept)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, concept=conceptQname.msg)
modelConcept = None
else:
modelConcept = xpCtx.modelXbrl.qnameConcepts[conceptQname]
if modelConcept is None or not modelConcept.isItem:
xpCtx.modelXbrl.error("xbrlfe:missingConceptRule",
_("Formula %(xlinkLabel)s concept %(concept)s is not an item"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, concept=conceptQname)
# entity
entityIdentScheme = aspectValue(xpCtx, formula, Aspect.SCHEME, "xbrlfe:missingEntityIdentifierRule")
if isinstance(entityIdentScheme, VariableBindingError):
xpCtx.modelXbrl.error(str(entityIdentScheme),
_("Formula %(xlinkLabel)s entity identifier scheme: %(scheme)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, scheme=entityIdentScheme.msg)
entityIdentValue = None
else:
entityIdentValue = aspectValue(xpCtx, formula, Aspect.VALUE, "xbrlfe:missingEntityIdentifierRule")
if isinstance(entityIdentValue, VariableBindingError):
xpCtx.modelXbrl.error(str(entityIdentScheme),
_("Formula %(xlinkLabel)s entity identifier value: %(entityIdentifier)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, entityIdentifier=entityIdentValue.msg)
# period
periodType = aspectValue(xpCtx, formula, Aspect.PERIOD_TYPE, "xbrlfe:missingPeriodRule")
periodStart = None
periodEndInstant = None
if isinstance(periodType, VariableBindingError):
xpCtx.modelXbrl.error(str(periodType),
_("Formula %(xlinkLabel)s period type: %(periodType)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, periodType=periodType.msg)
elif periodType == "instant":
periodEndInstant = aspectValue(xpCtx, formula, Aspect.INSTANT, "xbrlfe:missingPeriodRule")
if isinstance(periodEndInstant, VariableBindingError):
xpCtx.modelXbrl.error(str(periodEndInstant),
_("Formula %(xlinkLabel)s period end: %(period)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, period=periodEndInstant.msg)
elif periodType == "duration":
periodStart = aspectValue(xpCtx, formula, Aspect.START, "xbrlfe:missingPeriodRule")
if isinstance(periodStart, VariableBindingError):
xpCtx.modelXbrl.error(str(periodStart),
_("Formula %(xlinkLabel)s period start: %(period)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, period=periodStart.msg)
periodEndInstant = aspectValue(xpCtx, formula, Aspect.END, "xbrlfe:missingPeriodRule")
if isinstance(periodEndInstant, VariableBindingError):
xpCtx.modelXbrl.error(str(periodEndInstant),
_("Formula %(xlinkLabel)s period end: %(period)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, period=periodEndInstant.msg)
# unit
if modelConcept is not None and modelConcept.isNumeric:
unitSource = aspectValue(xpCtx, formula, Aspect.UNIT_MEASURES, None)
multDivBy = aspectValue(xpCtx, formula, Aspect.MULTIPLY_BY, "xbrlfe:missingUnitRule")
if isinstance(multDivBy, VariableBindingError):
xpCtx.modelXbrl.error(str(multDivBy) if isinstance(multDivBy, VariableBindingError) else "xbrlfe:missingUnitRule",
_("Formula %(xlinkLabel)s unit: %(unit)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, unit=multDivBy.msg)
multiplyBy = (); divideBy = () # prevent errors later if bad
else:
divMultBy = aspectValue(xpCtx, formula, Aspect.DIVIDE_BY, "xbrlfe:missingUnitRule")
if isinstance(divMultBy, VariableBindingError):
xpCtx.modelXbrl.error(str(multDivBy) if isinstance(divMultBy, VariableBindingError) else "xbrlfe:missingUnitRule",
_("Formula %(xlinkLabel)s unit: %(unit)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, unit=divMultBy.msg)
multiplyBy = (); divideBy = () # prevent errors later if bad
else:
multiplyBy = unitSource[0] + multDivBy[0] + divMultBy[1]
divideBy = unitSource[1] + multDivBy[1] + divMultBy[0]
# remove cancelling mult/div units
lookForCommonUnits = True
while lookForCommonUnits:
lookForCommonUnits = False
for commonUnit in multiplyBy:
if commonUnit in divideBy:
multiplyBy.remove(commonUnit)
divideBy.remove(commonUnit)
lookForCommonUnits = True
break
if len(multiplyBy) == 0: # if no units add pure
multiplyBy.append(XbrlConst.qnXbrliPure)
# dimensions
segOCCs = []
scenOCCs = []
if formula.aspectModel == "dimensional":
dimAspects = {}
dimQnames = aspectValue(xpCtx, formula, Aspect.DIMENSIONS, None)
if dimQnames:
for dimQname in dimQnames:
dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname]
dimErr = "xbrlfe:missing{0}DimensionRule".format("typed" if dimConcept is not None and dimConcept.isTypedDimension else "explicit")
dimValue = aspectValue(xpCtx, formula, dimQname, dimErr)
if isinstance(dimValue, VariableBindingError):
xpCtx.modelXbrl.error(dimErr,
_("Formula %(xlinkLabel)s dimension %(dimension)s: %(value)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel,
dimension=dimQname, value=dimValue.msg)
elif dimValue is not None and xpCtx.modelXbrl.qnameDimensionDefaults.get(dimQname) != dimValue:
dimAspects[dimQname] = dimValue
segOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SEGMENT, None)
scenOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SCENARIO, None)
else:
dimAspects = None # non-dimensional
segOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SEGMENT, None)
scenOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SCENARIO, None)
if priorErrorCount < len(xpCtx.modelXbrl.errors):
return None # had errors, don't produce output fact
# does context exist in out instance document
outputInstanceQname = formula.outputInstanceQname
outputXbrlInstance = xpCtx.inScopeVars[outputInstanceQname]
xbrlElt = outputXbrlInstance.modelDocument.xmlRootElement
# in source instance document
# add context
prevCntx = outputXbrlInstance.matchContext(
entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant,
dimAspects, segOCCs, scenOCCs)
if prevCntx is not None:
cntxId = prevCntx.id
newCntxElt = prevCntx
else:
cntxId = 'c-{0:02n}'.format( len(outputXbrlInstance.contexts) + 1)
newCntxElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId),
afterSibling=xpCtx.outputLastContext.get(outputInstanceQname))
xpCtx.outputLastContext[outputInstanceQname] = newCntxElt
entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity")
XmlUtil.addChild(entityElt, XbrlConst.xbrli, "identifier",
attributes=("scheme", entityIdentScheme),
text=entityIdentValue)
periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period")
if periodType == "forever":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever")
elif periodType == "instant":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "instant",
text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True))
elif periodType == "duration":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "startDate",
text=XmlUtil.dateunionValue(periodStart))
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "endDate",
text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True))
segmentElt = None
scenarioElt = None
from arelle.ModelInstanceObject import ModelDimensionValue
if dimAspects:
for dimQname in sorted(dimAspects.keys()):
dimValue = dimAspects[dimQname]
if isinstance(dimValue, ModelDimensionValue):
if dimValue.isExplicit:
dimMemberQname = dimValue.memberQname
contextEltName = dimValue.contextElement
else: # qname for explicit or node for typed
dimMemberQname = dimValue
contextEltName = xpCtx.modelXbrl.qnameDimensionContextElement.get(dimQname)
if contextEltName == "segment":
if segmentElt is None:
segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment")
contextElt = segmentElt
elif contextEltName == "scenario":
if scenarioElt is None:
scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario")
contextElt = scenarioElt
else:
continue
dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname]
dimAttr = ("dimension", XmlUtil.addQnameValue(xbrlElt, dimConcept.qname))
if dimConcept.isTypedDimension:
dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:typedMember",
attributes=dimAttr)
if isinstance(dimValue, ModelDimensionValue) and dimValue.isTyped:
XmlUtil.copyChildren(dimElt, dimValue)
elif dimMemberQname:
dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:explicitMember",
attributes=dimAttr,
text=XmlUtil.addQnameValue(xbrlElt, dimMemberQname))
if segOCCs:
if segmentElt is None:
segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment")
XmlUtil.copyNodes(segmentElt, segOCCs)
if scenOCCs:
if scenarioElt is None:
scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario")
XmlUtil.copyNodes(scenarioElt, scenOCCs)
outputXbrlInstance.modelDocument.contextDiscover(newCntxElt)
XmlValidate.validate(outputXbrlInstance, newCntxElt)
# does unit exist
# add unit
if modelConcept.isNumeric:
prevUnit = outputXbrlInstance.matchUnit(multiplyBy, divideBy)
if prevUnit is not None:
unitId = prevUnit.id
newUnitElt = prevUnit
else:
unitId = 'u-{0:02n}'.format( len(outputXbrlInstance.units) + 1)
newUnitElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId),
afterSibling=xpCtx.outputLastUnit.get(outputInstanceQname))
xpCtx.outputLastUnit[outputInstanceQname] = newUnitElt
if len(divideBy) == 0:
for multiply in multiplyBy:
XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply))
else:
divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide")
numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator")
denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator")
for multiply in multiplyBy:
XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply))
for divide in divideBy:
XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, divide))
outputXbrlInstance.modelDocument.unitDiscover(newUnitElt)
XmlValidate.validate(outputXbrlInstance, newUnitElt)
# add fact
attrs = [("contextRef", cntxId)]
precision = None
decimals = None
if modelConcept.isNumeric:
attrs.append(("unitRef", unitId))
value = formula.evaluate(xpCtx)
valueSeqLen = len(value)
if valueSeqLen > 1:
xpCtx.modelXbrl.error("xbrlfe:nonSingletonOutputValue",
_("Formula %(xlinkLabel)s value is a sequence of length %(valueSequenceLength)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, valueSequenceLength=valueSeqLen)
else:
if valueSeqLen == 0: #xsi:nil if no value
attrs.append((XbrlConst.qnXsiNil, "true"))
v = None
else:
# add precision/decimals for non-fraction numerics
if modelConcept.isNumeric and not modelConcept.isFraction:
if formula.hasDecimals:
decimals = formula.evaluateRule(xpCtx, Aspect.DECIMALS)
attrs.append(("decimals", decimals))
else:
if formula.hasPrecision:
precision = formula.evaluateRule(xpCtx, Aspect.PRECISION)
else:
precision = 0
attrs.append(("precision", precision))
x = value[0]
if isinstance(x,float):
from math import (log10, isnan, isinf, fabs)
if (isnan(x) or
(precision and (isinf(precision) or precision == 0)) or
(decimals and isinf(decimals))):
v = xsString(xpCtx, x)
elif decimals is not None:
v = "%.*f" % ( int(decimals), x)
elif precision is not None and precision != 0:
a = fabs(x)
log = log10(a) if a != 0 else 0
v = "%.*f" % ( int(precision) - int(log) - (1 if a >= 1 else 0), x)
else: # no implicit precision yet
v = xsString(xpCtx, x)
elif isinstance(x,QName):
v = XmlUtil.addQnameValue(xbrlElt, x)
elif isinstance(x,datetime.datetime):
v = XmlUtil.dateunionValue(x)
else:
v = xsString(xpCtx, x)
newFact = XmlUtil.addChild(xbrlElt, conceptQname,
attributes=attrs, text=v,
afterSibling=xpCtx.outputLastFact.get(outputInstanceQname))
xpCtx.outputLastFact[outputInstanceQname] = newFact
outputXbrlInstance.modelDocument.factDiscover(newFact, outputXbrlInstance.facts)
XmlValidate.validate(outputXbrlInstance, newFact)
return newFact
def backgroundSaveInstance(self, newFilename=None):
cntlr = self.modelXbrl.modelManager.cntlr
if newFilename and self.modelXbrl.modelDocument.type != ModelDocument.Type.INSTANCE:
self.modelXbrl.modelManager.showStatus(_("creating new instance {0}").format(os.path.basename(newFilename)))
self.modelXbrl.modelManager.cntlr.waitForUiThreadQueue() # force status update
self.modelXbrl.createInstance(newFilename) # creates an instance as this modelXbrl's entrypoing
instance = self.modelXbrl
cntlr.showStatus(_("Saving {0}").format(instance.modelDocument.basename))
cntlr.waitForUiThreadQueue() # force status update
newCntx = ModelXbrl.AUTO_LOCATE_ELEMENT
newUnit = ModelXbrl.AUTO_LOCATE_ELEMENT
# check user keyed changes
for bodyCell in self.gridBody.winfo_children():
if isinstance(bodyCell, gridCell) and bodyCell.isChanged:
value = bodyCell.value
objId = bodyCell.objectId
if objId:
if objId[0] == "f":
factPrototypeIndex = int(objId[1:])
factPrototype = self.factPrototypes[factPrototypeIndex]
concept = factPrototype.concept
entityIdentScheme = self.newFactItemOptions.entityIdentScheme
entityIdentValue = self.newFactItemOptions.entityIdentValue
periodType = factPrototype.concept.periodType
periodStart = self.newFactItemOptions.startDateDate if periodType == "duration" else None
periodEndInstant = self.newFactItemOptions.endDateDate
qnameDims = factPrototype.context.qnameDims
prevCntx = instance.matchContext(
entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant,
qnameDims, [], [])
if prevCntx is not None:
cntxId = prevCntx.id
else: # need new context
newCntx = instance.createContext(entityIdentScheme, entityIdentValue,
periodType, periodStart, periodEndInstant,
concept.qname, qnameDims, [], [],
afterSibling=newCntx)
cntxId = newCntx.id
# new context
if concept.isNumeric:
if concept.isMonetary:
unitMeasure = qname(XbrlConst.iso4217, self.newFactItemOptions.monetaryUnit)
unitMeasure.prefix = "iso4217" # want to save with a recommended prefix
decimals = self.newFactItemOptions.monetaryDecimals
elif concept.isShares:
unitMeasure = XbrlConst.qnXbrliShares
decimals = self.newFactItemOptions.nonMonetaryDecimals
else:
unitMeasure = XbrlConst.qnXbrliPure
decimals = self.newFactItemOptions.nonMonetaryDecimals
prevUnit = instance.matchUnit([unitMeasure],[])
if prevUnit is not None:
unitId = prevUnit.id
else:
newUnit = instance.createUnit([unitMeasure],[], afterSibling=newUnit)
unitId = newUnit.id
attrs = [("contextRef", cntxId)]
if concept.isNumeric:
attrs.append(("unitRef", unitId))
attrs.append(("decimals", decimals))
value = Locale.atof(self.modelXbrl.locale, value, str.strip)
newFact = instance.createFact(concept.qname, attributes=attrs, text=value)
bodyCell.objectId = newFact.objectId() # switch cell to now use fact ID
if self.factPrototypes[factPrototypeIndex] is not None:
self.factPrototypes[factPrototypeIndex].clear()
self.factPrototypes[factPrototypeIndex] = None #dereference fact prototype
else: # instance fact, not prototype
fact = self.modelXbrl.modelObject(objId)
if fact.concept.isNumeric:
value = Locale.atof(self.modelXbrl.locale, value, str.strip)
if fact.value != value:
if fact.concept.isNumeric and fact.isNil != (not value):
fact.isNil = not value
if value: # had been nil, now it needs decimals
fact.decimals = (self.newFactItemOptions.monetaryDecimals
if fact.concept.isMonetary else
self.newFactItemOptions.nonMonetaryDecimals)
fact.text = value
XmlValidate.validate(instance, fact)
bodyCell.isChanged = False # clear change flag
instance.saveInstance(newFilename) # may override prior filename for instance from main menu
cntlr.showStatus(_("Saved {0}").format(instance.modelDocument.basename), clearAfter=3000)
def createContext(self, entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, priItem, dims, segOCCs, scenOCCs,
afterSibling=None, beforeSibling=None):
xbrlElt = self.modelDocument.xmlRootElement
if afterSibling == AUTO_LOCATE_ELEMENT:
afterSibling = XmlUtil.lastChild(xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context"))
cntxId = 'c-{0:02n}'.format( len(self.contexts) + 1)
newCntxElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId),
afterSibling=afterSibling, beforeSibling=beforeSibling)
entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity")
XmlUtil.addChild(entityElt, XbrlConst.xbrli, "identifier",
attributes=("scheme", entityIdentScheme),
text=entityIdentValue)
periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period")
if periodType == "forever":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever")
elif periodType == "instant":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "instant",
text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True))
elif periodType == "duration":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "startDate",
text=XmlUtil.dateunionValue(periodStart))
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "endDate",
text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True))
segmentElt = None
scenarioElt = None
from arelle.ModelInstanceObject import ModelDimensionValue
if dims: # requires primary item to determin ambiguous concepts
''' in theory we have to check full set of dimensions for validity in source or any other
context element, but for shortcut will see if each dimension is already reported in an
unambiguous valid contextElement
'''
from arelle.PrototypeInstanceObject import FactPrototype, ContextPrototype, DimValuePrototype
fp = FactPrototype(self, priItem, dims.items())
# force trying a valid prototype's context Elements
if not isFactDimensionallyValid(self, fp, setPrototypeContextElements=True):
self.info("arelleLinfo",
_("Create context for %(priItem)s, cannot determine valid context elements, no suitable hypercubes"),
modelObject=self, priItem=priItem)
fpDims = fp.context.qnameDims
for dimQname in sorted(fpDims.keys()):
dimValue = fpDims[dimQname]
if isinstance(dimValue, DimValuePrototype):
dimMemberQname = dimValue.memberQname # None if typed dimension
contextEltName = dimValue.contextElement
else: # qname for explicit or node for typed
dimMemberQname = None
contextEltName = None
if contextEltName == "segment":
if segmentElt is None:
segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment")
contextElt = segmentElt
elif contextEltName == "scenario":
if scenarioElt is None:
scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario")
contextElt = scenarioElt
else:
self.info("arelleLinfo",
_("Create context, %(dimension)s, cannot determine context element, either no all relationship or validation issue"),
modelObject=self, dimension=dimQname),
continue
dimConcept = self.qnameConcepts[dimQname]
dimAttr = ("dimension", XmlUtil.addQnameValue(xbrlElt, dimConcept.qname))
if dimConcept.isTypedDimension:
dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:typedMember",
attributes=dimAttr)
if isinstance(dimValue, (ModelDimensionValue, DimValuePrototype)) and dimValue.isTyped:
XmlUtil.copyNodes(dimElt, dimValue.typedMember)
elif dimMemberQname:
dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:explicitMember",
attributes=dimAttr,
text=XmlUtil.addQnameValue(xbrlElt, dimMemberQname))
if segOCCs:
if segmentElt is None:
segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment")
XmlUtil.copyNodes(segmentElt, segOCCs)
if scenOCCs:
if scenarioElt is None:
scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario")
XmlUtil.copyNodes(scenarioElt, scenOCCs)
self.modelDocument.contextDiscover(newCntxElt)
XmlValidate.validate(self, newCntxElt)
return newCntxElt
