def insertFactSet(modelFacts, tupleFactId):
table = self.getTable('fact', 'fact_id',
('accession_id', 'tuple_fact_id', 'context_id', 'unit_id', 'element_id', 'effective_value', 'fact_value',
'xml_id', 'precision_value', 'decimals_value',
'is_precision_infinity', 'is_decimals_infinity', ),
('accession_id', 'xml_id'),
tuple((accsId,
tupleFactId,
self.cntxId.get((accsId,fact.contextID)),
self.unitId.get((accsId,fact.unitID)),
self.conceptElementId(fact.concept),
roundValue(fact.value, fact.precision, fact.decimals) if fact.isNumeric and not fact.isNil else None,
fact.value,
elementFragmentIdentifier(fact),
fact.xAttributes['precision'].xValue if ('precision' in fact.xAttributes and isinstance(fact.xAttributes['precision'].xValue,int)) else None,
fact.xAttributes['decimals'].xValue if ('decimals' in fact.xAttributes and isinstance(fact.xAttributes['decimals'].xValue,int)) else None,
'precision' in fact.xAttributes and fact.xAttributes['precision'].xValue == 'INF',
'decimals' in fact.xAttributes and fact.xAttributes['decimals'].xValue == 'INF',
)
for fact in modelFacts))
factId = dict((xmlId, id)
for id, _accsId, xmlId in table)
for fact in modelFacts:
if fact.isTuple:
insertFactSet(fact.modelTupleFacts,
factId[elementFragmentIdentifier(fact)])
def insertRoleTypes(self):
self.showStatus("insert role types")
roleTypesByIds = dict(((self.documentIds[roleType.modelDocument],
roleType.roleURI), # key on docId, uriId
roleType) # value is roleType object
for roleTypes in self.modelXbrl.roleTypes.values()
for roleType in roleTypes
if roleType.modelDocument not in self.existingDocumentIds)
table = self.getTable('role_type', 'role_type_id',
('document_id', 'xml_id', 'role_uri', 'definition'),
('document_id', 'role_uri'),
tuple((roleTypeIDs[0], # doc Id
elementFragmentIdentifier(roleType),
roleTypeIDs[1], # uri Id
roleType.definition)
for roleTypeIDs, roleType in roleTypesByIds.items()))
self.roleTypeIds = {}
for roleId, docId, uri in table:
self.roleTypeIds[(docId, uri)] = roleId
table = self.getTable('used_on',
None, # no record id in this table
('object_id', 'aspect_id'),
('object_id', 'aspect_id'),
tuple((self.roleTypeIds[(roleTypeIDs[0], roleType.roleURI)],
self.aspectQnameId[usedOnQn])
for roleTypeIDs, roleType in roleTypesByIds.items()
for usedOnQn in roleType.usedOns
if usedOnQn in self.aspectQnameId),
checkIfExisting=True)
def insertArcroleTypes(self):
self.showStatus("insert arcrole types")
arcroleTypesByIds = dict(((self.documentIds[arcroleType.modelDocument],
arcroleType.arcroleURI), # key on docId, uriId
arcroleType) # value is roleType object
for arcroleTypes in self.modelXbrl.arcroleTypes.values()
for arcroleType in arcroleTypes
if arcroleType.modelDocument not in self.existingDocumentIds)
table = self.getTable('arcrole_type', 'arcrole_type_id',
('document_id', 'xml_id', 'arcrole_uri', 'cycles_allowed', 'definition'),
('document_id', 'arcrole_uri'),
tuple((arcroleTypeIDs[0], # doc Id
elementFragmentIdentifier(arcroleType),
arcroleType.arcroleURI,
arcroleType.cyclesAllowed,
arcroleType.definition)
for arcroleTypeIDs, arcroleType in arcroleTypesByIds.items()))
self.arcroleTypeIds = {}
for arcroleId, docId, uri in table:
self.arcroleTypeIds[(docId, uri)] = arcroleId
table = self.getTable('used_on',
None, # no record id in this table
('object_id', 'aspect_id'),
('object_id', 'aspect_id'),
tuple((self.arcroleTypeIds[(arcroleTypeIDs[0], arcroleType.arcroleURI)],
self.aspectQnameId[usedOnQn])
for arcroleTypeIDs, arcroleType in arcroleTypesByIds.items()
for usedOnQn in arcroleType.usedOns
if usedOnQn in self.aspectQnameId),
checkIfExisting=True)
def insertResources(self):
self.showStatus("insert resources")
# deduplicate resources (may be on multiple arcs)
# note that lxml has no column numbers, use objectIndex as pseudo-column number
uniqueResources = dict(((self.documentIds[resource.modelDocument],
resource.objectIndex), resource)
for arcrole in (XbrlConst.conceptLabel, XbrlConst.conceptReference)
for rel in self.modelXbrl.relationshipSet(arcrole).modelRelationships
if rel.fromModelObject is not None and rel.toModelObject is not None
for resource in (rel.fromModelObject, rel.toModelObject)
if isinstance(resource, ModelResource))
table = self.getTable('resource', 'resource_id',
('document_id', 'xml_id', 'qname', 'role', 'value', 'xml_lang'),
('document_id', 'xml_id'),
tuple((self.documentIds[resource.modelDocument],
elementFragmentIdentifier(resource),
resource.qname.clarkNotation,
resource.role,
resource.textValue,
resource.xmlLang)
for resource in uniqueResources.values()),
checkIfExisting=True)
self.resourceId = dict(((docId, xml_id), id)
for id, docId, xml_id in table)
uniqueResources.clear()
def modelObjectId(self, modelObject):
if isinstance(modelObject, ModelConcept):
return self.aspectQnameId.get(modelObject.qname)
elif isinstance(modelObject, ModelType):
return self.aspectTypeIds.get(modelObject.qname)
elif isinstance(modelObject, ModelResource):
return self.resourceId.get((self.documentIds[modelObject.modelDocument],
elementFragmentIdentifier(modelObject)))
else:
return None
def insertFactSet(modelFacts, parentDatapointId):
table = self.getTable('data_point', 'datapoint_id',
('report_id', 'document_id', 'xml_id', 'source_line',
'parent_datapoint_id', # tuple
'aspect_id',
'context_xml_id', 'entity_id', 'period_id', 'aspect_value_selections_id', 'unit_id',
'is_nil', 'precision_value', 'decimals_value', 'effective_value', 'value'),
('document_id', 'xml_id'),
tuple((reportId,
documentId,
elementFragmentIdentifier(fact),
fact.sourceline,
parentDatapointId, # parent ID
self.aspectQnameId.get(fact.qname),
fact.contextID,
self.entityId.get((reportId, cntx.entityIdentifier[0], cntx.entityIdentifier[1]))
if cntx is not None else None,
self.periodId.get((reportId,
cntx.startDatetime if cntx.isStartEndPeriod else None,
cntx.endDatetime if cntx.isStartEndPeriod else None,
cntx.isInstantPeriod,
cntx.isForeverPeriod)) if cntx is not None else None,
cntxAspectValueSelectionSetId.get(cntx) if cntx is not None else None,
self.unitId.get((reportId,fact.unit.md5hash)) if fact.unit is not None else None,
fact.isNil,
fact.precision,
fact.decimals,
roundValue(fact.value, fact.precision, fact.decimals) if fact.isNumeric and not fact.isNil else None,
fact.value
)
for fact in modelFacts
for cntx in (fact.context,)
for documentId in (self.documentIds[fact.modelDocument],)))
xmlIdDataPointId = dict(((docId, xmlId), datapointId)
for datapointId, docId, xmlId in table)
self.factDataPointId.update(xmlIdDataPointId)
for fact in modelFacts:
if fact.isTuple:
insertFactSet(fact.modelTupleFacts,
xmlIdDataPointId[(self.documentIds[fact.modelDocument],
elementFragmentIdentifier(fact))])
def createTargetInstance(modelXbrl,
targetUrl,
targetDocumentSchemaRefs,
filingFiles,
baseXmlLang=None,
defaultXmlLang=None):
def addLocallyReferencedFile(elt, filingFiles):
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src") and not isHttpUrl(
attrValue) and not os.path.isabs(attrValue):
attrValue = attrValue.partition('#')[0] # remove anchor
if attrValue: # ignore anchor references to base document
attrValue = os.path.normpath(
attrValue
) # change url path separators to host separators
file = os.path.join(sourceDir, attrValue)
if modelXbrl.fileSource.isInArchive(
file,
checkExistence=True) or os.path.exists(file):
filingFiles.add(file)
targetInstance = ModelXbrl.create(
modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True,
discover=False) # don't attempt to load DTS
ixTargetRootElt = modelXbrl.ixTargetRootElements[getattr(
modelXbrl, "ixdsTarget", None)]
langIsSet = False
# copy ix resources target root attributes
for attrName, attrValue in ixTargetRootElt.items():
if attrName != "target": # ix:references target is not mapped to xbrli:xbrl
targetInstance.modelDocument.xmlRootElement.set(
attrName, attrValue)
if attrName == "{http://www.w3.org/XML/1998/namespace}lang":
langIsSet = True
defaultXmlLang = attrValue
if attrName.startswith("{"):
ns, _sep, ln = attrName[1:].rpartition("}")
if ns:
prefix = xmlnsprefix(ixTargetRootElt, ns)
if prefix not in (None, "xml"):
setXmlns(targetInstance.modelDocument, prefix, ns)
if not langIsSet and baseXmlLang:
targetInstance.modelDocument.xmlRootElement.set(
"{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
if defaultXmlLang is None:
defaultXmlLang = baseXmlLang # allows facts/footnotes to override baseXmlLang
ValidateXbrlDimensions.loadDimensionDefaults(
targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
addChild(targetInstance.modelDocument.xmlRootElement,
roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in sorted(modelXbrl.contexts.values(),
key=lambda c: c.objectIndex
): # contexts may come from multiple IXDS files
ignore = targetInstance.createContext(
context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in sorted(modelXbrl.units.values(), key=lambda u: u.objectIndex
): # units may come from multiple IXDS files
measures = unit.measures
ignore = targetInstance.createUnit(measures[0],
measures[1],
id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
for attrName, attrValue in fact.items():
if attrName.startswith("{"):
attrs[qname(
attrName, fact.nsmap
)] = attrValue # using qname allows setting prefix in extracted instance
newFact = targetInstance.createFact(fact.qname,
attributes=attrs,
text=text,
parent=parent)
# if fact.isFraction, create numerator and denominator
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references so that referenced files are included in the zip.
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML("<body>\n{0}\n</body>\n".format(
xmltext)).iter():
addLocallyReferencedFile(elt, filingFiles)
except (XMLSyntaxError, UnicodeDecodeError):
pass # TODO: Why ignore UnicodeDecodeError?
elif fact.isTuple:
attrs = {}
if fact.id:
attrs["id"] = fact.id
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
for attrName, attrValue in fact.items():
if attrName.startswith("{"):
attrs[qname(attrName, fact.nsmap)] = attrValue
newTuple = targetInstance.createFact(fact.qname,
attributes=attrs,
parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
modelXbrl.modelManager.showStatus(
_("Creating and validating footnotes and relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
footnoteIdCount = {}
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and any(
lP.modelDocument.type == Type.INLINEXBRL
for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype):
if HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ArcPrototype):
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID,
0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(
attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
newChild = addChild(newLink,
linkChild.qname,
attributes=attributes)
xmlLang = linkChild.xmlLang
if xmlLang is not None and xmlLang != defaultXmlLang: # default
newChild.set(
"{http://www.w3.org/XML/1998/namespace}lang",
xmlLang)
copyIxFootnoteHtml(
linkChild,
newChild,
targetModelDocument=targetInstance.modelDocument,
withText=True)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyIxFootnoteHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
addLocallyReferencedFile(elt, filingFiles)
return targetInstance
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None, *args, **kwargs):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
def addLocallyReferencedFile(elt,filingFiles):
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrValue):
attrValue = attrValue.partition('#')[0] # remove anchor
if attrValue: # ignore anchor references to base document
attrValue = os.path.normpath(attrValue) # change url path separators to host separators
file = os.path.join(sourceDir,attrValue)
if modelXbrl.fileSource.isInArchive(file, checkExistence=True) or os.path.exists(file):
filingFiles.add(file)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
rootElt = modelXbrl.modelDocument.xmlRootElement
# take baseXmlLang from <html> or <base>
baseXmlLang = rootElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang")
for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/1999/xhtml}body"):
baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang") or baseXmlLang
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True,
discover=False) # don't attempt to load DTS
if baseXmlLang:
targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in sorted(modelXbrl.contexts.values(), key=lambda c: elementChildSequence(c)):
ignore = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default
xmlLang = fact.xmlLang
if xmlLang is not None and xmlLang != baseXmlLang:
attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
# if fact.isFraction, create numerator and denominator
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references so that referenced files are included in the zip.
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter():
addLocallyReferencedFile(elt, filingFiles)
except (XMLSyntaxError, UnicodeDecodeError):
pass # TODO: Why ignore UnicodeDecodeError?
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
footnoteIdCount = {}
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and
any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype):
if HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
addChild(newLink, linkChild.qname,
attributes=attributes)
elif isinstance(linkChild, ArcPrototype):
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
newChild = addChild(newLink, linkChild.qname,
attributes=attributes)
xmlLang = linkChild.xmlLang
if xmlLang is not None and xmlLang != baseXmlLang: # default
newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang)
copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyIxFootnoteHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
addLocallyReferencedFile(elt,filingFiles)
targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip)
if getattr(modelXbrl, "isTestcaseVariation", False):
modelXbrl.extractedInlineInstance = True # for validation comparison
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def saveTargetDocument(modelXbrl,
targetDocumentFilename,
targetDocumentSchemaRefs,
outputZip=None,
filingFiles=None,
*args,
**kwargs):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
targetDocumentFilename, modelXbrl.modelDocument.filepath)
def addLocallyReferencedFile(elt, filingFiles):
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src") and not isHttpUrl(
attrValue) and not os.path.isabs(attrValue):
attrValue = attrValue.partition('#')[0] # remove anchor
if attrValue: # ignore anchor references to base document
attrValue = os.path.normpath(
attrValue
) # change url path separators to host separators
file = os.path.join(sourceDir, attrValue)
if modelXbrl.fileSource.isInArchive(
file,
checkExistence=True) or os.path.exists(file):
filingFiles.add(file)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(
_("Extracting instance ") + os.path.basename(targetUrl))
rootElt = modelXbrl.modelDocument.xmlRootElement
# take baseXmlLang from <html> or <base>
baseXmlLang = rootElt.get(
"{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang")
for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.w3.org/1999/xhtml}body"):
baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang"
) or rootElt.get("lang") or baseXmlLang
targetInstance = ModelXbrl.create(
modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True,
discover=False) # don't attempt to load DTS
if baseXmlLang:
targetInstance.modelDocument.xmlRootElement.set(
"{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
ValidateXbrlDimensions.loadDimensionDefaults(
targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
addChild(targetInstance.modelDocument.xmlRootElement,
roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in sorted(modelXbrl.contexts.values(),
key=lambda c: elementChildSequence(c)):
ignore = targetInstance.createContext(
context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
ignore = targetInstance.createUnit(measures[0],
measures[1],
id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
if fact.concept is not None and fact.concept.baseXsdType in (
"string", "normalizedString"): # default
xmlLang = fact.xmlLang
if xmlLang is not None and xmlLang != baseXmlLang:
attrs[
"{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang
newFact = targetInstance.createFact(fact.qname,
attributes=attrs,
text=text,
parent=parent)
# if fact.isFraction, create numerator and denominator
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references so that referenced files are included in the zip.
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML("<body>\n{0}\n</body>\n".format(
xmltext)).iter():
addLocallyReferencedFile(elt, filingFiles)
except (XMLSyntaxError, UnicodeDecodeError):
pass # TODO: Why ignore UnicodeDecodeError?
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
modelXbrl.modelManager.showStatus(
_("Creating and validating footnotes and relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
footnoteIdCount = {}
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and any(
lP.modelDocument.type == Type.INLINEXBRL
for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype):
if HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ArcPrototype):
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID,
0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(
attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
newChild = addChild(newLink,
linkChild.qname,
attributes=attributes)
xmlLang = linkChild.xmlLang
if xmlLang is not None and xmlLang != baseXmlLang: # default
newChild.set(
"{http://www.w3.org/XML/1998/namespace}lang",
xmlLang)
copyIxFootnoteHtml(
linkChild,
newChild,
targetModelDocument=targetInstance.modelDocument,
withText=True)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyIxFootnoteHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
addLocallyReferencedFile(elt, filingFiles)
targetInstance.saveInstance(overrideFilepath=targetUrl,
outputZip=outputZip)
if getattr(modelXbrl, "isTestcaseVariation", False):
modelXbrl.extractedInlineInstance = True # for validation comparison
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def saveTargetDocument(modelXbrl,
targetDocumentFilename,
targetDocumentSchemaRefs,
outputZip=None,
filingFiles=None,
*args,
**kwargs):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
targetDocumentFilename, modelXbrl.modelDocument.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(
_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(
targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
addChild(targetInstance.modelDocument.xmlRootElement,
roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in modelXbrl.contexts.values():
newCntx = targetInstance.createContext(
context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0],
measures[1],
id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem:
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
newFact = targetInstance.createFact(fact.qname,
attributes=attrs,
text=text,
parent=parent)
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML(
"<body>\n{0}\n</body>\n".format(xmltext)):
if elt.tag in ("a", "img") and not isHttpUrl(
attrValue) and not os.path.isabs(
attrvalue):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src"):
filingFiles.add(attrValue)
except (XMLSyntaxError, UnicodeDecodeError):
pass
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
# footnote links
footnoteIdCount = {}
modelXbrl.modelManager.showStatus(
_("Creating and validating footnotes & relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and any(
lP.modelDocument.type == Type.INLINEXBRL
for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype):
if HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ArcPrototype):
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID,
0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(
attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
newChild = addChild(newLink,
linkChild.qname,
attributes=attributes)
copyIxFootnoteHtml(
linkChild,
newChild,
targetModelDocument=targetInstance.modelDocument,
withText=True)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyIxFootnoteHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in (
"href", "src") and not isHttpUrl(
attrValue
) and not os.path.isabs(attrvalue):
filingFiles.add(attrValue)
targetInstance.saveInstance(overrideFilepath=targetUrl,
outputZip=outputZip)
if getattr(modelXbrl, "isTestcaseVariation", False):
modelXbrl.extractedInlineInstance = True # for validation comparison
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in modelXbrl.contexts.values():
newCntx = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem:
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)):
if elt.tag in ("a", "img") and not isHttpUrl(attrValue) and not os.path.isabs(attrvalue):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src"):
filingFiles.add(attrValue)
except (XMLSyntaxError, UnicodeDecodeError):
pass
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
# footnote links
footnoteIdCount = {}
modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and
any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype):
if HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
addChild(newLink, linkChild.qname,
attributes=attributes)
elif isinstance(linkChild, ArcPrototype):
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
newChild = addChild(newLink, linkChild.qname,
attributes=attributes)
copyIxFootnoteHtml(linkChild, newChild, withText=True)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyIxFootnoteHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrvalue):
filingFiles.add(attrValue)
targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def createTargetInstance(modelXbrl, targetUrl, targetDocumentSchemaRefs, filingFiles, baseXmlLang=None, defaultXmlLang=None):
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True,
discover=False) # don't attempt to load DTS
if baseXmlLang:
targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
if defaultXmlLang is None:
defaultXmlLang = baseXmlLang # allows facts/footnotes to override baseXmlLang
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in sorted(modelXbrl.contexts.values(), key=lambda c: c.objectIndex): # contexts may come from multiple IXDS files
ignore = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in sorted(modelXbrl.units.values(), key=lambda u: u.objectIndex): # units may come from multiple IXDS files
measures = unit.measures
ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default
xmlLang = fact.xmlLang
if xmlLang is not None and xmlLang != defaultXmlLang:
attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
# if fact.isFraction, create numerator and denominator
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references so that referenced files are included in the zip.
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter():
addLocallyReferencedFile(elt, filingFiles)
except (XMLSyntaxError, UnicodeDecodeError):
pass # TODO: Why ignore UnicodeDecodeError?
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
footnoteIdCount = {}
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and
any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype):
if HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
addChild(newLink, linkChild.qname,
attributes=attributes)
elif isinstance(linkChild, ArcPrototype):
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
newChild = addChild(newLink, linkChild.qname,
attributes=attributes)
xmlLang = linkChild.xmlLang
if xmlLang is not None and xmlLang != defaultXmlLang: # default
newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang)
copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyIxFootnoteHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
addLocallyReferencedFile(elt,filingFiles)
return targetInstance
def bodyCells(self, row, yParentStructuralNode, xStructuralNodes, zAspects, yChildrenFirst):
if yParentStructuralNode is not None:
rendrCntx = getattr(self.modelXbrl, "rendrCntx", None) # none for EU 2010 tables
dimDefaults = self.modelXbrl.qnameDimensionDefaults
for yStructuralNode in yParentStructuralNode.childStructuralNodes:
if yChildrenFirst:
row = self.bodyCells(row, yStructuralNode, xStructuralNodes, zAspects, yChildrenFirst)
if not yStructuralNode.isAbstract:
if self.type == XML:
self.xCells = etree.SubElement(self.yCells, tableModelQName("cells"),
attrib={"disposition": "x"})
yAspects = defaultdict(set)
for aspect in aspectModels[self.aspectModel]:
for ruleAspect in aspectRuleAspects.get(aspect, (aspect,)):
if yStructuralNode.hasAspect(ruleAspect):
if ruleAspect == Aspect.DIMENSIONS:
for dim in (yStructuralNode.aspectValue(Aspect.DIMENSIONS) or emptyList):
yAspects[dim].add(yStructuralNode)
else:
yAspects[ruleAspect].add(yStructuralNode)
# data for columns of rows
ignoreDimValidity = self.ignoreDimValidity.get()
for i, xStructuralNode in enumerate(xStructuralNodes):
xAspects = defaultdict(set)
for aspect in aspectModels[self.aspectModel]:
for ruleAspect in aspectRuleAspects.get(aspect, (aspect,)):
if xStructuralNode.hasAspect(ruleAspect):
if ruleAspect == Aspect.DIMENSIONS:
for dim in (xStructuralNode.aspectValue(Aspect.DIMENSIONS) or emptyList):
xAspects[dim].add(xStructuralNode)
else:
xAspects[ruleAspect].add(xStructuralNode)
cellAspectValues = {}
matchableAspects = set()
for aspect in _DICT_SET(xAspects.keys()) | _DICT_SET(yAspects.keys()) | _DICT_SET(zAspects.keys()):
aspectValue = inheritedAspectValue(self, aspect, xAspects, yAspects, zAspects, xStructuralNode, yStructuralNode)
if dimDefaults.get(aspect) != aspectValue: # don't include defaulted dimensions
cellAspectValues[aspect] = aspectValue
matchableAspects.add(aspectModelAspect.get(aspect,aspect)) #filterable aspect from rule aspect
cellDefaultedDims = _DICT_SET(dimDefaults) - _DICT_SET(cellAspectValues.keys())
priItemQname = cellAspectValues.get(Aspect.CONCEPT)
concept = self.modelXbrl.qnameConcepts.get(priItemQname)
conceptNotAbstract = concept is None or not concept.isAbstract
from arelle.ValidateXbrlDimensions import isFactDimensionallyValid
value = None
objectId = None
justify = None
fp = FactPrototype(self, cellAspectValues)
if conceptNotAbstract:
# reduce set of matchable facts to those with pri item qname and have dimension aspects
facts = self.modelXbrl.factsByQname[priItemQname] if priItemQname else self.modelXbrl.factsInInstance
for aspect in matchableAspects: # trim down facts with explicit dimensions match or just present
if isinstance(aspect, QName):
aspectValue = cellAspectValues.get(aspect, None)
if isinstance(aspectValue, ModelDimensionValue):
if aspectValue.isExplicit:
dimMemQname = aspectValue.memberQname # match facts with this explicit value
else:
dimMemQname = None # match facts that report this dimension
elif isinstance(aspectValue, QName):
dimMemQname = aspectValue # match facts that have this explicit value
else:
dimMemQname = None # match facts that report this dimension
facts = facts & self.modelXbrl.factsByDimMemQname(aspect, dimMemQname)
for fact in facts:
if (all(aspectMatches(rendrCntx, fact, fp, aspect)
for aspect in matchableAspects) and
all(fact.context.dimMemberQname(dim,includeDefaults=True) in (dimDefaults[dim], None)
for dim in cellDefaultedDims)):
if yStructuralNode.hasValueExpression(xStructuralNode):
value = yStructuralNode.evalValueExpression(fact, xStructuralNode)
else:
value = fact.effectiveValue
justify = "right" if fact.isNumeric else "left"
break
if conceptNotAbstract:
if value is not None or ignoreDimValidity or isFactDimensionallyValid(self, fp):
if self.type == HTML:
etree.SubElement(self.rowElts[row - 1],
"{http://www.w3.org/1999/xhtml}td",
attrib={"class":"cell",
"style":"text-align:{0};width:8em".format(justify)}
).text = value or "\u00A0"
elif self.type == XML:
if value is not None and fact is not None:
self.xCells.append(etree.Comment("{0}: context {1}, value {2}, file {3}, line {4}"
.format(fact.qname,
fact.contextID,
value,
fact.modelDocument.basename,
fact.sourceline)))
cellElt = etree.SubElement(self.xCells, tableModelQName("cell"))
etree.SubElement(cellElt, tableModelQName("fact")
).text = '#' + elementFragmentIdentifier(fact)
else:
if self.type == HTML:
etree.SubElement(self.rowElts[row - 1],
"{http://www.w3.org/1999/xhtml}td",
attrib={"class":"blockedCell",
"style":"text-align:{0};width:8em".format(justify)}
).text = "\u00A0\u00A0"
elif self.type == XML:
etree.SubElement(self.xCells, tableModelQName("cell"),
attrib={"blocked":"true"})
else: # concept is abstract
if self.type == HTML:
etree.SubElement(self.rowElts[row - 1],
"{http://www.w3.org/1999/xhtml}td",
attrib={"class":"abstractCell",
"style":"text-align:{0};width:8em".format(justify)}
).text = "\u00A0\u00A0"
elif self.type == XML:
etree.SubElement(self.xCells, tableModelQName("cell"),
attrib={"abstract":"true"})
fp.clear() # dereference
row += 1
if not yChildrenFirst:
row = self.bodyCells(row, yStructuralNode, xStructuralNodes, zAspects, yChildrenFirst)
return row
def insertValidationResults(self):
reportId = self.reportId
if self.filingPreviouslyInDB:
self.showStatus("deleting prior messages of this report")
# remove prior messages for this report
self.execute("DELETE from {0} "
"USING {1} "
"WHERE {1}.report_id = {2} AND {1}.message_id = {0}.message_id"
.format(self.dbTableName("message_reference"),
self.dbTableName("message"),
reportId),
close=False, fetch=False)
self.execute("DELETE FROM {} WHERE message.report_id = {}"
.format(self.dbTableName("message"),reportId),
close=False, fetch=False)
messages = []
messageRefs = defaultdict(set) # direct link to objects
for i, logEntry in enumerate(self.loggingEntries):
sequenceInReport = i+1
for ref in logEntry['refs']:
modelObject = self.modelXbrl.modelObject(ref.get('objectId',''))
# for now just find a concept
objectId = None
if isinstance(modelObject, ModelFact):
objectId = self.factDataPointId[(self.documentIds[modelObject.modelDocument],
elementFragmentIdentifier(modelObject))]
elif isinstance(modelObject, ModelRelationship):
objectId = self.relSetId[(modelObject.linkrole,
modelObject.arcrole,
modelObject.linkQname.clarkNotation,
modelObject.arcElement.qname.clarkNotation)]
elif isinstance(modelObject, ModelConcept):
objectId = self.aspectQnameId.get(modelObject.qname)
elif isinstance(modelObject, ModelXbrl):
objectId = reportId
elif hasattr(modelObject, "modelDocument"):
objectId = self.documentIds[modelObject.modelDocument]
if objectId is not None:
messageRefs[sequenceInReport].add(objectId)
messages.append((reportId,
sequenceInReport,
logEntry['code'],
logEntry['level'],
logEntry['message']['text']))
if messages:
self.showStatus("insert validation messages")
table = self.getTable('message', 'message_id',
('report_id', 'sequence_in_report', 'message_code', 'message_level', 'value'),
('report_id', 'sequence_in_report'),
messages)
messageIds = dict((sequenceInReport, messageId)
for messageId, _reportId, sequenceInReport in table)
table = self.getTable('message_reference', None,
('message_id', 'object_id'),
('message_id', 'object_id'),
tuple((messageId,
objectId)
for sequenceInReport, objectIds in messageRefs.items()
for objectId in objectIds
for messageId in (messageIds[sequenceInReport],)))
def insertFacts(self):
accsId = self.accessionId
self.showStatus("insert facts")
# units
table = self.getTable('unit', 'unit_id',
('accession_id', 'unit_xml_id'),
('accession_id', 'unit_xml_id'),
tuple((accsId,
unitId)
for unitId in self.modelXbrl.units.keys()))
self.unitId = dict(((_accsId, xmlId), id)
for id, _accsId, xmlId in table)
# measures
table = self.getTable('unit_measure', 'unit_measure_id',
('unit_id', 'qname_id', 'location_id'),
('qname_id', 'location_id'),
tuple((self.unitId[(accsId,unit.id)],
self.qnameId[measure],
1 if (not unit.measures[1]) else (i + 1))
for unit in self.modelXbrl.units.values()
for i in range(2)
for measure in unit.measures[i]))
#table = self.getTable('enumeration_measure_location', 'enumeration_measure_location_id',
# ('description',),
# ('description',),
# (('measure',), ('numerator',), ('denominator',)))
# context
table = self.getTable('context', 'context_id',
('accession_id', 'period_start', 'period_end', 'period_instant', 'specifies_dimensions', 'context_xml_id', 'entity_scheme', 'entity_identifier'),
('accession_id', 'context_xml_id'),
tuple((accsId,
cntx.startDatetime if cntx.isStartEndPeriod else None,
cntx.endDatetime if cntx.isStartEndPeriod else None,
cntx.instantDatetime if cntx.isInstantPeriod else None,
bool(cntx.qnameDims),
cntx.id,
cntx.entityIdentifier[0],
cntx.entityIdentifier[1])
for cntx in self.modelXbrl.contexts.values()))
self.cntxId = dict(((_accsId, xmlId), id)
for id, _accsId, xmlId in table)
# context_dimension
values = []
for cntx in self.modelXbrl.contexts.values():
for dim in cntx.qnameDims.values():
values.append((self.cntxId[(accsId,cntx.id)],
self.qnameId[dim.dimensionQname],
self.qnameId.get(dim.memberQname), # may be None
self.qnameId.get(dim.typedMember.qname) if dim.isTyped else None,
False, # not default
dim.contextElement == "segment",
dim.typedMember.stringValue if dim.isTyped else None))
for dimQname, memQname in self.modelXbrl.qnameDimensionDefaults.items():
if dimQname not in cntx.qnameDims:
values.append((self.cntxId[(accsId,cntx.id)],
self.qnameId[dimQname],
self.qnameId[memQname],
None,
True, # is default
True, # ambiguous and irrelevant for the XDT model
None))
table = self.getTable('context_dimension', 'context_dimension_id',
('context_id', 'dimension_qname_id', 'member_qname_id', 'typed_qname_id', 'is_default', 'is_segment', 'typed_text_content'),
('dimension_qname_id',),
values)
# facts
table = self.getTable('fact', 'fact_id',
('accession_id', 'context_id', 'unit_id', 'element_id', 'effective_value', 'fact_value',
'xml_id', 'precision_value', 'decimals_value',
'is_precision_infinity', 'is_decimals_infinity', ),
('accession_id', 'context_id', 'unit_id', 'element_id', 'fact_value'),
tuple((accsId,
self.cntxId.get((accsId,fact.contextID)),
self.unitId.get((accsId,fact.unitID)),
self.conceptElementId(fact.concept),
roundValue(fact.value, fact.precision, fact.decimals) if fact.isNumeric else None,
fact.value,
elementFragmentIdentifier(fact),
fact.xAttributes['precision'].xValue if ('precision' in fact.xAttributes and isinstance(fact.xAttributes['precision'].xValue,int)) else None,
fact.xAttributes['decimals'].xValue if ('decimals' in fact.xAttributes and isinstance(fact.xAttributes['decimals'].xValue,int)) else None,
'precision' in fact.xAttributes and fact.xAttributes['precision'].xValue == 'INF',
'decimals' in fact.xAttributes and fact.xAttributes['decimals'].xValue == 'INF',
)
for fact in self.modelXbrl.facts))
def insertRelationships(self):
self.showStatus("insert relationship sets")
table = self.getTable('relationship_set', 'relationship_set_id',
('report_id', 'link_role', 'arc_role', 'link_qname', 'arc_qname'),
('report_id', 'link_role', 'arc_role', 'link_qname', 'arc_qname'),
tuple((self.reportId,
ELR,
arcrole,
linkqname.clarkNotation,
arcqname.clarkNotation)
for arcrole, ELR, linkqname, arcqname in self.modelXbrl.baseSets.keys()
if ELR and linkqname and arcqname and not arcrole.startswith("XBRL-")))
self.relSetId = dict(((linkRole, arcRole, lnkQn, arcQn), id)
for id, reportId, linkRole, arcRole, lnkQn, arcQn in table)
# do tree walk to build relationships with depth annotated, no targetRole navigation
dbRels = []
def walkTree(rels, seq, depth, relationshipSet, visited, dbRels, relSetId):
for rel in rels:
if rel not in visited and rel.toModelObject is not None:
visited.add(rel)
dbRels.append((rel, seq, depth, relSetId))
seq += 1
seq = walkTree(relationshipSet.fromModelObject(rel.toModelObject), seq, depth+1, relationshipSet, visited, dbRels, relSetId)
visited.remove(rel)
return seq
for arcrole, ELR, linkqname, arcqname in self.modelXbrl.baseSets.keys():
if ELR and linkqname and arcqname and not arcrole.startswith("XBRL-"):
relSetId = self.relSetId[(ELR,
arcrole,
linkqname.clarkNotation,
arcqname.clarkNotation)]
relationshipSet = self.modelXbrl.relationshipSet(arcrole, ELR, linkqname, arcqname)
seq = 1
for rootConcept in relationshipSet.rootConcepts:
seq = walkTree(relationshipSet.fromModelObject(rootConcept), seq, 1, relationshipSet, set(), dbRels, relSetId)
def resourceResourceId(resource):
if isinstance(resource, ModelResource):
return self.resourceId.get((self.documentIds[resource.modelDocument],
resource.sourceline,
resource.objectIndex))
else:
return None
table = self.getTable('relationship', 'relationship_id',
('report_id', 'document_id', 'xml_id',
'relationship_set_id', 'reln_order',
'from_id', 'to_id', 'calculation_weight',
'tree_sequence', 'tree_depth', 'preferred_label_role'),
('relationship_set_id', 'document_id', 'xml_id'),
tuple((self.reportId,
self.documentIds[rel.modelDocument],
elementFragmentIdentifier(rel.arcElement),
relSetId,
self.dbNum(rel.order),
self.modelObjectId(rel.fromModelObject),
self.modelObjectId(rel.toModelObject),
self.dbNum(rel.weight), # none if no weight
sequence,
depth,
rel.preferredLabel)
for rel, sequence, depth, relSetId in dbRels
if rel.fromModelObject is not None and rel.toModelObject is not None))
self.relationshipId = dict(((docId,xmlId), relationshipId)
for relationshipId, relSetId, docId, xmlId in table)
del dbRels[:] # dererefence
def insertAspects(self):
self.showStatus("insert aspects")
# determine new filing documents and types they use
filingDocumentAspects = set()
existingDocumentUsedAspects = set()
for concept in self.modelXbrl.qnameConcepts.values():
if concept.modelDocument not in self.existingDocumentIds:
filingDocumentAspects.add(concept)
filingDocumentAspectType = concept.type
if filingDocumentAspectType is not None and filingDocumentAspectType not in self.typesUsed:
self.typesUsed.add(filingDocumentAspectType)
elif concept in self.aspectsUsed:
existingDocumentUsedAspects.add(concept)
filingDocumentTypes = set()
existingDocumentUsedTypes = set()
for modelType in self.modelXbrl.qnameTypes.values():
if modelType.modelDocument not in self.existingDocumentIds:
filingDocumentTypes.add(modelType)
elif modelType in self.typesUsed:
existingDocumentUsedTypes.add(modelType)
# get existing element IDs
self.typeQnameId = {}
if existingDocumentUsedTypes:
typeQnameIds = []
table = self.getTable('data_type', 'data_type_id',
('document_id', 'qname',),
('document_id', 'qname',),
tuple((self.documentIds[modelType.modelDocument],
modelType.qname.clarkNotation)
for modelType in existingDocumentUsedTypes
if modelType.modelDocument in self.documentIds),
checkIfExisting=True,
insertIfNotMatched=False)
for typeId, docId, qn in table:
self.typeQnameId[qname(qn)] = typeId
table = self.getTable('data_type', 'data_type_id',
('document_id', 'xml_id',
'qname', 'name', 'base_type', 'derived_from_type_id'),
('document_id', 'qname',),
tuple((self.documentIds[modelType.modelDocument],
elementFragmentIdentifier(modelType),
modelType.qname.clarkNotation,
modelType.name,
modelType.baseXsdType,
self.typeQnameId.get(modelType.typeDerivedFrom)
if isinstance(modelType.typeDerivedFrom, ModelType) else None)
for modelType in filingDocumentTypes
if modelType.modelDocument in self.documentIds)
)
for typeId, docId, qn in table:
self.typeQnameId[qname(qn)] = typeId
updatesToDerivedFrom = set()
for modelType in filingDocumentTypes:
if isinstance(modelType.typeDerivedFrom, ModelType) and modelType.typeDerivedFrom in filingDocumentTypes:
updatesToDerivedFrom.add( (self.typeQnameId[modelType.qname],
self.typeQnameId[modelType.typeDerivedFrom.qname]) )
# update derivedFrom's of newly added types
if updatesToDerivedFrom:
self.updateTable('data_type',
('data_type_id', 'derived_from_type_id'),
updatesToDerivedFrom)
existingDocumentUsedTypes.clear() # dereference
filingDocumentTypes.clear() # dereference
self.aspectQnameId = {}
# get existing element IDs
if existingDocumentUsedAspects:
table = self.getTable('aspect', 'aspect_id',
('document_id', 'qname',),
('document_id', 'qname',),
tuple((self.documentIds[concept.modelDocument],
concept.qname.clarkNotation)
for concept in existingDocumentUsedAspects
if concept.modelDocument in self.documentIds),
checkIfExisting=True,
insertIfNotMatched=False)
for aspectId, docId, qn in table:
self.aspectQnameId[qname(qn)] = aspectId
table = self.getTable('aspect', 'aspect_id',
('document_id', 'xml_id',
'qname', 'name', 'datatype_id', 'base_type', 'substitution_group_aspect_id',
'balance', 'period_type', 'abstract', 'nillable',
'is_numeric', 'is_monetary', 'is_text_block'),
('document_id', 'qname'),
tuple((self.documentIds[concept.modelDocument],
elementFragmentIdentifier(concept),
concept.qname.clarkNotation,
concept.name,
self.typeQnameId.get(concept.typeQname),
concept.niceType,
self.aspectQnameId.get(concept.substitutionGroupQname),
concept.balance,
concept.periodType,
concept.isAbstract,
concept.isNillable,
concept.isNumeric,
concept.isMonetary,
concept.isTextBlock)
for concept in filingDocumentAspects
if concept.modelDocument in self.documentIds)
)
for aspectId, docId, qn in table:
self.aspectQnameId[qname(qn)] = aspectId
updatesToSubstitutionGroup = set()
for concept in filingDocumentAspects:
if concept.substitutionGroup in filingDocumentAspects and concept.modelDocument in self.documentIds:
updatesToSubstitutionGroup.add( (self.aspectQnameId[concept.qname],
self.aspectQnameId.get(concept.substitutionGroupQname)) )
# update derivedFrom's of newly added types
if updatesToSubstitutionGroup:
self.updateTable('aspect',
('aspect_id', 'substitution_group_aspect_id'),
updatesToSubstitutionGroup)
filingDocumentAspects.clear() # dereference
existingDocumentUsedAspects.clear() # dereference
