def factAspects(fact):
aspects = {qnOimConceptAspect: oimValue(fact.qname)}
if hasId and fact.id:
aspects[qnOimIdAspect] = fact.id
if hasLocation:
aspects[qnOimLocationAspect] = elementChildSequence(fact)
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString",
"token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
aspects[qnOimTypeAspect] = concept.baseXbrliType
if fact.isItem:
aspects[qnOimValueAspect] = (NILVALUE if fact.isNil else oimValue(
fact.xValue, inferredDecimals(fact)))
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(
qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
for dim in cntx.qnameDims.values():
aspects[dim.dimensionQname] = (oimValue(dim.memberQname)
if dim.isExplicit else
dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
if isJSON:
aspects[
qnOimUnitAspect] = ( # use tuple instead of list for hashability
tuple(
oimValue(m)
for m in sorted(_mMul, key=lambda m: str(m))),
tuple(
oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m))))
else: # CSV
if _mMul:
aspects[qnOimUnitMulAspect] = ",".join(
oimValue(m)
for m in sorted(_mMul, key=lambda m: str(m)))
if _mDiv:
aspects[qnOimUnitDivAspect] = ",".join(
oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m)))
return aspects
def factAspects(fact):
aspects = {qnOimConceptAspect: oimValue(fact.qname)}
if hasId and fact.id:
aspects[qnOimIdAspect] = fact.id
if hasLocation:
aspects[qnOimLocationAspect] = elementChildSequence(fact)
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
aspects[qnOimTypeAspect] = concept.baseXbrliType
if fact.isItem:
aspects[qnOimValueAspect] = (NILVALUE if fact.isNil else
oimValue(fact.xValue, inferredDecimals(fact)))
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
for dim in cntx.qnameDims.values():
aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit
else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
if isJSON:
aspects[qnOimUnitAspect] = ( # use tuple instead of list for hashability
tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))),
tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m))))
else: # CSV
if _mMul:
aspects[qnOimUnitMulAspect] = ",".join(oimValue(m)
for m in sorted(_mMul, key=lambda m: str(m)))
if _mDiv:
aspects[qnOimUnitDivAspect] = ",".join(oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m)))
return aspects
def saveTargetDocument(modelXbrl,
targetDocumentFilename,
targetDocumentSchemaRefs,
outputZip=None,
filingFiles=None,
*args,
**kwargs):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
targetDocumentFilename, modelXbrl.modelDocument.filepath)
def addLocallyReferencedFile(elt, filingFiles):
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src") and not isHttpUrl(
attrValue) and not os.path.isabs(attrValue):
attrValue = attrValue.partition('#')[0] # remove anchor
if attrValue: # ignore anchor references to base document
attrValue = os.path.normpath(
attrValue
) # change url path separators to host separators
file = os.path.join(sourceDir, attrValue)
if modelXbrl.fileSource.isInArchive(
file,
checkExistence=True) or os.path.exists(file):
filingFiles.add(file)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(
_("Extracting instance ") + os.path.basename(targetUrl))
rootElt = modelXbrl.modelDocument.xmlRootElement
# take baseXmlLang from <html> or <base>
baseXmlLang = rootElt.get(
"{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang")
for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants(
tag="{http://www.w3.org/1999/xhtml}body"):
baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang"
) or rootElt.get("lang") or baseXmlLang
targetInstance = ModelXbrl.create(
modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True,
discover=False) # don't attempt to load DTS
if baseXmlLang:
targetInstance.modelDocument.xmlRootElement.set(
"{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
ValidateXbrlDimensions.loadDimensionDefaults(
targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
addChild(targetInstance.modelDocument.xmlRootElement,
roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in sorted(modelXbrl.contexts.values(),
key=lambda c: elementChildSequence(c)):
ignore = targetInstance.createContext(
context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
ignore = targetInstance.createUnit(measures[0],
measures[1],
id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
if fact.concept is not None and fact.concept.baseXsdType in (
"string", "normalizedString"): # default
xmlLang = fact.xmlLang
if xmlLang is not None and xmlLang != baseXmlLang:
attrs[
"{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang
newFact = targetInstance.createFact(fact.qname,
attributes=attrs,
text=text,
parent=parent)
# if fact.isFraction, create numerator and denominator
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references so that referenced files are included in the zip.
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML("<body>\n{0}\n</body>\n".format(
xmltext)).iter():
addLocallyReferencedFile(elt, filingFiles)
except (XMLSyntaxError, UnicodeDecodeError):
pass # TODO: Why ignore UnicodeDecodeError?
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
modelXbrl.modelManager.showStatus(
_("Creating and validating footnotes and relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
footnoteIdCount = {}
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and any(
lP.modelDocument.type == Type.INLINEXBRL
for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype):
if HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ArcPrototype):
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID,
0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(
attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
newChild = addChild(newLink,
linkChild.qname,
attributes=attributes)
xmlLang = linkChild.xmlLang
if xmlLang is not None and xmlLang != baseXmlLang: # default
newChild.set(
"{http://www.w3.org/XML/1998/namespace}lang",
xmlLang)
copyIxFootnoteHtml(
linkChild,
newChild,
targetModelDocument=targetInstance.modelDocument,
withText=True)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyIxFootnoteHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
addLocallyReferencedFile(elt, filingFiles)
targetInstance.saveInstance(overrideFilepath=targetUrl,
outputZip=outputZip)
if getattr(modelXbrl, "isTestcaseVariation", False):
modelXbrl.extractedInlineInstance = True # for validation comparison
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def saveLoadableOIM(modelXbrl, oimFile, oimStyle, oimQNameSeparator):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
namespacePrefixes = {}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {
qnOimConceptAspect,
qnOimLocationAspect,
qnOimValueAspect,
qnOimPeriodAspect,
qnOimEntityAspect}
def oimValue(object, decimals=None):
if isinstance(object, QName):
if oimQNameSeparator == "clark":
return object.clarkNotation;
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}{}{}".format(namespacePrefixes[object.namespaceURI],
oimQNameSeparator,
object.localName)
if isinstance(object, Decimal):
try:
if decimals is not None and not isnan(decimals) and not isinf(decimals):
if decimals != 0:
object = object / (TEN ** -decimals)
return "{}e{}".format(object, -decimals)
else:
return "{}".format(object) # force to string to prevent json floating error
except:
return str(object)
if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
return "forever"
elif cntx.isStartEndPeriod:
return "{}/{}".format(dateunionValue(cntx.startDatetime, dateOnlyHour=0),
dateunionValue(cntx.endDatetime, subtractOneDay=True, dateOnlyHour=24))
else: # instant
return "PT0S/{}".format(dateunionValue(cntx.endDatetime, subtractOneDay=True, dateOnlyHour=24))
hasId = False
hasLocation = False # may be optional based on style?
hasType = True
hasLang = False
hasUnits = False
hasUnitMulMeasures = False
hasUnitDivMeasures = False
hasTuple = False
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if fact.id:
hasId = True
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
hasLang = True
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if unit.measures[0]:
hasUnitMulMeasures = True
if unit.measures[1]:
hasUnitDivMeasures = True
if fact.modelTupleFacts:
hasTuple = True
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme-{}".format(len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
if hasId: aspectsDefined.add(qnOimIdAspect)
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple: aspectsDefined.add(qnOimTupleAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
if hasUnitMulMeasures: aspectsDefined.add(qnOimUnitMulAspect)
if hasUnitDivMeasures: aspectsDefined.add(qnOimUnitDivAspect)
# compile footnotes and relationships
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.fromModelObjects]
oimRel["toIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.toModelObjects]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {"role": obj.role,
"id": obj.id if obj.id
else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
dtsReferences = [
{"type": "schema" if doc.type == ModelDocument.Type.SCHEMA
else "linkbase" if doc.type == ModelDocument.Type.LINKBASE
else "other",
"href": doc.basename}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in SCHEMA_LB_REFS]
roleTypes = [
{"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef",
"href": ref.referringModelObject["href"]}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in ROLE_REFS]
def factAspects(fact):
aspects = {qnOimConceptAspect: oimValue(fact.qname)}
if hasId and fact.id:
aspects[qnOimIdAspect] = fact.id
if hasLocation:
aspects[qnOimLocationAspect] = elementChildSequence(fact)
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
aspects[qnOimTypeAspect] = concept.baseXbrliType
if fact.isItem:
aspects[qnOimValueAspect] = (NILVALUE if fact.isNil else
oimValue(fact.xValue, inferredDecimals(fact)))
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
for dim in cntx.qnameDims.values():
aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit
else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
if isJSON:
aspects[qnOimUnitAspect] = ( # use tuple instead of list for hashability
tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))),
tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m))))
else: # CSV
if _mMul:
aspects[qnOimUnitMulAspect] = ",".join(oimValue(m)
for m in sorted(_mMul, key=lambda m: str(m)))
if _mDiv:
aspects[qnOimUnitDivAspect] = ",".join(oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m)))
return aspects
if isJSON:
# save JSON
oim = {} # top level of oim json output
oimFacts = []
oimReport = []
oimReport.append({"url": modelXbrl.modelDocument.uri})
if oimQNameSeparator != "clark":
oimReport.append({"prefixMap": dict((p,ns) for ns,p in namespacePrefixes.items())})
oimReport.append({"DTSreferences": dtsReferences})
oimReport.append({"roleTypes": roleTypes})
oimReport.append({"facts": oimFacts})
oimReport.append({"footnotes": factFootnotes})
oimReport.append({"relationships": factRelationships})
if oimStyle == "flat":
def saveFlatJsonFacts(facts, oimFacts):
for fact in facts:
oimFact = factAspects(fact)
if fact.modelTupleFacts:
tupleFacts = []
oimFact[qnOimTupleAspect] = tupleFacts
saveFlatJsonFacts(fact.modelTupleFacts, tupleFacts)
oimFacts.append(dict((oimValue(k),v) for k,v in oimFact.items()))
saveFlatJsonFacts(modelXbrl.facts, oimFacts)
elif oimStyle == "clustered":
# build aspect-value usage per fact for every fact
categoricalAspectValueSets = {} # for each aspect, value facts-set
aspectIndex = {}
indexAspect = {}
def addCategoricalAspect(aspectQn):
i = len(aspectIndex)
aspectIndex[aspectQn] = i
indexAspect[i] = oimValue(aspectQn)
categoricalAspectValueSets[i] = defaultdict(set)
addCategoricalAspect(qnOimConceptAspect)
addCategoricalAspect(qnOimEntityAspect)
addCategoricalAspect(qnOimPeriodAspect)
for aspectQn in aspectsDefined:
if aspectQn.namespaceURI != nsOim or aspectQn in (
qnOimIdAspect, qnOimLangAspect, qnOimUnitAspect):
addCategoricalAspect(aspectQn)
for fact in modelXbrl.facts:
fact._factAspectValues = {}
fact._factAspectSet = set()
for aspectQn, value in factAspects(fact).items():
if aspectQn in aspectIndex:
i = aspectIndex[aspectQn]
v = oimValue(value)
categoricalAspectValueSets[i][v].add(fact)
fact._factAspectValues[i] = v
fact._factAspectSet.add(i)
# order aspectValues by largest population
maxAspectValuePopulation = [(aspectIndex, max(len(factSet) for factSet in oimValueFacts.values()))
for aspectIndex, oimValueFacts in categoricalAspectValueSets.items()]
maxAspectValuePopulation.sort(key=lambda ai_max: -ai_max[1])
factsClustered = set()
_aspectValue = {}
def clusterAspect(_avpi, _data):
if _avpi >= len(maxAspectValuePopulation):
return # end of aspects
_ai = maxAspectValuePopulation[_avpi][0]
for _v, _vFactsSet in categoricalAspectValueSets[_ai].items():
_aspectValue[_ai] = _v
_nestedData = []
_nestedAspect = {indexAspect[_ai]: _v, "data": _nestedData}
for _fact in _vFactsSet - factsClustered:
if (_fact._factAspectSet == _aspectValue.keys() and
all([_fact._factAspectValues[__ai] == _aspectValue[__ai]
for __ai in _aspectValue])):
_factAspects = factAspects(_fact)
_oimFactItem = {oimValue(qnOimValueAspect): _factAspects[qnOimValueAspect]}
if hasLocation:
_oimFactItem[oimValue(qnOimLocationAspect)] = _factAspects[qnOimLocationAspect]
if hasType:
_oimFactItem[oimValue(qnOimTypeAspect)] = _factAspects[qnOimTypeAspect]
_nestedData.append(_oimFactItem)
factsClustered.add(_fact)
clusterAspect(_avpi+1, _nestedData)
if _nestedData:
_data.append(_nestedAspect)
del _aspectValue[_ai]
clusterAspect(0, oimFacts)
with open(oimFile, "w", encoding="utf-8") as fh:
fh.write(json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=True))
elif isCSV:
# save CSV
# levels of tuple nesting
def tupleDepth(facts, parentDepth):
_levelDepth = parentDepth
for fact in facts:
_factDepth = tupleDepth(fact.modelTupleFacts, parentDepth + 1)
if _factDepth > _levelDepth:
_levelDepth = _factDepth
return _levelDepth
maxDepth = tupleDepth(modelXbrl.facts, 0)
aspectQnCol = {oimValue(qnOimConceptAspect): maxDepth - 1}
aspectsHeader = [oimValue(qnOimConceptAspect)]
for i in range(maxDepth - 1):
aspectsHeader.append(None)
def addAspectQnCol(aspectQn):
aspectQnCol[aspectQn] = len(aspectsHeader)
aspectsHeader.append(oimValue(aspectQn))
# pre-ordered aspect columns
if hasId:
addAspectQnCol(qnOimIdAspect)
if hasLocation:
addAspectQnCol(qnOimLocationAspect)
if hasType:
addAspectQnCol(qnOimTypeAspect)
addAspectQnCol(qnOimValueAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodAspect)
if qnOimUnitMulAspect in aspectsDefined:
addAspectQnCol(qnOimUnitMulAspect)
if qnOimUnitDivAspect in aspectsDefined:
addAspectQnCol(qnOimUnitDivAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact, depth):
cols = [None for i in range(len(aspectsHeader))]
for aspectQn, aspectValue in factAspects(fact).items():
if aspectQn == qnOimConceptAspect:
cols[depth - 1] = aspectValue
elif aspectQn in aspectQnCol:
cols[aspectQnCol[aspectQn]] = aspectValue
return cols
# save facts
csvFile = open(oimFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(aspectsHeader)
def saveCSVfacts(facts, thisDepth):
for fact in facts:
csvWriter.writerow(aspectCols(fact, thisDepth))
saveCSVfacts(fact.modelTupleFacts, thisDepth + 1)
saveCSVfacts(modelXbrl.facts, 1)
csvFile.close()
# save namespaces
if oimQNameSeparator == "clark":
csvFile = open(oimFile.replace(".csv", "-prefixMap.csv"), csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("prefix", "mappedURI"))
for namespaceURI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]):
csvWriter.writerow((prefix, namespaceURI))
csvFile.close()
# save dts references
csvFile = open(oimFile.replace(".csv", "-dts.csv"), csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("type", "href"))
for oimRef in dtsReferences:
csvWriter.writerow((oimRef["type"], oimRef["href"]))
csvFile.close()
# save role and arc type references
if roleTypes:
csvFile = open(oimFile.replace(".csv", "-roleTypes.csv"), csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("type", "href"))
for oimRef in roleTypes:
csvWriter.writerow((oimRef["type"], oimRef["href"]))
csvFile.close()
# save relationships
csvFile = open(oimFile.replace(".csv", "-relationships.csv"), csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
hasOrder = any(hasattribute(imRel,"order") for oimRel in factRelationships)
csvWriter.writerow(("fromIds", "toIds", "linkrole", "arcrole") +
(("order",) if hasOrder else ()))
for oimRel in factRelationships:
csvWriter.writerow((",".join(oimRel["fromIds"]),
",".join(oimRel["toIds"]),
oimRel["linkrole"],
oimRel["arcrole"]) +
((oimRel.get("order",None),) if hasOrder else ()))
csvFile.close()
# save footnotes
csvFile = open(oimFile.replace(".csv", "-footnotes.csv"), csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
hasLang = any(hasattribute(oimFnt,"lang") for oimFnt in factFootnotes)
csvWriter.writerow(("id", "role") + (("lang",) if hasLang else ()) + ("value",))
for oimFnt in factFootnotes:
csvWriter.writerow((oimFtn["id"], oimFtn["role"]) +
((oimFtn.get("lang",None),) if hasLang else ()) +
(oimFtn["value"],))
csvFile.close()
def saveLoadableOIM(modelXbrl, oimFile, oimStyle, oimQNameSeparator):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
namespacePrefixes = {}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {
qnOimConceptAspect, qnOimLocationAspect, qnOimValueAspect,
qnOimPeriodAspect, qnOimEntityAspect
}
def oimValue(object, decimals=None):
if isinstance(object, QName):
if oimQNameSeparator == "clark":
return object.clarkNotation
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(
sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}{}{}".format(namespacePrefixes[object.namespaceURI],
oimQNameSeparator, object.localName)
if isinstance(object, Decimal):
try:
if decimals is not None and not isnan(decimals) and not isinf(
decimals):
if decimals != 0:
object = object / (TEN**-decimals)
return "{}e{}".format(object, -decimals)
else:
return "{}".format(
object
) # force to string to prevent json floating error
except:
return str(object)
if isinstance(object,
(DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
return "forever"
elif cntx.isStartEndPeriod:
return "{}/{}".format(
dateunionValue(cntx.startDatetime, dateOnlyHour=0),
dateunionValue(cntx.endDatetime,
subtractOneDay=True,
dateOnlyHour=24))
else: # instant
return "PT0S/{}".format(
dateunionValue(cntx.endDatetime,
subtractOneDay=True,
dateOnlyHour=24))
hasId = False
hasLocation = False # may be optional based on style?
hasType = True
hasLang = False
hasUnits = False
hasUnitMulMeasures = False
hasUnitDivMeasures = False
hasTuple = False
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if fact.id:
hasId = True
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString",
"token") and fact.xmlLang:
hasLang = True
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if unit.measures[0]:
hasUnitMulMeasures = True
if unit.measures[1]:
hasUnitDivMeasures = True
if fact.modelTupleFacts:
hasTuple = True
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme-{}".format(
len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[
XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
if hasId: aspectsDefined.add(qnOimIdAspect)
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple: aspectsDefined.add(qnOimTupleAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
if hasUnitMulMeasures: aspectsDefined.add(qnOimUnitMulAspect)
if hasUnitDivMeasures: aspectsDefined.add(qnOimUnitDivAspect)
# compile footnotes and relationships
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl,
"XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [
obj.id if obj.id else elementChildSequence(obj)
for obj in rel.fromModelObjects
]
oimRel["toIds"] = [
obj.id if obj.id else elementChildSequence(obj)
for obj in rel.toModelObjects
]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {
"role": obj.role,
"id": obj.id if obj.id else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)
}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
dtsReferences = [{
"type":
"schema" if doc.type == ModelDocument.Type.SCHEMA else
"linkbase" if doc.type == ModelDocument.Type.LINKBASE else "other",
"href":
doc.basename
} for doc, ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in SCHEMA_LB_REFS]
roleTypes = [{
"type":
"role"
if ref.referringModelObject.localName == "roleRef" else "arcroleRef",
"href":
ref.referringModelObject["href"]
} for doc, ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in ROLE_REFS]
def factAspects(fact):
aspects = {qnOimConceptAspect: oimValue(fact.qname)}
if hasId and fact.id:
aspects[qnOimIdAspect] = fact.id
if hasLocation:
aspects[qnOimLocationAspect] = elementChildSequence(fact)
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString",
"token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
aspects[qnOimTypeAspect] = concept.baseXbrliType
if fact.isItem:
aspects[qnOimValueAspect] = (NILVALUE if fact.isNil else oimValue(
fact.xValue, inferredDecimals(fact)))
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(
qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
for dim in cntx.qnameDims.values():
aspects[dim.dimensionQname] = (oimValue(dim.memberQname)
if dim.isExplicit else
dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
if isJSON:
aspects[
qnOimUnitAspect] = ( # use tuple instead of list for hashability
tuple(
oimValue(m)
for m in sorted(_mMul, key=lambda m: str(m))),
tuple(
oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m))))
else: # CSV
if _mMul:
aspects[qnOimUnitMulAspect] = ",".join(
oimValue(m)
for m in sorted(_mMul, key=lambda m: str(m)))
if _mDiv:
aspects[qnOimUnitDivAspect] = ",".join(
oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m)))
return aspects
if isJSON:
# save JSON
oim = {} # top level of oim json output
oimFacts = []
oimReport = []
oimReport.append({"url": modelXbrl.modelDocument.uri})
if oimQNameSeparator != "clark":
oimReport.append({
"prefixMap":
dict((p, ns) for ns, p in namespacePrefixes.items())
})
oimReport.append({"DTSreferences": dtsReferences})
oimReport.append({"roleTypes": roleTypes})
oimReport.append({"facts": oimFacts})
oimReport.append({"footnotes": factFootnotes})
oimReport.append({"relationships": factRelationships})
if oimStyle == "flat":
def saveFlatJsonFacts(facts, oimFacts):
for fact in facts:
oimFact = factAspects(fact)
if fact.modelTupleFacts:
tupleFacts = []
oimFact[qnOimTupleAspect] = tupleFacts
saveFlatJsonFacts(fact.modelTupleFacts, tupleFacts)
oimFacts.append(
dict((oimValue(k), v) for k, v in oimFact.items()))
saveFlatJsonFacts(modelXbrl.facts, oimFacts)
elif oimStyle == "clustered":
# build aspect-value usage per fact for every fact
categoricalAspectValueSets = {} # for each aspect, value facts-set
aspectIndex = {}
indexAspect = {}
def addCategoricalAspect(aspectQn):
i = len(aspectIndex)
aspectIndex[aspectQn] = i
indexAspect[i] = oimValue(aspectQn)
categoricalAspectValueSets[i] = defaultdict(set)
addCategoricalAspect(qnOimConceptAspect)
addCategoricalAspect(qnOimEntityAspect)
addCategoricalAspect(qnOimPeriodAspect)
for aspectQn in aspectsDefined:
if aspectQn.namespaceURI != nsOim or aspectQn in (
qnOimIdAspect, qnOimLangAspect, qnOimUnitAspect):
addCategoricalAspect(aspectQn)
for fact in modelXbrl.facts:
fact._factAspectValues = {}
fact._factAspectSet = set()
for aspectQn, value in factAspects(fact).items():
if aspectQn in aspectIndex:
i = aspectIndex[aspectQn]
v = oimValue(value)
categoricalAspectValueSets[i][v].add(fact)
fact._factAspectValues[i] = v
fact._factAspectSet.add(i)
# order aspectValues by largest population
maxAspectValuePopulation = [
(aspectIndex,
max(len(factSet) for factSet in oimValueFacts.values()))
for aspectIndex, oimValueFacts in
categoricalAspectValueSets.items()
]
maxAspectValuePopulation.sort(key=lambda ai_max: -ai_max[1])
factsClustered = set()
_aspectValue = {}
def clusterAspect(_avpi, _data):
if _avpi >= len(maxAspectValuePopulation):
return # end of aspects
_ai = maxAspectValuePopulation[_avpi][0]
for _v, _vFactsSet in categoricalAspectValueSets[_ai].items():
_aspectValue[_ai] = _v
_nestedData = []
_nestedAspect = {indexAspect[_ai]: _v, "data": _nestedData}
for _fact in _vFactsSet - factsClustered:
if (_fact._factAspectSet == _aspectValue.keys()
and all([
_fact._factAspectValues[__ai]
== _aspectValue[__ai]
for __ai in _aspectValue
])):
_factAspects = factAspects(_fact)
_oimFactItem = {
oimValue(qnOimValueAspect):
_factAspects[qnOimValueAspect]
}
if hasLocation:
_oimFactItem[oimValue(
qnOimLocationAspect
)] = _factAspects[qnOimLocationAspect]
if hasType:
_oimFactItem[
oimValue(qnOimTypeAspect
)] = _factAspects[qnOimTypeAspect]
_nestedData.append(_oimFactItem)
factsClustered.add(_fact)
clusterAspect(_avpi + 1, _nestedData)
if _nestedData:
_data.append(_nestedAspect)
del _aspectValue[_ai]
clusterAspect(0, oimFacts)
with open(oimFile, "w", encoding="utf-8") as fh:
fh.write(
json.dumps(oimReport,
ensure_ascii=False,
indent=1,
sort_keys=True))
elif isCSV:
# save CSV
# levels of tuple nesting
def tupleDepth(facts, parentDepth):
_levelDepth = parentDepth
for fact in facts:
_factDepth = tupleDepth(fact.modelTupleFacts, parentDepth + 1)
if _factDepth > _levelDepth:
_levelDepth = _factDepth
return _levelDepth
maxDepth = tupleDepth(modelXbrl.facts, 0)
aspectQnCol = {oimValue(qnOimConceptAspect): maxDepth - 1}
aspectsHeader = [oimValue(qnOimConceptAspect)]
for i in range(maxDepth - 1):
aspectsHeader.append(None)
def addAspectQnCol(aspectQn):
aspectQnCol[aspectQn] = len(aspectsHeader)
aspectsHeader.append(oimValue(aspectQn))
# pre-ordered aspect columns
if hasId:
addAspectQnCol(qnOimIdAspect)
if hasLocation:
addAspectQnCol(qnOimLocationAspect)
if hasType:
addAspectQnCol(qnOimTypeAspect)
addAspectQnCol(qnOimValueAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodAspect)
if qnOimUnitMulAspect in aspectsDefined:
addAspectQnCol(qnOimUnitMulAspect)
if qnOimUnitDivAspect in aspectsDefined:
addAspectQnCol(qnOimUnitDivAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact, depth):
cols = [None for i in range(len(aspectsHeader))]
for aspectQn, aspectValue in factAspects(fact).items():
if aspectQn == qnOimConceptAspect:
cols[depth - 1] = aspectValue
elif aspectQn in aspectQnCol:
cols[aspectQnCol[aspectQn]] = aspectValue
return cols
# save facts
csvFile = open(oimFile,
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(aspectsHeader)
def saveCSVfacts(facts, thisDepth):
for fact in facts:
csvWriter.writerow(aspectCols(fact, thisDepth))
saveCSVfacts(fact.modelTupleFacts, thisDepth + 1)
saveCSVfacts(modelXbrl.facts, 1)
csvFile.close()
# save namespaces
if oimQNameSeparator == "clark":
csvFile = open(oimFile.replace(".csv", "-prefixMap.csv"),
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("prefix", "mappedURI"))
for namespaceURI, prefix in sorted(namespacePrefixes.items(),
key=lambda item: item[1]):
csvWriter.writerow((prefix, namespaceURI))
csvFile.close()
# save dts references
csvFile = open(oimFile.replace(".csv", "-dts.csv"),
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("type", "href"))
for oimRef in dtsReferences:
csvWriter.writerow((oimRef["type"], oimRef["href"]))
csvFile.close()
# save role and arc type references
if roleTypes:
csvFile = open(oimFile.replace(".csv", "-roleTypes.csv"),
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("type", "href"))
for oimRef in roleTypes:
csvWriter.writerow((oimRef["type"], oimRef["href"]))
csvFile.close()
# save relationships
csvFile = open(oimFile.replace(".csv", "-relationships.csv"),
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
hasOrder = any(
hasattribute(imRel, "order") for oimRel in factRelationships)
csvWriter.writerow(("fromIds", "toIds", "linkrole", "arcrole") +
(("order", ) if hasOrder else ()))
for oimRel in factRelationships:
csvWriter.writerow(
(",".join(oimRel["fromIds"]), ",".join(oimRel["toIds"]),
oimRel["linkrole"], oimRel["arcrole"]) +
((oimRel.get("order", None), ) if hasOrder else ()))
csvFile.close()
# save footnotes
csvFile = open(oimFile.replace(".csv", "-footnotes.csv"),
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
hasLang = any(hasattribute(oimFnt, "lang") for oimFnt in factFootnotes)
csvWriter.writerow(("id", "role") + (("lang", ) if hasLang else ()) +
("value", ))
for oimFnt in factFootnotes:
csvWriter.writerow((oimFtn["id"], oimFtn["role"]) + (
(oimFtn.get("lang", None), ) if hasLang else ()) +
(oimFtn["value"], ))
csvFile.close()
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None, *args, **kwargs):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
def addLocallyReferencedFile(elt,filingFiles):
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrValue):
attrValue = attrValue.partition('#')[0] # remove anchor
if attrValue: # ignore anchor references to base document
attrValue = os.path.normpath(attrValue) # change url path separators to host separators
file = os.path.join(sourceDir,attrValue)
if modelXbrl.fileSource.isInArchive(file, checkExistence=True) or os.path.exists(file):
filingFiles.add(file)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
rootElt = modelXbrl.modelDocument.xmlRootElement
# take baseXmlLang from <html> or <base>
baseXmlLang = rootElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang")
for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/1999/xhtml}body"):
baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang") or baseXmlLang
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True,
discover=False) # don't attempt to load DTS
if baseXmlLang:
targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in sorted(modelXbrl.contexts.values(), key=lambda c: elementChildSequence(c)):
ignore = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default
xmlLang = fact.xmlLang
if xmlLang is not None and xmlLang != baseXmlLang:
attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
# if fact.isFraction, create numerator and denominator
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references so that referenced files are included in the zip.
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter():
addLocallyReferencedFile(elt, filingFiles)
except (XMLSyntaxError, UnicodeDecodeError):
pass # TODO: Why ignore UnicodeDecodeError?
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
footnoteIdCount = {}
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and
any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype):
if HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
addChild(newLink, linkChild.qname,
attributes=attributes)
elif isinstance(linkChild, ArcPrototype):
addChild(newLink, linkChild.qname, attributes=attributes)
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
newChild = addChild(newLink, linkChild.qname,
attributes=attributes)
xmlLang = linkChild.xmlLang
if xmlLang is not None and xmlLang != baseXmlLang: # default
newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang)
copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyIxFootnoteHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
addLocallyReferencedFile(elt,filingFiles)
targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip)
if getattr(modelXbrl, "isTestcaseVariation", False):
modelXbrl.extractedInlineInstance = True # for validation comparison
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)