def lbTreeWalk(lbType, parentElt, lbList, roleRefs, locs=None, fromPrefix=None, fromName=None):
order = 1.0
for toPrefix, toName, rel, list in lbList:
if rel == "_ELR_":
role = "unspecified"
if toPrefix and toPrefix.startswith("http://"): # have a role specified
role = toPrefix
elif toName: #may be a definition
for linkroleUri, modelRoleTypes in dts.roleTypes.items():
definition = modelRoleTypes[0].definition
if toName == definition:
role = linkroleUri
break
if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef
roleType = modelRoleTypes[0]
roleRef = ("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id)
roleRefs.add(roleRef)
linkElt = XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Link",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", role)))
locs = set()
lbTreeWalk(lbType, linkElt, list, roleRefs, locs)
else:
toHref = extensionHref(toPrefix, toName)
toLabel = toPrefix + "_" + toName
if toHref not in locs:
XmlUtil.addChild(parentElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", toHref),
("{http://www.w3.org/1999/xlink}label", toLabel)))
locs.add(toHref)
if rel != "_root_":
fromLabel = fromPrefix + "_" + fromName
if lbType == "calculation":
otherAttrs = ( ("weight", list), )
else:
otherAttrs = ( )
if rel == "_dimensions_": # pick proper consecutive arcrole
fromConcept = hrefConcept(fromPrefix, fromName)
toConcept = hrefConcept(toPrefix, toName)
if toConcept is not None and toConcept.isHypercubeItem:
rel = XbrlConst.all
elif toConcept is not None and toConcept.isDimensionItem:
rel = XbrlConst.hypercubeDimension
elif fromConcept is not None and fromConcept.isDimensionItem:
rel = XbrlConst.dimensionDomain
else:
rel = XbrlConst.domainMember
XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Arc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", rel),
("{http://www.w3.org/1999/xlink}from", fromLabel),
("{http://www.w3.org/1999/xlink}to", toLabel),
("order", order)) + otherAttrs )
order += 1.0
if lbType != "calculation" or rel == "_root_":
lbTreeWalk(lbType, parentElt, list, roleRefs, locs, toPrefix, toName)
def createUnit(self, multiplyBy, divideBy, afterSibling=None, beforeSibling=None):
"""Creates new unit, by measures, as in formula usage, if any
:param multiplyBy: List of multiply-by measure QNames (or top level measures if no divideBy)
:type multiplyBy: [QName]
:param divideBy: List of multiply-by measure QNames (or empty list if no divideBy)
:type divideBy: [QName]
:param beforeSibling: lxml element in instance to insert new concept before
:type beforeSibling: ModelObject
:param afterSibling: lxml element in instance to insert new concept after
:type afterSibling: ModelObject
:returns: ModelUnit -- New unit object
"""
xbrlElt = self.modelDocument.xmlRootElement
if afterSibling == AUTO_LOCATE_ELEMENT:
afterSibling = XmlUtil.lastChild(xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context", "unit"))
unitId = 'u-{0:02n}'.format( len(self.units) + 1)
newUnitElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId),
afterSibling=afterSibling, beforeSibling=beforeSibling)
if len(divideBy) == 0:
for multiply in multiplyBy:
XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply))
else:
divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide")
numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator")
denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator")
for multiply in multiplyBy:
XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply))
for divide in divideBy:
XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, divide))
self.modelDocument.unitDiscover(newUnitElt)
XmlValidate.validate(self, newUnitElt)
return newUnitElt
def createFact(self, conceptQname, attributes=None, text=None, parent=None, afterSibling=None, beforeSibling=None):
if parent is None: parent = self.modelDocument.xmlRootElement
newFact = XmlUtil.addChild(parent, conceptQname, attributes=attributes, text=text,
afterSibling=afterSibling, beforeSibling=beforeSibling)
self.modelDocument.factDiscover(newFact, parentElement=parent)
XmlValidate.validate(self, newFact)
return newFact
def addLinkbaseRef(lbType, lbFilename, lbDoc):
role = "http://www.xbrl.org/2003/role/{0}LinkbaseRef".format(lbType)
lbRefElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "linkbaseRef",
attributes=(("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", lbFilename),
("{http://www.w3.org/1999/xlink}role", role),
("{http://www.w3.org/1999/xlink}arcrole", "http://www.w3.org/1999/xlink/properties/linkbase"),
))
dtsSchemaDocument.referencesDocument[lbDoc] = ModelDocumentReference("href", lbRefElt)
def createUnit(self,
multiplyBy,
divideBy,
afterSibling=None,
beforeSibling=None):
xbrlElt = self.modelDocument.xmlRootElement
if afterSibling == AUTO_LOCATE_ELEMENT:
afterSibling = XmlUtil.lastChild(
xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType",
"arcroleType", "context", "unit"))
unitId = 'u-{0:02n}'.format(len(self.units) + 1)
newUnitElt = XmlUtil.addChild(xbrlElt,
XbrlConst.xbrli,
"unit",
attributes=("id", unitId),
afterSibling=afterSibling,
beforeSibling=beforeSibling)
if len(divideBy) == 0:
for multiply in multiplyBy:
XmlUtil.addChild(newUnitElt,
XbrlConst.xbrli,
"measure",
text=XmlUtil.addQnameValue(xbrlElt, multiply))
else:
divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide")
numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator")
denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli,
"unitDenominator")
for multiply in multiplyBy:
XmlUtil.addChild(numElt,
XbrlConst.xbrli,
"measure",
text=XmlUtil.addQnameValue(xbrlElt, multiply))
for divide in divideBy:
XmlUtil.addChild(denElt,
XbrlConst.xbrli,
"measure",
text=XmlUtil.addQnameValue(xbrlElt, divide))
self.modelDocument.unitDiscover(newUnitElt)
XmlValidate.validate(self, newUnitElt)
return newUnitElt
def createUnit(self, multiplyBy, divideBy, afterSibling=None, beforeSibling=None):
xbrlElt = self.modelDocument.xmlRootElement
if afterSibling == AUTO_LOCATE_ELEMENT:
afterSibling = XmlUtil.lastChild(xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context", "unit"))
unitId = 'u-{0:02n}'.format( len(self.units) + 1)
newUnitElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId),
afterSibling=afterSibling, beforeSibling=beforeSibling)
if len(divideBy) == 0:
for multiply in multiplyBy:
XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply))
else:
divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide")
numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator")
denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator")
for multiply in multiplyBy:
XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply))
for divide in divideBy:
XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, divide))
self.modelDocument.unitDiscover(newUnitElt)
XmlValidate.validate(self, newUnitElt)
return newUnitElt
def createFact(self,
conceptQname,
attributes=None,
text=None,
parent=None,
afterSibling=None,
beforeSibling=None):
if parent is None: parent = self.modelDocument.xmlRootElement
newFact = XmlUtil.addChild(parent,
conceptQname,
attributes=attributes,
text=text,
afterSibling=afterSibling,
beforeSibling=beforeSibling)
self.modelDocument.factDiscover(newFact, parentElement=parent)
XmlValidate.validate(self, newFact)
return newFact
def createFact(self, conceptQname, attributes=None, text=None, parent=None, afterSibling=None, beforeSibling=None):
"""Creates new fact, as in formula output instance creation, and validates into object model
:param conceptQname: QNames of concept
:type conceptQname: QName
:param attributes: Tuple of name, value, or tuples of name, value tuples (name,value) or ((name,value)[,(name,value...)]), where name is either QName or clark-notation name string
:param text: Text content of fact
:type text: str
:param parent: lxml element in instance to append as child of
:type parent: ModelObject
:param beforeSibling: lxml element in instance to insert new concept before
:type beforeSibling: ModelObject
:param afterSibling: lxml element in instance to insert new concept after
:type afterSibling: ModelObject
:returns: ModelFact -- New fact object
"""
if parent is None: parent = self.modelDocument.xmlRootElement
newFact = XmlUtil.addChild(parent, conceptQname, attributes=attributes, text=text,
afterSibling=afterSibling, beforeSibling=beforeSibling)
self.modelDocument.factDiscover(newFact, parentElement=parent)
XmlValidate.validate(self, newFact)
return newFact
def lbTreeWalk(lbType, parentElt, lbStruct, roleRefs, locs=None, arcsFromTo=None, fromPrefix=None, fromName=None):
order = 1.0
for lbEntry in lbStruct:
if lbEntry.isELR:
role = "unspecified"
if lbEntry.role and lbEntry.role.startswith("http://"): # have a role specified
role = lbEntry.role
elif lbEntry.name: # may be a definition
for linkroleUri, modelRoleTypes in dts.roleTypes.items():
definition = modelRoleTypes[0].definition
if lbEntry.name == definition:
role = linkroleUri
break
if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef
roleType = modelRoleTypes[0]
roleRefs.add(("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id))
linkElt = XmlUtil.addChild(
parentElt,
XbrlConst.link,
lbType + "Link",
attributes=(
("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", role),
),
)
locs = set()
arcsFromTo = set()
lbTreeWalk(lbType, linkElt, lbEntry.childStruct, roleRefs, locs, arcsFromTo)
else:
toPrefix = lbEntry.prefix
toName = lbEntry.name
toHref = extensionHref(toPrefix, toName)
toLabel = toPrefix + "_" + toName
toLabelAlt = None
if not lbEntry.isRoot:
fromLabel = fromPrefix + "_" + fromName
if (fromLabel, toLabel) in arcsFromTo:
# need extra loc to prevent arc from/to duplication in ELR
for i in range(1, 1000):
toLabelAlt = "{}_{}".format(toLabel, i)
if (fromLabel, toLabelAlt) not in arcsFromTo:
toLabel = toLabelAlt
break
if toHref not in locs or toLabelAlt:
XmlUtil.addChild(
parentElt,
XbrlConst.link,
"loc",
attributes=(
("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", toHref),
("{http://www.w3.org/1999/xlink}label", toLabel),
),
)
locs.add(toHref)
if not lbEntry.isRoot:
arcsFromTo.add((fromLabel, toLabel))
if lbType == "calculation" and lbEntry.weight is not None:
otherAttrs = (("weight", lbEntry.weight),)
elif lbType == "presentation" and lbEntry.role:
otherAttrs = (("preferredLabel", lbEntry.role),)
if lbEntry.role and lbEntry.role in dts.roleTypes:
roleType = dts.roleTypes[lbEntry.role][0]
roleRefs.add(("roleRef", lbEntry.role, roleType.modelDocument.uri + "#" + roleType.id))
else:
otherAttrs = ()
if lbEntry.arcrole == "_dimensions_": # pick proper consecutive arcrole
fromConcept = hrefConcept(fromPrefix, fromName)
toConcept = hrefConcept(toPrefix, toName)
if toConcept is not None and toConcept.isHypercubeItem:
arcrole = XbrlConst.all
otherAttrs += ((XbrlConst.qnXbrldtContextElement, "segment"),)
elif toConcept is not None and toConcept.isDimensionItem:
arcrole = XbrlConst.hypercubeDimension
elif fromConcept is not None and fromConcept.isDimensionItem:
arcrole = XbrlConst.dimensionDomain
else:
arcrole = XbrlConst.domainMember
else:
arcrole = lbEntry.arcrole
XmlUtil.addChild(
parentElt,
XbrlConst.link,
lbType + "Arc",
attributes=(
("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", arcrole),
("{http://www.w3.org/1999/xlink}from", fromLabel),
("{http://www.w3.org/1999/xlink}to", toLabel),
("order", order),
)
+ otherAttrs,
)
order += 1.0
if lbType != "calculation" or lbEntry.isRoot:
lbTreeWalk(lbType, parentElt, lbEntry.childStruct, roleRefs, locs, arcsFromTo, toPrefix, toName)
def createContext(self, entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, priItem, dims, segOCCs, scenOCCs,
afterSibling=None, beforeSibling=None):
xbrlElt = self.modelDocument.xmlRootElement
if afterSibling == AUTO_LOCATE_ELEMENT:
afterSibling = XmlUtil.lastChild(xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context"))
cntxId = 'c-{0:02n}'.format( len(self.contexts) + 1)
newCntxElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId),
afterSibling=afterSibling, beforeSibling=beforeSibling)
entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity")
XmlUtil.addChild(entityElt, XbrlConst.xbrli, "identifier",
attributes=("scheme", entityIdentScheme),
text=entityIdentValue)
periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period")
if periodType == "forever":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever")
elif periodType == "instant":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "instant",
text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True))
elif periodType == "duration":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "startDate",
text=XmlUtil.dateunionValue(periodStart))
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "endDate",
text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True))
segmentElt = None
scenarioElt = None
from arelle.ModelInstanceObject import ModelDimensionValue
if dims: # requires primary item to determin ambiguous concepts
''' in theory we have to check full set of dimensions for validity in source or any other
context element, but for shortcut will see if each dimension is already reported in an
unambiguous valid contextElement
'''
from arelle.PrototypeInstanceObject import FactPrototype, ContextPrototype, DimValuePrototype
fp = FactPrototype(self, priItem, dims.items())
# force trying a valid prototype's context Elements
if not isFactDimensionallyValid(self, fp, setPrototypeContextElements=True):
self.info("arelleLinfo",
_("Create context for %(priItem)s, cannot determine valid context elements, no suitable hypercubes"),
modelObject=self, priItem=priItem)
fpDims = fp.context.qnameDims
for dimQname in sorted(fpDims.keys()):
dimValue = fpDims[dimQname]
if isinstance(dimValue, DimValuePrototype):
dimMemberQname = dimValue.memberQname # None if typed dimension
contextEltName = dimValue.contextElement
else: # qname for explicit or node for typed
dimMemberQname = None
contextEltName = None
if contextEltName == "segment":
if segmentElt is None:
segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment")
contextElt = segmentElt
elif contextEltName == "scenario":
if scenarioElt is None:
scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario")
contextElt = scenarioElt
else:
self.info("arelleLinfo",
_("Create context, %(dimension)s, cannot determine context element, either no all relationship or validation issue"),
modelObject=self, dimension=dimQname),
continue
dimConcept = self.qnameConcepts[dimQname]
dimAttr = ("dimension", XmlUtil.addQnameValue(xbrlElt, dimConcept.qname))
if dimConcept.isTypedDimension:
dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:typedMember",
attributes=dimAttr)
if isinstance(dimValue, (ModelDimensionValue, DimValuePrototype)) and dimValue.isTyped:
XmlUtil.copyNodes(dimElt, dimValue.typedMember)
elif dimMemberQname:
dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:explicitMember",
attributes=dimAttr,
text=XmlUtil.addQnameValue(xbrlElt, dimMemberQname))
if segOCCs:
if segmentElt is None:
segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment")
XmlUtil.copyNodes(segmentElt, segOCCs)
if scenOCCs:
if scenarioElt is None:
scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario")
XmlUtil.copyNodes(scenarioElt, scenOCCs)
self.modelDocument.contextDiscover(newCntxElt)
XmlValidate.validate(self, newCntxElt)
return newCntxElt
def genFact(dts, concept, preferredLabel, arcrole, relationshipSet, level, visited, elrInfo):
try:
if concept is not None:
if concept.isHypercubeItem:
elrInfo["inCube"] = level
elrInfo["dims"] = {}
elrInfo["lineItems"] =False
elrInfo.pop("instant", None)
elrInfo.pop("duration", None)
elif concept.isDimensionItem:
elrInfo["currentDim"] = concept
if concept.isTypedDimension:
elrInfo["dims"][concept.qname] = (concept, concept.typedDomainElement)
if concept.typedDomainElement.isNumeric:
elrInfo["domainIter"] = 1
elif concept.name.endswith("Member") or concept.name.endswith("_member"): # don't generate entries for default dim (Domain) (for now)
dimConcept = elrInfo["currentDim"]
if dimConcept.qname not in elrInfo["dims"]:
elrInfo["dims"][dimConcept.qname] = (dimConcept, concept)
else:
if concept.name.endswith("LineItems") or concept.name.endswith("_line_items"):
elrInfo["lineItems"] = True
elif ((not elrInfo["inCube"] or # before any hypercube
elrInfo["lineItems"]) # in Cube and within Line Items
and not concept.isAbstract): # or within line items
# generate a fact
sampVals = sampleDataValues[elrInfo.get("domainIter",1)] # use first entry if no domain iter
if concept.periodType not in elrInfo:
qnameDims = {}
for _dimConcept, _domConcept in elrInfo["dims"].values():
if _dimConcept.isExplicitDimension:
_memVal = _domConcept.qname
else:
if _domConcept.type is not None and not _domConcept.isNumeric:
_memEltVal = genSampleValue(sampVals, _domConcept)
else:
_memEltVal = str(elrInfo["domainIter"])
_memVal = XmlUtil.addChild(dts.modelDocument.xmlRootElement,
_domConcept.qname,
text=_memEltVal,
appendChild=False)
_dimObj = DimValuePrototype(dts, None, _dimConcept.qname, _memVal, "segment")
qnameDims[_dimConcept.qname] = _dimObj
elrInfo[concept.periodType] = dts.createContext(
dts.conceptSampleScheme or "http://www.treasury.gov",
"entityId",
concept.periodType,
sampVals["periodStart"] if concept.periodType == "duration"
else None,
sampVals["periodEnd"],
concept.qname, qnameDims, [], [])
cntx = elrInfo[concept.periodType]
cntxId = cntx.id
if concept.isNumeric:
if concept.isMonetary:
unitMeasure = qname(XbrlConst.iso4217, "USD")
unitMeasure.prefix = "iso4217" # want to save with a recommended prefix
decimals = 2
elif concept.isShares:
unitMeasure = XbrlConst.qnXbrliShares
decimals = 0
else:
unitMeasure = XbrlConst.qnXbrliPure
decimals = 0
# check if utr unitId is specified
utrUnitId = genSampleUtrUnitId(concept)
if utrUnitId is not None:
_utrEntries = dts.modelManager.disclosureSystem.utrItemTypeEntries[concept.type.name]
if _utrEntries:
for _utrEntry in _utrEntries.values():
if _utrEntry.unitId == utrUnitId and _utrEntry.isSimple:
unitMeasure = qname(_utrEntry.nsUnit, _utrEntry.unitId)
break
prevUnit = dts.matchUnit([unitMeasure], [])
if prevUnit is not None:
unitId = prevUnit.id
else:
newUnit = dts.createUnit([unitMeasure], [])
unitId = newUnit.id
value = genSampleValue(sampVals, concept)
attrs = [("contextRef", cntxId)]
if concept.isNumeric:
attrs.append(("unitRef", unitId))
attrs.append(("decimals", decimals))
value = Locale.atof(dts.locale, str(value), str.strip)
newFact = dts.createFact(concept.qname, attributes=attrs, text=value)
if concept not in visited:
visited.add(concept)
rels = relationshipSet.fromModelObject(concept)
lenRels = len(rels)
iRel = 0
iFirstLineItem = None
while iRel <= lenRels:
if iRel == lenRels: # check if cube needs re-iterating
if iFirstLineItem is None or elrInfo.get("domainIter",0) >= 2:
break
reIterateCube = True # cube can re-iterate
else:
modelRel = rels[iRel]
toConcept = modelRel.toModelObject
reIterateCube = (toConcept.isHypercubeItem and # finished prior line items and hitting next table
iFirstLineItem is not None and
elrInfo["lineItems"] and 1 <= elrInfo.get("domainIter",0) < 2)
if reIterateCube: # repeat typed dim container
iRel = iFirstLineItem
elrInfo["domainIter"] += 1
elrInfo.pop("instant", None) # want new contexts for next iteration
elrInfo.pop("duration", None)
isFirstLineItem = not elrInfo["lineItems"]
genFact(dts, toConcept, modelRel.preferredLabel, arcrole, relationshipSet, level+1, visited, elrInfo)
if isFirstLineItem and elrInfo["lineItems"] and elrInfo.get("domainIter",0) > 0:
iFirstLineItem = iRel
iRel += 1
visited.remove(concept)
except AttributeError as ex: # bad relationship
print ("[exception] {}".format(ex))
return
def saveTargetDocument(modelXbrl,
targetDocumentFilename,
targetDocumentSchemaRefs,
outputZip=None,
filingFiles=None):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
targetDocumentFilename, modelXbrl.modelDocument.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(
_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(
targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement,
roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in modelXbrl.contexts.values():
if context is not None:
newCntx = targetInstance.createContext(
context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
if unit is not None:
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0],
measures[1],
id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem:
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
newFact = targetInstance.createFact(fact.qname,
attributes=attrs,
text=text,
parent=parent)
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML(
"<body>\n{0}\n</body>\n".format(xmltext)):
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src"):
filingFiles.add(attrValue)
except (XMLSyntaxError, UnicodeDecodeError):
pass
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
# footnote links
footnoteIdCount = {}
modelXbrl.modelManager.showStatus(
_("Creating and validating footnotes & relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and any(
lP.modelDocument.type == Type.INLINEXBRL
for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = XmlUtil.addChild(
targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(
linkChild,
LocPrototype) and HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID,
0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(
attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
XmlUtil.addChild(newLink,
linkChild.qname,
attributes=attributes,
text=linkChild.textValue)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src"):
filingFiles.add(attrValue)
targetInstance.saveInstance(overrideFilepath=targetUrl,
outputZip=outputZip)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def createContext(self,
entityIdentScheme,
entityIdentValue,
periodType,
periodStart,
periodEndInstant,
priItem,
dims,
segOCCs,
scenOCCs,
afterSibling=None,
beforeSibling=None):
xbrlElt = self.modelDocument.xmlRootElement
if afterSibling == AUTO_LOCATE_ELEMENT:
afterSibling = XmlUtil.lastChild(
xbrlElt, XbrlConst.xbrli,
("schemaLocation", "roleType", "arcroleType", "context"))
cntxId = 'c-{0:02n}'.format(len(self.contexts) + 1)
newCntxElt = XmlUtil.addChild(xbrlElt,
XbrlConst.xbrli,
"context",
attributes=("id", cntxId),
afterSibling=afterSibling,
beforeSibling=beforeSibling)
entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity")
XmlUtil.addChild(entityElt,
XbrlConst.xbrli,
"identifier",
attributes=("scheme", entityIdentScheme),
text=entityIdentValue)
periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period")
if periodType == "forever":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever")
elif periodType == "instant":
XmlUtil.addChild(periodElt,
XbrlConst.xbrli,
"instant",
text=XmlUtil.dateunionValue(periodEndInstant,
subtractOneDay=True))
elif periodType == "duration":
XmlUtil.addChild(periodElt,
XbrlConst.xbrli,
"startDate",
text=XmlUtil.dateunionValue(periodStart))
XmlUtil.addChild(periodElt,
XbrlConst.xbrli,
"endDate",
text=XmlUtil.dateunionValue(periodEndInstant,
subtractOneDay=True))
segmentElt = None
scenarioElt = None
from arelle.ModelInstanceObject import ModelDimensionValue
if dims: # requires primary item to determin ambiguous concepts
''' in theory we have to check full set of dimensions for validity in source or any other
context element, but for shortcut will see if each dimension is already reported in an
unambiguous valid contextElement
'''
from arelle.PrototypeInstanceObject import FactPrototype, ContextPrototype, DimValuePrototype
fp = FactPrototype(self, priItem, dims.items())
# force trying a valid prototype's context Elements
if not isFactDimensionallyValid(
self, fp, setPrototypeContextElements=True):
self.info(
"arelleLinfo",
_("Create context for %(priItem)s, cannot determine valid context elements, no suitable hypercubes"
),
modelObject=self,
priItem=priItem)
fpDims = fp.context.qnameDims
for dimQname in sorted(fpDims.keys()):
dimValue = fpDims[dimQname]
if isinstance(dimValue, DimValuePrototype):
dimMemberQname = dimValue.memberQname # None if typed dimension
contextEltName = dimValue.contextElement
else: # qname for explicit or node for typed
dimMemberQname = None
contextEltName = None
if contextEltName == "segment":
if segmentElt is None:
segmentElt = XmlUtil.addChild(entityElt,
XbrlConst.xbrli,
"segment")
contextElt = segmentElt
elif contextEltName == "scenario":
if scenarioElt is None:
scenarioElt = XmlUtil.addChild(newCntxElt,
XbrlConst.xbrli,
"scenario")
contextElt = scenarioElt
else:
self.info(
"arelleLinfo",
_("Create context, %(dimension)s, cannot determine context element, either no all relationship or validation issue"
),
modelObject=self,
dimension=dimQname),
continue
dimConcept = self.qnameConcepts[dimQname]
dimAttr = ("dimension",
XmlUtil.addQnameValue(xbrlElt, dimConcept.qname))
if dimConcept.isTypedDimension:
dimElt = XmlUtil.addChild(contextElt,
XbrlConst.xbrldi,
"xbrldi:typedMember",
attributes=dimAttr)
if isinstance(dimValue,
(ModelDimensionValue,
DimValuePrototype)) and dimValue.isTyped:
XmlUtil.copyNodes(dimElt, dimValue.typedMember)
elif dimMemberQname:
dimElt = XmlUtil.addChild(contextElt,
XbrlConst.xbrldi,
"xbrldi:explicitMember",
attributes=dimAttr,
text=XmlUtil.addQnameValue(
xbrlElt, dimMemberQname))
if segOCCs:
if segmentElt is None:
segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli,
"segment")
XmlUtil.copyNodes(segmentElt, segOCCs)
if scenOCCs:
if scenarioElt is None:
scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli,
"scenario")
XmlUtil.copyNodes(scenarioElt, scenOCCs)
self.modelDocument.contextDiscover(newCntxElt)
XmlValidate.validate(self, newCntxElt)
return newCntxElt
def saveTargetDocument(modelXbrl, targetDocumentFilename,
targetDocumentSchemaRefs):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
targetDocumentFilename, modelXbrl.modelDocument.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(
_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(
targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement,
roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in modelXbrl.contexts.values():
newCntx = targetInstance.createContext(
context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0],
measures[1],
id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in modelXbrl.facts:
if fact.isItem:
attrs = {"contextRef": fact.contextID}
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
newFact = targetInstance.createFact(fact.qname,
attributes=attrs,
text=text,
parent=parent)
newFactForOldObjId[fact.objectIndex] = newFact
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
# footnote links
modelXbrl.modelManager.showStatus(
_("Creating and validating footnotes & relationships"))
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
any(lP.modelDocument.type == Type.INLINEXBRL
for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
newLink = XmlUtil.addChild(
targetInstance.modelDocument.xmlRootElement,
linkqname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
if isinstance(
linkChild, LocPrototype
) and "{http://www.w3.org/1999/xlink}href" not in linkChild.attributes:
linkChild.attributes["{http://www.w3.org/1999/xlink}href"] = \
"#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
XmlUtil.addChild(newLink,
linkChild.qname,
attributes=linkChild.attributes,
text=linkChild.textValue)
targetInstance.saveInstance(overrideFilepath=targetUrl)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def genFact(dts, concept, preferredLabel, arcrole, relationshipSet, level, visited, elrInfo):
try:
if concept is not None:
if concept.isHypercubeItem:
elrInfo["inCube"] = level
elrInfo["dims"] = {}
elrInfo["lineItems"] =False
elrInfo.pop("instant", None)
elrInfo.pop("duration", None)
elif concept.isDimensionItem:
elrInfo["currentDim"] = concept
if concept.isTypedDimension:
elrInfo["dims"][concept.qname] = (concept, concept.typedDomainElement)
if concept.typedDomainElement.isNumeric:
elrInfo["domainIter"] = 1
elif concept.name.endswith("Member"): # don't generate entries for default dim (Domain) (for now)
dimConcept = elrInfo["currentDim"]
if dimConcept.qname not in elrInfo["dims"]:
elrInfo["dims"][dimConcept.qname] = (dimConcept, concept)
else:
if concept.name.endswith("LineItems"):
elrInfo["lineItems"] = True
elif ((not elrInfo["inCube"] or # before any hypercube
elrInfo["lineItems"]) # in Cube and within Line Items
and not concept.isAbstract): # or within line items
# generate a fact
sampVals = sampleDataValues[elrInfo.get("domainIter",1)] # use first entry if no domain iter
if concept.periodType not in elrInfo:
qnameDims = {}
for _dimConcept, _domConcept in elrInfo["dims"].values():
if _dimConcept.isExplicitDimension:
_memVal = _domConcept.qname
else:
if _domConcept.type is not None and not _domConcept.isNumeric:
_memEltVal = genSampleValue(sampVals, _domConcept)
else:
_memEltVal = str(elrInfo["domainIter"])
_memVal = XmlUtil.addChild(dts.modelDocument.xmlRootElement,
_domConcept.qname,
text=_memEltVal,
appendChild=False)
_dimObj = DimValuePrototype(dts, None, _dimConcept.qname, _memVal, "segment")
qnameDims[_dimConcept.qname] = _dimObj
elrInfo[concept.periodType] = dts.createContext(
"http://treasury.gov", "entityId",
concept.periodType,
sampVals["periodStart"] if concept.periodType == "duration"
else None,
sampVals["periodEnd"],
concept.qname, qnameDims, [], [])
cntx = elrInfo[concept.periodType]
cntxId = cntx.id
if concept.isNumeric:
if concept.isMonetary:
unitMeasure = qname(XbrlConst.iso4217, "USD")
unitMeasure.prefix = "iso4217" # want to save with a recommended prefix
decimals = 2
elif concept.isShares:
unitMeasure = XbrlConst.qnXbrliShares
decimals = 0
else:
unitMeasure = XbrlConst.qnXbrliPure
decimals = 0
prevUnit = dts.matchUnit([unitMeasure], [])
if prevUnit is not None:
unitId = prevUnit.id
else:
newUnit = dts.createUnit([unitMeasure], [])
unitId = newUnit.id
value = genSampleValue(sampVals, concept)
attrs = [("contextRef", cntxId)]
if concept.isNumeric:
attrs.append(("unitRef", unitId))
attrs.append(("decimals", decimals))
value = Locale.atof(dts.locale, str(value), str.strip)
newFact = dts.createFact(concept.qname, attributes=attrs, text=value)
if concept not in visited:
visited.add(concept)
rels = relationshipSet.fromModelObject(concept)
lenRels = len(rels)
iRel = 0
iFirstLineItem = None
while iRel <= lenRels:
if iRel == lenRels: # check if cube needs re-iterating
if iFirstLineItem is None or elrInfo.get("domainIter",0) >= 2:
break
reIterateCube = True # cube can re-iterate
else:
modelRel = rels[iRel]
toConcept = modelRel.toModelObject
reIterateCube = (toConcept.isHypercubeItem and # finished prior line items and hitting next table
iFirstLineItem is not None and
elrInfo["lineItems"] and 1 <= elrInfo.get("domainIter",0) < 2)
if reIterateCube: # repeat typed dim container
iRel = iFirstLineItem
elrInfo["domainIter"] += 1
elrInfo.pop("instant", None) # want new contexts for next iteration
elrInfo.pop("duration", None)
isFirstLineItem = not elrInfo["lineItems"]
genFact(dts, toConcept, modelRel.preferredLabel, arcrole, relationshipSet, level+1, visited, elrInfo)
if isFirstLineItem and elrInfo["lineItems"] and elrInfo.get("domainIter",0) > 0:
iFirstLineItem = iRel
iRel += 1
visited.remove(concept)
except AttributeError as ex: # bad relationship
print ("[exception] {}".format(ex))
return
def loadFromExcel(cntlr, excelFile):
from openpyxl import load_workbook
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
startedAt = time.time()
if os.path.isabs(excelFile):
# allow relative filenames to loading directory
priorCWD = os.getcwd()
os.chdir(os.path.dirname(excelFile))
else:
priorCWD = None
importExcelBook = load_workbook(excelFile, read_only=True, data_only=True)
sheetNames = importExcelBook.get_sheet_names()
if "DTS" in sheetNames:
dtsWs = importExcelBook["DTS"]
elif "Sheet2" in sheetNames:
dtsWs = importExcelBook["Sheet2"]
else:
dtsWs = None
imports = {"xbrli": ( ("namespace", XbrlConst.xbrli),
("schemaLocation", "http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd") )} # xml of imports
importXmlns = {}
linkbaseRefs = []
labelLinkbases = []
hasPreLB = hasCalLB = hasDefLB = False
# xxxLB structure [ (elr1, def1, "_ELR_", [roots]), (elr2, def2, "_ELR_", [rootw]) ...]
# roots = (rootHref, None, "_root_", [children])
# children = (childPrefix, childName, arcrole, [grandChildren])
preLB = []
defLB = []
calLB = []
def lbDepthList(lbStruct, depth, parentList=None):
if depth == topDepth:
if len(lbStruct) > 0:
return lbStruct[-1].childStruct
else:
cntlr.addToLog("Depth error, Excel row: {excelRow}"
.format(excelRow=iRow),
messageCode="importExcel:depth")
return None
return lbDepthList(lbStruct[-1].childStruct, depth-1, list)
extensionElements = {}
extensionRoles = {} # key is roleURI, value is role definition
extensionLabels = {} # key = (prefix, name, lang, role), value = label text
importSheetName = None
skipRows = [] # [(from,to),(from,to)] row number starting at 1
def extensionHref(prefix, name):
if prefix == extensionSchemaPrefix:
filename = extensionSchemaFilename
elif prefix in imports:
filename = imports[prefix][1][1]
else:
return None
return "{0}#{1}_{2}".format(filename, prefix, name)
isUSGAAP = False
for iRow, row in enumerate(dtsWs.rows if dtsWs else ()):
try:
if (len(row) < 1 or row[0].value is None): # skip if col 1 is empty
continue
action = filetype = prefix = filename = namespaceURI = None
if len(row) > 0: action = row[0].value
if len(row) > 1: filetype = row[1].value
if len(row) > 2: prefix = row[2].value
if len(row) > 3: filename = row[3].value
if len(row) > 4: namespaceURI = row[4].value
lbType = lang = None
if action == "import":
imports[prefix] = ( ("namespace", namespaceURI), ("schemaLocation", filename) )
importXmlns[prefix] = namespaceURI
if re.match(r"http://[^/]+/us-gaap/", namespaceURI):
isUSGAAP = True
elif action == "extension":
if filetype == "schema":
extensionSchemaPrefix = prefix
extensionSchemaFilename = filename
extensionSchemaNamespaceURI = namespaceURI
elif filetype == "linkbase":
typeLang = prefix.split()
if len(typeLang) > 0:
lbType = typeLang[0]
else:
lbType = "unknown"
if len(typeLang) > 1:
lang = typeLang[1]
else:
lang = "en"
if lbType == "label":
labelLinkbases.append((lang, filename))
elif lbType == "presentation":
hasPreLB = True
elif lbType == "definition":
hasDefLB = True
elif lbType == "calculation":
hasCalLB = True
linkbaseRefs.append( (lbType, filename) )
elif filetype == "role" and namespaceURI:
extensionRoles[namespaceURI] = filename
elif action == "worksheet" and filename:
importSheetName = filename
elif action == "colheader" and filename and namespaceURI:
importColHeaderMap[filename].append(namespaceURI)
elif action == "skip rows":
if filename:
fromRow, _sep, toRow = filename.partition("-")
try:
skipRows.append((int(fromRow), int(toRow) if toRow else int(fromRow)))
except (ValueError, TypeError):
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:skip rows")
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:exception")
if not isUSGAAP: # need extra namespace declaration
importXmlns["iod"] = "http://disclosure.edinet-fsa.go.jp/taxonomy/common/2013-03-31/iod"
# find column headers row
headerCols = {}
hasLinkroleSeparateRow = True
headerRows = set()
topDepth = 999999
if importSheetName and importSheetName in sheetNames:
conceptsWs = importExcelBook[importSheetName]
elif "Concepts" in sheetNames:
conceptsWs = importExcelBook["Concepts"]
elif "Sheet1" in sheetNames:
conceptsWs = importExcelBook["Sheet1"]
else:
conceptsWs = None
def setHeaderCols(row):
headerCols.clear()
for iCol, colCell in enumerate(row):
v = colCell.value
if v in importColHeaderMap:
for hdr in importColHeaderMap[v]:
if hdr in importColumnHeaders:
headerCols[importColumnHeaders[hdr]] = iCol
elif v in importColumnHeaders:
headerCols[importColumnHeaders[v]] = iCol
# find out which rows are header rows
for iRow, row in enumerate(conceptsWs.rows if conceptsWs else ()):
if any(fromRow <= iRow+1 <= toRow for fromRow,toRow in skipRows):
continue
for iCol, colCell in enumerate(row):
setHeaderCols(row)
if all(colName in headerCols
for colName in ("name", "type", "depth")): # must have these to be a header col
# it's a header col
headerRows.add(iRow+1)
if 'linkrole' in headerCols:
hasLinkroleSeparateRow = False
headerCols.clear()
def cellHasValue(row, header, _type):
if header in headerCols:
iCol = headerCols[header]
return iCol < len(row) and isinstance(row[iCol].value, _type)
return False
def cellValue(row, header, strip=False, nameChars=False):
if header in headerCols:
iCol = headerCols[header]
if iCol < len(row):
v = row[iCol].value
if strip and isinstance(v, str):
v = v.strip()
if nameChars and isinstance(v, str):
v = ''.join(c for c in v if c.isalnum() or c in ('.', '_', '-'))
return v
return None
def checkImport(qname):
prefix, sep, localName = qname.partition(":")
if sep:
if prefix not in imports:
if prefix == "xbrldt":
imports["xbrldt"] = ("namespace", XbrlConst.xbrldt), ("schemaLocation", "http://www.xbrl.org/2005/xbrldt-2005.xsd")
elif prefix == "nonnum":
imports["nonnum"] = ("namespace", "http://www.xbrl.org/dtr/type/non-numeric"), ("schemaLocation", "http://www.xbrl.org/dtr/type/nonNumeric-2009-12-16.xsd")
else:
cntlr.addToLog("Warning: prefix schema file is not imported for: {qname}"
.format(qname=qname),
messageCode="importExcel:warning")
# find top depth
for iRow, row in enumerate(conceptsWs.rows if conceptsWs else ()):
if (iRow + 1) in headerRows:
setHeaderCols(row)
elif not (hasLinkroleSeparateRow and (iRow + 1) in headerRows) and 'depth' in headerCols:
depth = cellValue(row, 'depth')
if isinstance(depth, int) and depth < topDepth:
topDepth = depth
# find header rows
currentELR = currentELRdefinition = None
for iRow, row in enumerate(conceptsWs.rows if conceptsWs else ()):
useLabels = False
if any(fromRow <= iRow+1 <= toRow for fromRow,toRow in skipRows):
continue
if (all(col is None for col in row) or
all(isinstance(row[i].value, str) and row[i].value.strip() == "n/a"
for i in (headerCols.get("name"), headerCols.get("type"), headerCols.get("value"))
if i)):
continue # skip blank row
try:
isHeaderRow = (iRow + 1) in headerRows
isELRrow = hasLinkroleSeparateRow and (iRow + 2) in headerRows
if isHeaderRow:
setHeaderCols(row)
elif isELRrow:
currentELR = currentELRdefinition = None
for colCell in row:
v = str(colCell.value or '')
if v.startswith("http://"):
currentELR = v
elif not currentELRdefinition and v.endswith(" 科目一覧"):
currentELRdefinition = v[0:-5]
elif not currentELRdefinition:
currentELRdefinition = v
if currentELR or currentELRdefinition:
if hasPreLB:
preLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasDefLB:
defLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasCalLB:
calLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
calRels = set() # prevent duplications when same rel in different parts of tree
elif headerCols:
if "linkrole" in headerCols and cellHasValue(row, 'linkrole', str):
v = cellValue(row, 'linkrole', strip=True)
_trialELR = _trialELRdefinition = None
if v.startswith("http://"):
_trialELR = v
elif v.endswith(" 科目一覧"):
_trialELRdefinition = v[0:-5]
else:
_trialELRdefinition = v
if (_trialELR and _trialELR != currentELR) or (_trialELRdefinition and _trialELRdefinition != currentELRdefinition):
currentELR = _trialELR
currentELRdefinition = _trialELRdefinition
if currentELR or currentELRdefinition:
if hasPreLB:
preLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasDefLB:
defLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasCalLB:
calLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
calRels = set() # prevent duplications when same rel in different parts of tree
prefix = cellValue(row, 'prefix', nameChars=True) or extensionSchemaPrefix
if cellHasValue(row, 'name', str):
name = cellValue(row, 'name', nameChars=True)
else:
name = None
if cellHasValue(row, 'depth', int):
depth = cellValue(row, 'depth')
else:
depth = None
if (not prefix or prefix == extensionSchemaPrefix) and name not in extensionElements and name:
# elements row
eltType = cellValue(row, 'type')
eltTypePrefix = cellValue(row, 'typePrefix')
if not eltType:
eltType = 'xbrli:stringItemType'
elif eltTypePrefix and ':' not in eltType:
eltType = eltTypePrefix + ':' + eltType
elif ':' not in eltType and eltType.endswith("ItemType"):
eltType = 'xbrli:' + eltType
subsGrp = cellValue(row, 'substitutionGroup') or 'xbrli:item'
abstract = cellValue(row, 'abstract') or (cellValue(row, 'abstractMarker') is not None)
nillable = cellValue(row, 'nillable')
balance = cellValue(row, 'balance')
periodType = cellValue(row, 'periodType')
newElt = [ ("name", name), ("id", (prefix or "") + "_" + name) ]
if eltType:
newElt.append( ("type", eltType) )
checkImport(eltType)
if subsGrp:
newElt.append( ("substitutionGroup", subsGrp) )
checkImport(subsGrp)
if abstract or subsGrp in ("xbrldt:hypercubeItem", "xbrldt:dimensionItem"):
newElt.append( ("abstract", abstract or "true") )
if nillable:
newElt.append( ("nillable", nillable) )
if balance:
newElt.append( ("{http://www.xbrl.org/2003/instance}balance", balance) )
if periodType:
newElt.append( ("{http://www.xbrl.org/2003/instance}periodType", periodType) )
extensionElements[name] = newElt
useLabels = True
if depth is not None:
if hasPreLB:
entryList = lbDepthList(preLB, depth)
preferredLabel = cellValue(row, 'preferredLabel')
if preferredLabel and not preferredLabel.startswith("http://"):
preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel
if entryList is not None:
if depth == topDepth:
entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) )
else:
entryList.append( LBentry(prefix=prefix, name=name, arcrole=XbrlConst.parentChild,
role=preferredLabel) )
if hasDefLB:
entryList = lbDepthList(defLB, depth)
if entryList is not None:
if depth == topDepth:
entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) )
else:
if (not preferredLabel or # prevent start/end labels from causing duplicate dim-mem relationships
not any(lbEntry.prefix == prefix and lbEntry.name == name
for lbEntry in entryList)):
entryList.append( LBentry(prefix=prefix, name=name, arcrole="_dimensions_") )
if hasCalLB:
calcParents = cellValue(row, 'calculationParent').split()
calcWeights = (str(cellValue(row, 'calculationWeight')) or '').split() # may be float or string
if calcParents and calcWeights:
# may be multiple parents split by whitespace
for i, calcParent in enumerate(calcParents):
calcWeight = calcWeights[i] if i < len(calcWeights) else calcWeights[-1]
calcParentPrefix, sep, calcParentName = calcParent.partition(":")
entryList = lbDepthList(calLB, topDepth)
if entryList is not None:
calRel = (calcParentPrefix, calcParentName, prefix, name)
if calRel not in calRels:
entryList.append( LBentry(prefix=calcParentPrefix, name=calcParentName, isRoot=True, childStruct=
[LBentry(prefix=prefix, name=name, arcrole=XbrlConst.summationItem, weight=calcWeight )]) )
calRels.add(calRel)
else:
pass
# accumulate extension labels
if useLabels:
prefix = cellValue(row, 'prefix', nameChars=True) or extensionSchemaPrefix
name = cellValue(row, 'name', nameChars=True)
if name is not None:
preferredLabel = cellValue(row, 'preferredLabel')
if preferredLabel and not preferredLabel.startswith("http://"):
preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel
for colItem, iCol in headerCols.items():
if isinstance(colItem, tuple):
colItemType = colItem[0]
role = colItem[1]
lang = colItem[2]
cell = row[iCol]
if cell.value is None:
values = ()
elif colItemType == "label":
values = (cell.value,)
elif colItemType == "labels":
values = cell.value.split('\n')
else:
values = ()
if preferredLabel and "indented" in colItem: # indented column sets preferredLabel if any
role = preferredLabel
for value in values:
extensionLabels[prefix, name, lang, role] = value.strip()
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:exception")
if isUSGAAP and hasDefLB:
# move line items above table
def fixUsggapTableDims(lvl1Struct):
foundLineItems = False
for i1, lvl1Entry in enumerate(lvl1Struct):
for i2, lvl2Entry in enumerate(lvl1Entry.childStruct):
for i3, lvl3Entry in enumerate(lvl2Entry.childStruct):
if lvl3Entry.name.endswith("LineItems") and lvl2Entry.name.endswith("Table"):
foundLineItems = True
break
if foundLineItems:
break
else:
fixUsggapTableDims(lvl1Entry.childStruct)
if foundLineItems:
lvl1Struct.insert(i1 + 1, LBentry(prefix=lvl3Entry.prefix, name=lvl3Entry.name, arcrole=lvl1Entry.arcrole, childStruct=lvl3Entry.childStruct)) # must keep lvl1Rel if it is __root__
lvl3Entry.childStruct.insert(0, lvl2Entry)
if lvl1Entry.name.endswith("Abstract"):
del lvl1Struct[i1]
if i3 < len(lvl2Entry.childStruct):
del lvl2Entry.childStruct[i3]
pass
fixUsggapTableDims(defLB)
dts = cntlr.modelManager.create(newDocumentType=ModelDocument.Type.SCHEMA,
url=extensionSchemaFilename,
isEntry=True,
base='', # block pathname from becomming absolute
initialXml='''
<schema xmlns="http://www.w3.org/2001/XMLSchema"
targetNamespace="{targetNamespace}"
attributeFormDefault="unqualified"
elementFormDefault="qualified"
xmlns:xsd="http://www.w3.org/2001/XMLSchema"
xmlns:{extensionPrefix}="{targetNamespace}"
{importXmlns}
xmlns:nonnum="http://www.xbrl.org/dtr/type/non-numeric"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xbrli="http://www.xbrl.org/2003/instance"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrldt="http://xbrl.org/2005/xbrldt"/>
'''.format(targetNamespace=extensionSchemaNamespaceURI,
extensionPrefix=extensionSchemaPrefix,
importXmlns=''.join('xmlns:{0}="{1}"\n'.format(prefix, namespaceURI)
for prefix, namespaceURI in importXmlns.items())
)
)
dtsSchemaDocument = dts.modelDocument
dtsSchemaDocument.inDTS = True # entry document always in DTS
dtsSchemaDocument.targetNamespace = extensionSchemaNamespaceURI # not set until schemaDiscover too late otherwise
schemaElt = dtsSchemaDocument.xmlRootElement
#foreach linkbase
annotationElt = XmlUtil.addChild(schemaElt, XbrlConst.xsd, "annotation")
appinfoElt = XmlUtil.addChild(annotationElt, XbrlConst.xsd, "appinfo")
# add linkbaseRefs
appinfoElt = XmlUtil.descendant(schemaElt, XbrlConst.xsd, "appinfo")
# don't yet add linkbase refs, want to process imports first to get roleType definitions
# add imports
for importAttributes in sorted(imports.values()):
XmlUtil.addChild(schemaElt,
XbrlConst.xsd, "import",
attributes=importAttributes)
# add elements
for eltName, eltAttrs in sorted(extensionElements.items(), key=lambda item: item[0]):
XmlUtil.addChild(schemaElt,
XbrlConst.xsd, "element",
attributes=eltAttrs)
# add role definitions (for discovery)
for roleURI, roleDefinition in extensionRoles.items():
roleElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "roleType",
attributes=(("roleURI", roleURI),
("id", "roleType_" + roleURI.rpartition("/")[2])))
if roleDefinition:
XmlUtil.addChild(roleElt, XbrlConst.link, "definition", text=roleDefinition)
if hasPreLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:presentationLink")
if hasDefLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:definitionLink")
if hasCalLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:calculationLink")
dtsSchemaDocument.schemaDiscover(schemaElt, False, extensionSchemaNamespaceURI)
def addLinkbaseRef(lbType, lbFilename, lbDoc):
role = "http://www.xbrl.org/2003/role/{0}LinkbaseRef".format(lbType)
lbRefElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "linkbaseRef",
attributes=(("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", lbFilename),
("{http://www.w3.org/1999/xlink}role", role),
("{http://www.w3.org/1999/xlink}arcrole", "http://www.w3.org/1999/xlink/properties/linkbase"),
))
dtsSchemaDocument.referencesDocument[lbDoc] = ModelDocumentReference("href", lbRefElt)
# label linkbase
for lang, filename in labelLinkbases:
lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base="", initialXml="""
<linkbase
xmlns="http://www.xbrl.org/2003/linkbase"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase
http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance"/>
""")
lbDoc.inDTS = True
addLinkbaseRef("label", filename, lbDoc)
lbElt = lbDoc.xmlRootElement
linkElt = XmlUtil.addChild(lbElt,
XbrlConst.link, "labelLink",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/link")))
firstLinkElt = linkElt
locs = set()
roleRefs = set()
for labelKey, text in extensionLabels.items():
prefix, name, labelLang, role = labelKey
if lang == labelLang:
locLabel = prefix + "_" + name
if locLabel not in locs:
locs.add(locLabel)
XmlUtil.addChild(linkElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", extensionHref(prefix, name)),
("{http://www.w3.org/1999/xlink}label", locLabel)))
XmlUtil.addChild(linkElt,
XbrlConst.link, "labelArc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", "http://www.xbrl.org/2003/arcrole/concept-label"),
("{http://www.w3.org/1999/xlink}from", locLabel),
("{http://www.w3.org/1999/xlink}to", "label_" + locLabel),
("order", 1.0)))
XmlUtil.addChild(linkElt,
XbrlConst.link, "label",
attributes=(("{http://www.w3.org/1999/xlink}type", "resource"),
("{http://www.w3.org/1999/xlink}label", "label_" + locLabel),
("{http://www.w3.org/1999/xlink}role", role),
("{http://www.w3.org/XML/1998/namespace}lang", lang)),
text=text)
if role:
if role in dts.roleTypes:
roleType = dts.roleTypes[role][0]
roleRefs.add(("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id))
elif role.startswith("http://www.xbrl.org/2009/role/negated"):
roleRefs.add(("roleRef", role, "http://www.xbrl.org/lrr/role/negated-2009-12-16.xsd#" + role.rpartition("/")[2]))
# add arcrole references
for roleref, roleURI, href in roleRefs:
XmlUtil.addChild(lbElt,
XbrlConst.link, roleref,
attributes=(("arcroleURI" if roleref == "arcroleRef" else "roleURI", roleURI),
("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", href)),
beforeSibling=firstLinkElt)
lbDoc.linkbaseDiscover(lbElt)
def hrefConcept(prefix, name):
qn = schemaElt.prefixedNameQname(prefix + ":" + name)
if qn in dts.qnameConcepts:
return dts.qnameConcepts[qn]
return None
def lbTreeWalk(lbType, parentElt, lbStruct, roleRefs, locs=None, arcsFromTo=None, fromPrefix=None, fromName=None):
order = 1.0
for lbEntry in lbStruct:
if lbEntry.isELR:
role = "unspecified"
if lbEntry.role and lbEntry.role.startswith("http://"): # have a role specified
role = lbEntry.role
elif lbEntry.name: #may be a definition
for linkroleUri, modelRoleTypes in dts.roleTypes.items():
definition = modelRoleTypes[0].definition
if lbEntry.name == definition:
role = linkroleUri
break
if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef
roleType = modelRoleTypes[0]
roleRefs.add(("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id))
linkElt = XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Link",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", role)))
locs = set()
arcsFromTo = set()
lbTreeWalk(lbType, linkElt, lbEntry.childStruct, roleRefs, locs, arcsFromTo)
else:
toPrefix = lbEntry.prefix
toName = lbEntry.name
toHref = extensionHref(toPrefix, toName)
toLabel = toPrefix + "_" + toName
toLabelAlt = None
if not lbEntry.isRoot:
fromLabel = fromPrefix + "_" + fromName
if (fromLabel, toLabel) in arcsFromTo:
# need extra loc to prevent arc from/to duplication in ELR
for i in range(1, 1000):
toLabelAlt = "{}_{}".format(toLabel, i)
if (fromLabel, toLabelAlt) not in arcsFromTo:
toLabel = toLabelAlt
break
if toHref not in locs or toLabelAlt:
XmlUtil.addChild(parentElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", toHref),
("{http://www.w3.org/1999/xlink}label", toLabel)))
locs.add(toHref)
if not lbEntry.isRoot:
arcsFromTo.add( (fromLabel, toLabel) )
if lbType == "calculation" and lbEntry.weight is not None:
otherAttrs = ( ("weight", lbEntry.weight), )
elif lbType == "presentation" and lbEntry.role:
otherAttrs = ( ("preferredLabel", lbEntry.role), )
if lbEntry.role and lbEntry.role in dts.roleTypes:
roleType = dts.roleTypes[lbEntry.role][0]
roleRefs.add(("roleRef", lbEntry.role, roleType.modelDocument.uri + "#" + roleType.id))
else:
otherAttrs = ( )
if lbEntry.arcrole == "_dimensions_": # pick proper consecutive arcrole
fromConcept = hrefConcept(fromPrefix, fromName)
toConcept = hrefConcept(toPrefix, toName)
if toConcept is not None and toConcept.isHypercubeItem:
arcrole = XbrlConst.all
otherAttrs += ( (XbrlConst.qnXbrldtContextElement, "segment"), )
elif toConcept is not None and toConcept.isDimensionItem:
arcrole = XbrlConst.hypercubeDimension
elif fromConcept is not None and fromConcept.isDimensionItem:
arcrole = XbrlConst.dimensionDomain
else:
arcrole = XbrlConst.domainMember
else:
arcrole = lbEntry.arcrole
XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Arc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", arcrole),
("{http://www.w3.org/1999/xlink}from", fromLabel),
("{http://www.w3.org/1999/xlink}to", toLabel),
("order", order)) + otherAttrs )
order += 1.0
if lbType != "calculation" or lbEntry.isRoot:
lbTreeWalk(lbType, parentElt, lbEntry.childStruct, roleRefs, locs, arcsFromTo, toPrefix, toName)
for hasLB, lbType, lbLB in ((hasPreLB, "presentation", preLB),
(hasDefLB, "definition", defLB),
(hasCalLB, "calculation", calLB)):
if hasLB:
for lbRefType, filename in linkbaseRefs:
if lbType == lbRefType:
# output presentation linkbase
lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base='', initialXml="""
<linkbase
xmlns="http://www.xbrl.org/2003/linkbase"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase
http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance"/>
""")
lbDoc.inDTS = True
addLinkbaseRef(lbRefType, filename, lbDoc)
lbElt = lbDoc.xmlRootElement
roleRefs = set()
if lbType == "definition":
roleRefs.update((("arcroleRef", XbrlConst.all, "http://www.xbrl.org/2005/xbrldt-2005.xsd#all"),
("arcroleRef", XbrlConst.dimensionDefault, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-default"),
("arcroleRef", XbrlConst.dimensionDomain, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-domain"),
("arcroleRef", XbrlConst.domainMember, "http://www.xbrl.org/2005/xbrldt-2005.xsd#domain-member"),
("arcroleRef", XbrlConst.hypercubeDimension, "http://www.xbrl.org/2005/xbrldt-2005.xsd#hypercube-dimension")))
lbTreeWalk(lbType, lbElt, lbLB, roleRefs)
firstLinkElt = None
for firstLinkElt in lbElt.iterchildren():
break
# add arcrole references
for roleref, roleURI, href in roleRefs:
XmlUtil.addChild(lbElt,
XbrlConst.link, roleref,
attributes=(("arcroleURI" if roleref == "arcroleRef" else "roleURI", roleURI),
("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", href)),
beforeSibling=firstLinkElt)
lbDoc.linkbaseDiscover(lbElt)
break
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
if priorCWD:
os.chdir(priorCWD) # restore prior current working directory
return dts
def produceOutputFact(xpCtx, formula, result):
priorErrorCount = len(xpCtx.modelXbrl.errors)
# assemble context
conceptQname = aspectValue(xpCtx, formula, Aspect.CONCEPT, "xbrlfe:missingConceptRule")
if isinstance(conceptQname, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} concept: {1}").format( formula, conceptQname.msg),
"err", conceptQname.err)
modelConcept = None
else:
modelConcept = xpCtx.modelXbrl.qnameConcepts[conceptQname]
if modelConcept is None or not modelConcept.isItem:
xpCtx.modelXbrl.error( _("Formula {0} concept {1} is not an item").format( formula, conceptQname),
"err", "xbrlfe:missingConceptRule")
# entity
entityIdentScheme = aspectValue(xpCtx, formula, Aspect.SCHEME, "xbrlfe:missingEntityIdentifierRule")
if isinstance(entityIdentScheme, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} entity identifier scheme: {1}").format( formula, entityIdentScheme.msg ),
"err", str(entityIdentScheme))
entityIdentValue = None
else:
entityIdentValue = aspectValue(xpCtx, formula, Aspect.VALUE, "xbrlfe:missingEntityIdentifierRule")
if isinstance(entityIdentValue, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} entity identifier value: {1}").format( formula, entityIdentValue.msg ),
"err", str(entityIdentScheme))
# period
periodType = aspectValue(xpCtx, formula, Aspect.PERIOD_TYPE, "xbrlfe:missingPeriodRule")
periodStart = None
periodEndInstant = None
if isinstance(periodType, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} period type: {1}").format( formula, periodType.msg ),
"err", str(periodType))
elif periodType == "instant":
periodEndInstant = aspectValue(xpCtx, formula, Aspect.INSTANT, "xbrlfe:missingPeriodRule")
if isinstance(periodEndInstant, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} period start: {1}").format( formula, periodEndInstant.msg ),
"err", str(periodEndInstant))
elif periodType == "duration":
periodStart = aspectValue(xpCtx, formula, Aspect.START, "xbrlfe:missingPeriodRule")
if isinstance(periodStart, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} period start: {1}").format( formula, periodStart.msg ),
"err", str(periodStart))
periodEndInstant = aspectValue(xpCtx, formula, Aspect.END, "xbrlfe:missingPeriodRule")
if isinstance(periodEndInstant, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} period end: {1}").format( formula, periodEndInstant.msg ),
"err", str(periodEndInstant))
# unit
if modelConcept and modelConcept.isNumeric:
unitSource = aspectValue(xpCtx, formula, Aspect.UNIT_MEASURES, None)
multDivBy = aspectValue(xpCtx, formula, Aspect.MULTIPLY_BY, "xbrlfe:missingUnitRule")
if isinstance(multDivBy, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} unit: {1}").format( formula, multDivBy.msg ),
"err", str(multDivBy) if isinstance(multDivBy, VariableBindingError) else "xbrlfe:missingUnitRule")
multiplyBy = (); divideBy = () # prevent errors later if bad
else:
divMultBy = aspectValue(xpCtx, formula, Aspect.DIVIDE_BY, "xbrlfe:missingUnitRule")
if isinstance(divMultBy, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} unit: {1}").format( formula, divMultBy.msg ),
"err", str(multDivBy) if isinstance(divMultBy, VariableBindingError) else "xbrlfe:missingUnitRule")
multiplyBy = (); divideBy = () # prevent errors later if bad
else:
multiplyBy = unitSource[0] + multDivBy[0] + divMultBy[1]
divideBy = unitSource[1] + multDivBy[1] + divMultBy[0]
# remove cancelling mult/div units
lookForCommonUnits = True
while lookForCommonUnits:
lookForCommonUnits = False
for commonUnit in multiplyBy:
if commonUnit in divideBy:
multiplyBy.remove(commonUnit)
divideBy.remove(commonUnit)
lookForCommonUnits = True
break
if len(multiplyBy) == 0: # if no units add pure
multiplyBy.append(XbrlConst.qnXbrliPure)
# dimensions
segOCCs = []
scenOCCs = []
if formula.aspectModel == "dimensional":
dimAspects = {}
dimQnames = aspectValue(xpCtx, formula, Aspect.DIMENSIONS, None)
if dimQnames:
for dimQname in dimQnames:
dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname]
dimErr = "xbrlfe:missing{0}DimensionRule".format("typed" if dimConcept and dimConcept.isTypedDimension else "explicit")
dimValue = aspectValue(xpCtx, formula, dimQname, dimErr)
if isinstance(dimValue, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} dimension {1}: {2}").format( formula, dimQname, dimValue.msg ),
"err", dimErr)
elif dimValue and xpCtx.modelXbrl.qnameDimensionDefaults.get(dimQname) != dimValue:
dimAspects[dimQname] = dimValue
segOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SEGMENT, None)
scenOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SCENARIO, None)
else:
dimAspects = None # non-dimensional
segOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SEGMENT, None)
scenOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SCENARIO, None)
if priorErrorCount < len(xpCtx.modelXbrl.errors):
return None # had errors, don't produce output fact
# does context exist in out instance document
outputInstanceQname = formula.outputInstanceQname
outputXbrlInstance = xpCtx.inScopeVars[outputInstanceQname]
xbrlElt = outputXbrlInstance.modelDocument.xmlRootElement
# in source instance document
# add context
prevCntx = outputXbrlInstance.matchContext(
entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant,
dimAspects, segOCCs, scenOCCs)
if prevCntx:
cntxId = prevCntx.id
newCntxElt = prevCntx.element
else:
cntxId = 'c-{0:02n}'.format( len(outputXbrlInstance.contexts) + 1)
newCntxElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId),
afterSibling=xpCtx.outputLastContext.get(outputInstanceQname))
xpCtx.outputLastContext[outputInstanceQname] = newCntxElt
entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity")
XmlUtil.addChild(entityElt, XbrlConst.xbrli, "identifier",
attributes=("scheme", entityIdentScheme),
text=entityIdentValue)
periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period")
if periodType == "forever":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever")
elif periodType == "instant":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "instant",
text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True))
elif periodType == "duration":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "startDate",
text=XmlUtil.dateunionValue(periodStart))
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "endDate",
text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True))
segmentElt = None
scenarioElt = None
from arelle.ModelObject import ModelDimensionValue
if dimAspects:
for dimQname in sorted(dimAspects.keys()):
dimValue = dimAspects[dimQname]
if isinstance(dimValue, ModelDimensionValue):
if dimValue.isExplicit:
dimMemberQname = dimValue.memberQname
contextEltName = dimValue.contextElement
else: # qname for explicit or node for typed
dimMemberQname = dimValue
contextEltName = xpCtx.modelXbrl.qnameDimensionContextElement.get(dimQname)
if contextEltName == "segment":
if not segmentElt:
segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment")
contextElt = segmentElt
elif contextEltName == "scenario":
if not scenarioElt:
scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario")
contextElt = scenarioElt
else:
continue
dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname]
dimAttr = ("dimension", XmlUtil.addQnameValue(xbrlElt, dimConcept.qname))
if dimConcept.isTypedDimension:
dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "typedMember",
attributes=dimAttr)
if isinstance(dimValue, ModelDimensionValue) and dimValue.isTyped:
XmlUtil.copyChildren(dimElt, dimValue.typedMember)
elif dimMemberQname:
dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "explicitMember",
attributes=dimAttr,
text=XmlUtil.addQnameValue(xbrlElt, dimMemberQname))
if segOCCs:
if not segmentElt:
segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment")
XmlUtil.copyNodes(segmentElt, segOCCs)
if scenOCCs:
if not scenarioElt:
scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario")
XmlUtil.copyNodes(scenarioElt, scenOCCs)
outputXbrlInstance.modelDocument.contextDiscover(newCntxElt)
# does unit exist
# add unit
if modelConcept.isNumeric:
prevUnit = outputXbrlInstance.matchUnit(multiplyBy, divideBy)
if prevUnit:
unitId = prevUnit.id
newUnitElt = prevUnit.element
else:
unitId = 'u-{0:02n}'.format( len(outputXbrlInstance.units) + 1)
newUnitElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId),
afterSibling=xpCtx.outputLastUnit.get(outputInstanceQname))
xpCtx.outputLastUnit[outputInstanceQname] = newUnitElt
if len(divideBy) == 0:
for multiply in multiplyBy:
XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply))
else:
divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide")
numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator")
denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator")
for multiply in multiplyBy:
XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply))
for divide in divideBy:
XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, divide))
outputXbrlInstance.modelDocument.unitDiscover(newUnitElt)
# add fact
attrs = [("contextRef", cntxId)]
precision = None
decimals = None
if modelConcept.isNumeric:
attrs.append(("unitRef", unitId))
value = formula.evaluate(xpCtx)
valueSeqLen = len(value)
if valueSeqLen > 1:
xpCtx.modelXbrl.error( _("Formula {0} value is a sequence of length {1}").format( formula, valueSeqLen ),
"err", "xbrlfe:nonSingletonOutputValue")
else:
if valueSeqLen == 0: #xsi:nil if no value
attrs.append((XbrlConst.qnXsiNil, "true"))
v = None
else:
# add precision/decimals for non-fraction numerics
if modelConcept.isNumeric and not modelConcept.isFraction:
if formula.hasDecimals:
decimals = formula.evaluateRule(xpCtx, Aspect.DECIMALS)
attrs.append(("decimals", decimals))
else:
if formula.hasPrecision:
precision = formula.evaluateRule(xpCtx, Aspect.PRECISION)
else:
precision = 0
attrs.append(("precision", precision))
x = value[0]
if isinstance(x,float):
from math import (log10, isnan, isinf, fabs)
if (isnan(x) or
(precision and (isinf(precision) or precision == 0)) or
(decimals and isinf(decimals))):
v = string(xpCtx, x)
elif decimals is not None:
v = "%.*f" % ( int(decimals), x)
elif precision is not None:
a = fabs(x)
log = log10(a) if a != 0 else 0
v = "%.*f" % ( int(precision) - int(log) - (1 if a >= 1 else 0), x)
else: # no implicit precision yet
v = string(xpCtx, x)
elif isinstance(x,QName):
v = XmlUtil.addQnameValue(xbrlElt, x)
elif isinstance(x,datetime.datetime):
v = XmlUtil.dateunionValue(x)
else:
v = string(xpCtx, x)
itemElt = XmlUtil.addChild(xbrlElt, conceptQname,
attributes=attrs, text=v,
afterSibling=xpCtx.outputLastFact.get(outputInstanceQname))
xpCtx.outputLastFact[outputInstanceQname] = itemElt
newFact = outputXbrlInstance.modelDocument.factDiscover(itemElt, outputXbrlInstance.facts)
return newFact
def runFromExcel(self, options):
testGenFileName = options.excelfilename
#testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls"
testGenDir = os.path.dirname(testGenFileName)
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt'
modelTestcases = ModelXbrl.create(self.modelManager)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(
_("[info] xls loaded in {0:.2} secs at {1}").format(
time.time() - startedAt, timeNow))
# start index file
indexFiles = [
testGenDir + os.sep + 'creationTestcasesIndex.xml',
testGenDir + os.sep + 'consumptionTestcasesIndex.xml'
]
indexDocs = [
xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today))
for purpose in ("Creation", "Consumption")
]
testcasesElements = [
XmlUtil.child(indexDoc, None, "testcases")
for indexDoc in indexDocs
]
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
row = testcaseIndexSheet.row(iRow)
if row[0].ctype == xlrd.XL_CELL_EMPTY or row[
1].ctype == xlrd.XL_CELL_EMPTY or row[
2].ctype == xlrd.XL_CELL_EMPTY:
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
intention = row[3].value
if intention is None or len(intention) == 0:
continue # test not ready to run
reason = row[4].value
expectedEvent = row[5].value
base = os.path.join(os.path.dirname(testGenFileName),
testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and reason.lower() not in (
"n.a.", "error") and expectedEvent != "N.A.":
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"),
True), (uriTo, _("loading to DTS"),
False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager,
URIs,
msg,
base=base)
else:
modelDTS = ModelXbrl.create(
self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(
URIs.replace(", ", "_") + ".dts", base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS,
uri.strip(),
base=base)
DTSdoc.referencesDocument[
doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom and modelDTSto:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace(
"from.xsd", "report.xml")
reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile, base)
testcasesDir = os.path.dirname(
os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i, testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w",
encoding="utf-8") as fh:
XmlUtil.writexml(fh,
testcaseDocs[i],
encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [
testcasesDir + os.sep + testcaseName +
"-creation-testcase.xml", testcasesDir + os.sep +
testcaseName + "-consumption-testcase.xml"
]
for i, testcaseFile in enumerate(testcaseFiles):
XmlUtil.addChild(
testcasesElements[i], None, "testcase",
("uri", testcaseFile[len(testGenDir) +
1:].replace("\\", "/")))
# start testcase file
testcaseDocs = [
xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'<name>{0}</name>'
'<description>{0}</description>'
'</testcase>'.format(testcaseName, purpose,
today))
for purpose in ("Creation", "Consumption")
]
testcaseElements = [
XmlUtil.child(testcaseDoc, conformanceNS,
"testcase")
for testcaseDoc in testcaseDocs
]
priorTestcasesDir = testcasesDir
variationID = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(
modelTestcases)
modelVersReport.diffDTSes(reportFullPath, modelDTSfrom,
modelDTSto)
# check for expected elements
if expectedEvent and expectedEvent not in ("No change",
"N.A."):
if len(
modelVersReport.xmlDocument.
getElementsByTagNameNS('*',
expectedEvent)) == 0:
modelTestcases.error(
"Generated test case {0} missing expected event {1}"
.format(reportName,
expectedEvent), "wrn", "missingEvent")
modelVersReport.close([])
for i, testcaseElt in enumerate(testcaseElements):
variationElement = XmlUtil.addChild(
testcaseElt,
conformanceNS,
"variation",
attributes=("id", "_{0:02n}".format(variationID)))
XmlUtil.addChild(variationElement,
conformanceNS,
"name",
text=intention)
dataElement = XmlUtil.addChild(variationElement,
conformanceNS, "data")
for schemaURIs, dtsAttr in ((uriFrom, "from"), (uriTo,
"to")):
for schemaURI in schemaURIs.split(","):
XmlUtil.addChild(
dataElement,
conformanceNS,
"schema",
attributes=((("dts", dtsAttr), ) +
((("readMeFirst",
"true"), ) if i == 0 else
())),
text=os.path.basename(schemaURI.strip()))
resultElement = XmlUtil.addChild(
variationElement, conformanceNS, "result")
XmlUtil.addChild(
resultElement if i == 0 else dataElement,
conformanceNS,
"versioningReport",
attributes=(("readMeFirst", "true") if i == 1 else
()),
text="report/" + reportName)
variationID += 1
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i, testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i, indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def runFromExcel(self, options):
testGenFileName = options.excelfilename
#testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls"
testGenDir = os.path.dirname(testGenFileName)
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt'
modelTestcases = ModelXbrl.create(self.modelManager)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow))
# start index file
indexFiles = [testGenDir + os.sep + 'creationTestcasesIndex.xml',
testGenDir + os.sep + 'consumptionTestcasesIndex.xml']
indexDocs = [xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today)
) for purpose in ("Creation","Consumption")]
testcasesElements = [XmlUtil.child(indexDoc, None, "testcases") for indexDoc in indexDocs]
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
row = testcaseIndexSheet.row(iRow)
if row[0].ctype == xlrd.XL_CELL_EMPTY or row[1].ctype == xlrd.XL_CELL_EMPTY or row[2].ctype == xlrd.XL_CELL_EMPTY:
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
intention = row[3].value
if intention is None or len(intention) == 0:
continue # test not ready to run
reason = row[4].value
expectedEvent = row[5].value
base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and reason.lower() not in ("n.a.", "error") and expectedEvent != "N.A.":
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base)
else:
modelDTS = ModelXbrl.create(self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts",
base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS, uri.strip(), base=base)
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom and modelDTSto:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace("from.xsd","report.xml")
reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile,
base)
testcasesDir = os.path.dirname(os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml",
testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"]
for i,testcaseFile in enumerate(testcaseFiles):
XmlUtil.addChild(testcasesElements[i], None, "testcase",
("uri",
testcaseFile[len(testGenDir)+1:].replace("\\","/")) )
# start testcase file
testcaseDocs = [xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'<name>{0}</name>'
'<description>{0}</description>'
'</testcase>'.format(testcaseName,purpose,today)
) for purpose in ("Creation","Consumption")]
testcaseElements = [XmlUtil.child(testcaseDoc, conformanceNS, "testcase") for testcaseDoc in testcaseDocs]
priorTestcasesDir = testcasesDir
variationID = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(modelTestcases)
modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto)
# check for expected elements
if expectedEvent and expectedEvent not in (
"No change", "N.A."):
if len(modelVersReport.xmlDocument.getElementsByTagNameNS('*',expectedEvent)) == 0:
modelTestcases.error(
"Generated test case {0} missing expected event {1}".format(
reportName,
expectedEvent),
"wrn", "missingEvent")
modelVersReport.close([])
for i,testcaseElt in enumerate(testcaseElements):
variationElement = XmlUtil.addChild(testcaseElt, conformanceNS, "variation",
attributes=("id", "_{0:02n}".format(variationID)))
XmlUtil.addChild(variationElement, conformanceNS, "name", text=intention)
dataElement = XmlUtil.addChild(variationElement, conformanceNS, "data")
for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")):
for schemaURI in schemaURIs.split(","):
XmlUtil.addChild(dataElement, conformanceNS, "schema",
attributes=((("dts",dtsAttr),) +
((("readMeFirst","true"),) if i == 0 else ())),
text=os.path.basename(schemaURI.strip()))
resultElement = XmlUtil.addChild(variationElement, conformanceNS, "result")
XmlUtil.addChild(resultElement if i == 0 else dataElement,
conformanceNS, "versioningReport",
attributes=(("readMeFirst","true") if i == 1 else ()),
text="report/" + reportName)
variationID += 1
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i,indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def produceOutputFact(xpCtx, formula, result):
priorErrorCount = len(xpCtx.modelXbrl.errors)
# assemble context
conceptQname = aspectValue(xpCtx, formula, Aspect.CONCEPT,
"xbrlfe:missingConceptRule")
if isinstance(conceptQname, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} concept: {1}").format(formula, conceptQname.msg),
"err", conceptQname.err)
modelConcept = None
else:
modelConcept = xpCtx.modelXbrl.qnameConcepts[conceptQname]
if modelConcept is None or not modelConcept.isItem:
xpCtx.modelXbrl.error(
_("Formula {0} concept {1} is not an item").format(
formula, conceptQname), "err", "xbrlfe:missingConceptRule")
# entity
entityIdentScheme = aspectValue(xpCtx, formula, Aspect.SCHEME,
"xbrlfe:missingEntityIdentifierRule")
if isinstance(entityIdentScheme, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} entity identifier scheme: {1}").format(
formula, entityIdentScheme.msg), "err", str(entityIdentScheme))
entityIdentValue = None
else:
entityIdentValue = aspectValue(xpCtx, formula, Aspect.VALUE,
"xbrlfe:missingEntityIdentifierRule")
if isinstance(entityIdentValue, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} entity identifier value: {1}").format(
formula, entityIdentValue.msg), "err",
str(entityIdentScheme))
# period
periodType = aspectValue(xpCtx, formula, Aspect.PERIOD_TYPE,
"xbrlfe:missingPeriodRule")
periodStart = None
periodEndInstant = None
if isinstance(periodType, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} period type: {1}").format(formula, periodType.msg),
"err", str(periodType))
elif periodType == "instant":
periodEndInstant = aspectValue(xpCtx, formula, Aspect.INSTANT,
"xbrlfe:missingPeriodRule")
if isinstance(periodEndInstant, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} period start: {1}").format(
formula, periodEndInstant.msg), "err",
str(periodEndInstant))
elif periodType == "duration":
periodStart = aspectValue(xpCtx, formula, Aspect.START,
"xbrlfe:missingPeriodRule")
if isinstance(periodStart, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} period start: {1}").format(
formula, periodStart.msg), "err", str(periodStart))
periodEndInstant = aspectValue(xpCtx, formula, Aspect.END,
"xbrlfe:missingPeriodRule")
if isinstance(periodEndInstant, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} period end: {1}").format(
formula, periodEndInstant.msg), "err",
str(periodEndInstant))
# unit
if modelConcept and modelConcept.isNumeric:
unitSource = aspectValue(xpCtx, formula, Aspect.UNIT_MEASURES, None)
multDivBy = aspectValue(xpCtx, formula, Aspect.MULTIPLY_BY,
"xbrlfe:missingUnitRule")
if isinstance(multDivBy, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} unit: {1}").format(formula,
multDivBy.msg), "err",
str(multDivBy) if isinstance(multDivBy, VariableBindingError)
else "xbrlfe:missingUnitRule")
multiplyBy = ()
divideBy = () # prevent errors later if bad
else:
divMultBy = aspectValue(xpCtx, formula, Aspect.DIVIDE_BY,
"xbrlfe:missingUnitRule")
if isinstance(divMultBy, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} unit: {1}").format(formula,
divMultBy.msg), "err",
str(multDivBy)
if isinstance(divMultBy, VariableBindingError) else
"xbrlfe:missingUnitRule")
multiplyBy = ()
divideBy = () # prevent errors later if bad
else:
multiplyBy = unitSource[0] + multDivBy[0] + divMultBy[1]
divideBy = unitSource[1] + multDivBy[1] + divMultBy[0]
# remove cancelling mult/div units
lookForCommonUnits = True
while lookForCommonUnits:
lookForCommonUnits = False
for commonUnit in multiplyBy:
if commonUnit in divideBy:
multiplyBy.remove(commonUnit)
divideBy.remove(commonUnit)
lookForCommonUnits = True
break
if len(multiplyBy) == 0: # if no units add pure
multiplyBy.append(XbrlConst.qnXbrliPure)
# dimensions
segOCCs = []
scenOCCs = []
if formula.aspectModel == "dimensional":
dimAspects = {}
dimQnames = aspectValue(xpCtx, formula, Aspect.DIMENSIONS, None)
if dimQnames:
for dimQname in dimQnames:
dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname]
dimErr = "xbrlfe:missing{0}DimensionRule".format(
"typed" if dimConcept and dimConcept.isTypedDimension else
"explicit")
dimValue = aspectValue(xpCtx, formula, dimQname, dimErr)
if isinstance(dimValue, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} dimension {1}: {2}").format(
formula, dimQname, dimValue.msg), "err", dimErr)
elif dimValue and xpCtx.modelXbrl.qnameDimensionDefaults.get(
dimQname) != dimValue:
dimAspects[dimQname] = dimValue
segOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SEGMENT, None)
scenOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SCENARIO, None)
else:
dimAspects = None # non-dimensional
segOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SEGMENT, None)
scenOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SCENARIO, None)
if priorErrorCount < len(xpCtx.modelXbrl.errors):
return None # had errors, don't produce output fact
# does context exist in out instance document
outputInstanceQname = formula.outputInstanceQname
outputXbrlInstance = xpCtx.inScopeVars[outputInstanceQname]
xbrlElt = outputXbrlInstance.modelDocument.xmlRootElement
# in source instance document
# add context
prevCntx = outputXbrlInstance.matchContext(entityIdentScheme,
entityIdentValue, periodType,
periodStart, periodEndInstant,
dimAspects, segOCCs, scenOCCs)
if prevCntx:
cntxId = prevCntx.id
newCntxElt = prevCntx.element
else:
cntxId = 'c-{0:02n}'.format(len(outputXbrlInstance.contexts) + 1)
newCntxElt = XmlUtil.addChild(
xbrlElt,
XbrlConst.xbrli,
"context",
attributes=("id", cntxId),
afterSibling=xpCtx.outputLastContext.get(outputInstanceQname))
xpCtx.outputLastContext[outputInstanceQname] = newCntxElt
entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity")
XmlUtil.addChild(entityElt,
XbrlConst.xbrli,
"identifier",
attributes=("scheme", entityIdentScheme),
text=entityIdentValue)
periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period")
if periodType == "forever":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever")
elif periodType == "instant":
XmlUtil.addChild(periodElt,
XbrlConst.xbrli,
"instant",
text=XmlUtil.dateunionValue(periodEndInstant,
subtractOneDay=True))
elif periodType == "duration":
XmlUtil.addChild(periodElt,
XbrlConst.xbrli,
"startDate",
text=XmlUtil.dateunionValue(periodStart))
XmlUtil.addChild(periodElt,
XbrlConst.xbrli,
"endDate",
text=XmlUtil.dateunionValue(periodEndInstant,
subtractOneDay=True))
segmentElt = None
scenarioElt = None
from arelle.ModelObject import ModelDimensionValue
if dimAspects:
for dimQname in sorted(dimAspects.keys()):
dimValue = dimAspects[dimQname]
if isinstance(dimValue, ModelDimensionValue):
if dimValue.isExplicit:
dimMemberQname = dimValue.memberQname
contextEltName = dimValue.contextElement
else: # qname for explicit or node for typed
dimMemberQname = dimValue
contextEltName = xpCtx.modelXbrl.qnameDimensionContextElement.get(
dimQname)
if contextEltName == "segment":
if not segmentElt:
segmentElt = XmlUtil.addChild(entityElt,
XbrlConst.xbrli,
"segment")
contextElt = segmentElt
elif contextEltName == "scenario":
if not scenarioElt:
scenarioElt = XmlUtil.addChild(newCntxElt,
XbrlConst.xbrli,
"scenario")
contextElt = scenarioElt
else:
continue
dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname]
dimAttr = ("dimension",
XmlUtil.addQnameValue(xbrlElt, dimConcept.qname))
if dimConcept.isTypedDimension:
dimElt = XmlUtil.addChild(contextElt,
XbrlConst.xbrldi,
"typedMember",
attributes=dimAttr)
if isinstance(dimValue,
ModelDimensionValue) and dimValue.isTyped:
XmlUtil.copyChildren(dimElt, dimValue.typedMember)
elif dimMemberQname:
dimElt = XmlUtil.addChild(contextElt,
XbrlConst.xbrldi,
"explicitMember",
attributes=dimAttr,
text=XmlUtil.addQnameValue(
xbrlElt, dimMemberQname))
if segOCCs:
if not segmentElt:
segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli,
"segment")
XmlUtil.copyNodes(segmentElt, segOCCs)
if scenOCCs:
if not scenarioElt:
scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli,
"scenario")
XmlUtil.copyNodes(scenarioElt, scenOCCs)
outputXbrlInstance.modelDocument.contextDiscover(newCntxElt)
# does unit exist
# add unit
if modelConcept.isNumeric:
prevUnit = outputXbrlInstance.matchUnit(multiplyBy, divideBy)
if prevUnit:
unitId = prevUnit.id
newUnitElt = prevUnit.element
else:
unitId = 'u-{0:02n}'.format(len(outputXbrlInstance.units) + 1)
newUnitElt = XmlUtil.addChild(
xbrlElt,
XbrlConst.xbrli,
"unit",
attributes=("id", unitId),
afterSibling=xpCtx.outputLastUnit.get(outputInstanceQname))
xpCtx.outputLastUnit[outputInstanceQname] = newUnitElt
if len(divideBy) == 0:
for multiply in multiplyBy:
XmlUtil.addChild(newUnitElt,
XbrlConst.xbrli,
"measure",
text=XmlUtil.addQnameValue(
xbrlElt, multiply))
else:
divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli,
"divide")
numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli,
"unitNumerator")
denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli,
"unitDenominator")
for multiply in multiplyBy:
XmlUtil.addChild(numElt,
XbrlConst.xbrli,
"measure",
text=XmlUtil.addQnameValue(
xbrlElt, multiply))
for divide in divideBy:
XmlUtil.addChild(denElt,
XbrlConst.xbrli,
"measure",
text=XmlUtil.addQnameValue(
xbrlElt, divide))
outputXbrlInstance.modelDocument.unitDiscover(newUnitElt)
# add fact
attrs = [("contextRef", cntxId)]
precision = None
decimals = None
if modelConcept.isNumeric:
attrs.append(("unitRef", unitId))
value = formula.evaluate(xpCtx)
valueSeqLen = len(value)
if valueSeqLen > 1:
xpCtx.modelXbrl.error(
_("Formula {0} value is a sequence of length {1}").format(
formula, valueSeqLen), "err", "xbrlfe:nonSingletonOutputValue")
else:
if valueSeqLen == 0: #xsi:nil if no value
attrs.append((XbrlConst.qnXsiNil, "true"))
v = None
else:
# add precision/decimals for non-fraction numerics
if modelConcept.isNumeric and not modelConcept.isFraction:
if formula.hasDecimals:
decimals = formula.evaluateRule(xpCtx, Aspect.DECIMALS)
attrs.append(("decimals", decimals))
else:
if formula.hasPrecision:
precision = formula.evaluateRule(
xpCtx, Aspect.PRECISION)
else:
precision = 0
attrs.append(("precision", precision))
x = value[0]
if isinstance(x, float):
from math import (log10, isnan, isinf, fabs)
if (isnan(x)
or (precision and (isinf(precision) or precision == 0))
or (decimals and isinf(decimals))):
v = string(xpCtx, x)
elif decimals is not None:
v = "%.*f" % (int(decimals), x)
elif precision is not None:
a = fabs(x)
log = log10(a) if a != 0 else 0
v = "%.*f" % (int(precision) - int(log) -
(1 if a >= 1 else 0), x)
else: # no implicit precision yet
v = string(xpCtx, x)
elif isinstance(x, QName):
v = XmlUtil.addQnameValue(xbrlElt, x)
elif isinstance(x, datetime.datetime):
v = XmlUtil.dateunionValue(x)
else:
v = string(xpCtx, x)
itemElt = XmlUtil.addChild(
xbrlElt,
conceptQname,
attributes=attrs,
text=v,
afterSibling=xpCtx.outputLastFact.get(outputInstanceQname))
xpCtx.outputLastFact[outputInstanceQname] = itemElt
newFact = outputXbrlInstance.modelDocument.factDiscover(
itemElt, outputXbrlInstance.facts)
return newFact
def loadFromExcel(cntlr, excelFile):
from arelle import xlrd
from arelle.xlrd.sheet import empty_cell
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
startedAt = time.time()
importExcelBook = xlrd.open_workbook(excelFile)
controlSheet = importExcelBook.sheet_by_index(1)
imports = {"xbrli": ( ("namespace", XbrlConst.xbrli),
("schemaLocation", "http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd") )} # xml of imports
importXmlns = {}
linkbaseRefs = []
labelLinkbases = []
hasPreLB = hasCalLB = hasDefLB = False
# xxxLB structure [ (elr1, def1, "_ELR_", [roots]), (elr2, def2, "_ELR_", [rootw]) ...]
# roots = (rootHref, None, "_root_", [children])
# children = (childPrefix, childName, arcrole, [grandChildren])
preLB = []
defLB = []
calLB = []
def lbDepthList(lbStruct, depth, parentList=None):
if depth == 0:
return lbStruct[-1].childStruct
return lbDepthList(lbStruct[-1].childStruct, depth-1, list)
extensionElements = {}
extensionRoles = {} # key is roleURI, value is role definition
extensionLabels = {} # key = (prefix, name, lang, role), value = label text
def extensionHref(prefix, name):
if prefix == extensionSchemaPrefix:
filename = extensionSchemaFilename
elif prefix in imports:
filename = imports[prefix][1][1]
else:
return None
return "{0}#{1}_{2}".format(filename, prefix, name)
isUSGAAP = False
for iRow in range(1, controlSheet.nrows):
try:
row = controlSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY): # skip if col 1 is empty
continue
action = row[0].value
filetype = row[1].value
prefix = row[2].value
filename = row[3].value
namespaceURI = row[4].value
lbType = lang = None
if action == "import":
imports[prefix] = ( ("namespace", namespaceURI), ("schemaLocation", filename) )
importXmlns[prefix] = namespaceURI
if re.match(r"http://[^/]+/us-gaap/", namespaceURI):
isUSGAAP = True
elif action == "extension":
if filetype == "schema":
extensionSchemaPrefix = prefix
extensionSchemaFilename = filename
extensionSchemaNamespaceURI = namespaceURI
elif filetype == "linkbase":
typeLang = prefix.split()
if len(typeLang) > 0:
lbType = typeLang[0]
else:
lbType = "unknown"
if len(typeLang) > 1:
lang = typeLang[1]
else:
lang = "en"
if lbType == "label":
labelLinkbases.append((lang, filename))
elif lbType == "presentation":
hasPreLB = True
elif lbType == "definition":
hasDefLB = True
elif lbType == "calculation":
hasCalLB = True
linkbaseRefs.append( (lbType, filename) )
elif filetype == "role" and namespaceURI:
extensionRoles[namespaceURI] = filename
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:exception")
importExcelSheet = importExcelBook.sheet_by_index(0)
# find column headers row
headerCols = {}
headerRows = set()
# find out which rows are header rows
for iRow in range(0, importExcelSheet.nrows):
row = importExcelSheet.row(iRow)
for iCol, colCell in enumerate(row):
v = colCell.value
if v in importColumnHeaders:
headerCols[importColumnHeaders[v]] = iCol
if all(colName in headerCols
for colName in ("name", "type", "depth")): # must have these to be a header col
# it's a header col
headerRows.add(iRow)
headerCols.clear()
def cellValue(row, header):
if header in headerCols:
iCol = headerCols[header]
if iCol < len(row):
return row[iCol].value
return ''
def checkImport(qname):
prefix, sep, localName = qname.partition(":")
if sep:
if prefix not in imports:
if prefix == "xbrldt":
imports["xbrldt"] = ("namespace", XbrlConst.xbrldt), ("schemaLocation", "http://www.xbrl.org/2005/xbrldt-2005.xsd")
elif prefix == "nonnum":
imports["nonnum"] = ("namespace", "http://www.xbrl.org/dtr/type/non-numeric"), ("schemaLocation", "http://www.xbrl.org/dtr/type/nonNumeric-2009-12-16.xsd")
else:
cntlr.addToLog("Warning: prefix schema file is not imported for: {qname}"
.format(qname=qname),
messageCode="importExcel:warning")
# find header rows
currentELR = currentELRdefinition = None
for iRow in range(0, importExcelSheet.nrows):
useLabels = False
try:
row = importExcelSheet.row(iRow)
isHeaderRow = iRow in headerRows
isELRrow = (iRow + 1) in headerRows
if isHeaderRow:
headerCols.clear()
for iCol, colCell in enumerate(row):
v = colCell.value
if v in importColumnHeaders:
headerCols[importColumnHeaders[v]] = iCol
elif isELRrow:
currentELR = currentELRdefinition = None
for colCell in row:
v = colCell.value
if v.startswith("http://"):
currentELR = v
elif not currentELRdefinition and v.endswith(" 科目一覧"):
currentELRdefinition = v[0:-5]
elif not currentELRdefinition:
currentELRdefinition = v
if currentELR or currentELRdefinition:
if hasPreLB:
preLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasDefLB:
defLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasCalLB:
calLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
elif headerCols:
prefix = cellValue(row, 'prefix').strip()
name = cellValue(row, 'name').strip()
if "depth" in headerCols:
try:
depth = int(cellValue(row, 'depth'))
except ValueError:
depth = None
else:
depth = None
if prefix == extensionSchemaPrefix and name not in extensionElements:
# elements row
eltType = cellValue(row, 'type')
subsGrp = cellValue(row, 'substitutionGroup')
abstract = cellValue(row, 'abstract')
nillable = cellValue(row, 'nillable')
balance = cellValue(row, 'balance')
periodType = cellValue(row, 'periodType')
newElt = [ ("name", name), ("id", prefix + "_" + name) ]
if eltType:
newElt.append( ("type", eltType) )
checkImport(eltType)
if subsGrp:
newElt.append( ("substitutionGroup", subsGrp) )
checkImport(subsGrp)
if abstract:
newElt.append( ("abstract", abstract) )
if nillable:
newElt.append( ("nillable", nillable) )
if balance:
newElt.append( ("{http://www.xbrl.org/2003/instance}balance", balance) )
if periodType:
newElt.append( ("{http://www.xbrl.org/2003/instance}periodType", periodType) )
extensionElements[name] = newElt
useLabels = True
if depth is not None:
if hasPreLB:
entryList = lbDepthList(preLB, depth)
preferredLabel = cellValue(row, 'preferredLabel')
if preferredLabel and not preferredLabel.startswith("http://"):
preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel
if entryList is not None:
if depth == 0:
entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) )
else:
entryList.append( LBentry(prefix=prefix, name=name, arcrole=XbrlConst.parentChild,
role=preferredLabel) )
if hasDefLB:
entryList = lbDepthList(defLB, depth)
if entryList is not None:
if depth == 0:
entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) )
else:
entryList.append( LBentry(prefix=prefix, name=name, arcrole="_dimensions_") )
if hasCalLB:
calcParent = cellValue(row, 'calculationParent')
calcWeight = cellValue(row, 'calculationWeight')
if calcParent and calcWeight:
calcParentPrefix, sep, calcParentName = calcParent.partition(":")
entryList = lbDepthList(calLB, 0)
if entryList is not None:
entryList.append( LBentry(prefix=calcParentPrefix, name=calcParentName, isRoot=True, childStruct=
[LBentry(prefix=prefix, name=name, arcrole=XbrlConst.summationItem, weight=calcWeight )]) )
# accumulate extension labels
if useLabels:
prefix = cellValue(row, 'prefix').strip()
name = cellValue(row, 'name').strip()
preferredLabel = cellValue(row, 'preferredLabel')
if preferredLabel and not preferredLabel.startswith("http://"):
preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel
for colItem, iCol in headerCols.items():
if isinstance(colItem, tuple):
colItemType, role, lang = colItem
cell = row[iCol]
if cell.ctype == xlrd.XL_CELL_EMPTY:
values = ()
elif colItemType == "label":
values = (cell.value,)
elif colItemType == "labels":
values = cell.value.split('\n')
else:
values = ()
if preferredLabel: # first label column sets preferredLabel if any
role = preferredLabel
preferredLabel = None
for value in values:
extensionLabels[prefix, name, lang, role] = value.strip()
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:exception")
if isUSGAAP and hasDefLB:
# move line items above table
def fixUsggapTableDims(lvl1Struct):
foundLineItems = False
for i1, lvl1Entry in enumerate(lvl1Struct):
for i2, lvl2Entry in enumerate(lvl1Entry.childStruct):
for i3, lvl3Entry in enumerate(lvl2Entry.childStruct):
if lvl3Entry.name.endswith("LineItems") and lvl2Entry.name.endswith("Table"):
foundLineItems = True
break
if foundLineItems:
break
else:
fixUsggapTableDims(lvl1Entry.childStruct)
if foundLineItems:
lvl1Struct.insert(i1 + 1, LBentry(prefix=lvl3Entry.prefix, name=lvl3Entry.name, arcrole=lvl1Entry.arcrole, childStruct=lvl3Entry.childStruct)) # must keep lvl1Rel if it is __root__
lvl3Entry.childStruct.insert(0, lvl2Entry)
if lvl1Entry.name.endswith("Abstract"):
del lvl1Struct[i1]
del lvl2Entry.childStruct[i3]
pass
fixUsggapTableDims(defLB)
dts = cntlr.modelManager.create(newDocumentType=ModelDocument.Type.SCHEMA,
url=extensionSchemaFilename,
isEntry=True,
base='', # block pathname from becomming absolute
initialXml='''
<schema xmlns="http://www.w3.org/2001/XMLSchema"
targetNamespace="{targetNamespace}"
attributeFormDefault="unqualified"
elementFormDefault="qualified"
xmlns:xsd="http://www.w3.org/2001/XMLSchema"
xmlns:{extensionPrefix}="{targetNamespace}"
{importXmlns}
xmlns:iod="http://disclosure.edinet-fsa.go.jp/taxonomy/common/2013-03-31/iod"
xmlns:nonnum="http://www.xbrl.org/dtr/type/non-numeric"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xbrli="http://www.xbrl.org/2003/instance"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrldt="http://xbrl.org/2005/xbrldt"/>
'''.format(targetNamespace=extensionSchemaNamespaceURI,
extensionPrefix=extensionSchemaPrefix,
importXmlns=''.join('xmlns:{0}="{1}"\n'.format(prefix, namespaceURI)
for prefix, namespaceURI in importXmlns.items())
)
)
dtsSchemaDocument = dts.modelDocument
dtsSchemaDocument.inDTS = True # entry document always in DTS
dtsSchemaDocument.targetNamespace = extensionSchemaNamespaceURI # not set until schemaDiscover too late otherwise
schemaElt = dtsSchemaDocument.xmlRootElement
#foreach linkbase
annotationElt = XmlUtil.addChild(schemaElt, XbrlConst.xsd, "annotation")
appinfoElt = XmlUtil.addChild(annotationElt, XbrlConst.xsd, "appinfo")
for iRow in range(0, importExcelSheet.nrows):
try:
row = importExcelSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY): # skip if col 1 is empty
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="loadFromExcel:exception")
# add linkbaseRefs
appinfoElt = XmlUtil.descendant(schemaElt, XbrlConst.xsd, "appinfo")
# don't yet add linkbase refs, want to process imports first to get roleType definitions
# add imports
for importAttributes in sorted(imports.values()):
XmlUtil.addChild(schemaElt,
XbrlConst.xsd, "import",
attributes=importAttributes)
# add elements
for eltName, eltAttrs in sorted(extensionElements.items(), key=lambda item: item[0]):
XmlUtil.addChild(schemaElt,
XbrlConst.xsd, "element",
attributes=eltAttrs)
# add role definitions (for discovery)
for roleURI, roleDefinition in extensionRoles.items():
roleElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "roleType",
attributes=(("roleURI", roleURI),
("id", "roleType_" + roleURI.rpartition("/")[2])))
if roleDefinition:
XmlUtil.addChild(roleElt, XbrlConst.link, "definition", text=roleDefinition)
if hasPreLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:presentationLink")
if hasDefLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:definitionLink")
if hasCalLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:calculationLink")
dtsSchemaDocument.schemaDiscover(schemaElt, False, extensionSchemaNamespaceURI)
def addLinkbaseRef(lbType, lbFilename, lbDoc):
role = "http://www.xbrl.org/2003/role/{0}LinkbaseRef".format(lbType)
lbRefElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "linkbaseRef",
attributes=(("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", lbFilename),
("{http://www.w3.org/1999/xlink}role", role),
("{http://www.w3.org/1999/xlink}arcrole", "http://www.w3.org/1999/xlink/properties/linkbase"),
))
dtsSchemaDocument.referencesDocument[lbDoc] = ModelDocumentReference("href", lbRefElt)
# label linkbase
for lang, filename in labelLinkbases:
lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base="", initialXml="""
<link:linkbase
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase
http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance"/>
""")
lbDoc.inDTS = True
addLinkbaseRef("label", filename, lbDoc)
lbElt = lbDoc.xmlRootElement
linkElt = XmlUtil.addChild(lbElt,
XbrlConst.link, "labelLink",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/link")))
locs = set()
for labelKey, text in extensionLabels.items():
prefix, name, labelLang, role = labelKey
if lang == labelLang:
locLabel = prefix + "_" + name
if locLabel not in locs:
locs.add(locLabel)
XmlUtil.addChild(linkElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", extensionHref(prefix, name)),
("{http://www.w3.org/1999/xlink}label", locLabel)))
XmlUtil.addChild(linkElt,
XbrlConst.link, "labelArc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", "http://www.xbrl.org/2003/arcrole/concept-label"),
("{http://www.w3.org/1999/xlink}from", locLabel),
("{http://www.w3.org/1999/xlink}to", "label_" + locLabel),
("order", 1.0)))
XmlUtil.addChild(linkElt,
XbrlConst.link, "label",
attributes=(("{http://www.w3.org/1999/xlink}type", "resource"),
("{http://www.w3.org/1999/xlink}label", "label_" + locLabel),
("{http://www.w3.org/1999/xlink}role", role),
("{http://www.w3.org/XML/1998/namespace}lang", lang)),
text=text)
lbDoc.linkbaseDiscover(lbElt)
def hrefConcept(prefix, name):
qn = schemaElt.prefixedNameQname(prefix + ":" + name)
if qn in dts.qnameConcepts:
return dts.qnameConcepts[qn]
return None
def lbTreeWalk(lbType, parentElt, lbStruct, roleRefs, locs=None, fromPrefix=None, fromName=None):
order = 1.0
for lbEntry in lbStruct:
if lbEntry.isELR:
role = "unspecified"
if lbEntry.role and lbEntry.role.startswith("http://"): # have a role specified
role = lbEntry.role
elif lbEntry.name: #may be a definition
for linkroleUri, modelRoleTypes in dts.roleTypes.items():
definition = modelRoleTypes[0].definition
if lbEntry.name == definition:
role = linkroleUri
break
if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef
roleType = modelRoleTypes[0]
roleRef = ("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id)
roleRefs.add(roleRef)
linkElt = XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Link",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", role)))
locs = set()
lbTreeWalk(lbType, linkElt, lbEntry.childStruct, roleRefs, locs)
else:
toPrefix = lbEntry.prefix
toName = lbEntry.name
toHref = extensionHref(toPrefix, toName)
toLabel = toPrefix + "_" + toName
if toHref not in locs:
XmlUtil.addChild(parentElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", toHref),
("{http://www.w3.org/1999/xlink}label", toLabel)))
locs.add(toHref)
if not lbEntry.isRoot:
fromLabel = fromPrefix + "_" + fromName
if lbType == "calculation" and lbEntry.weight is not None:
otherAttrs = ( ("weight", lbEntry.weight), )
elif lbType == "presentation" and lbEntry.role is not None:
otherAttrs = ( ("preferredLabel", lbEntry.role), )
else:
otherAttrs = ( )
if lbEntry.arcrole == "_dimensions_": # pick proper consecutive arcrole
fromConcept = hrefConcept(fromPrefix, fromName)
toConcept = hrefConcept(toPrefix, toName)
if toConcept is not None and toConcept.isHypercubeItem:
arcrole = XbrlConst.all
elif toConcept is not None and toConcept.isDimensionItem:
arcrole = XbrlConst.hypercubeDimension
elif fromConcept is not None and fromConcept.isDimensionItem:
arcrole = XbrlConst.dimensionDomain
else:
arcrole = XbrlConst.domainMember
else:
arcrole = lbEntry.arcrole
XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Arc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", arcrole),
("{http://www.w3.org/1999/xlink}from", fromLabel),
("{http://www.w3.org/1999/xlink}to", toLabel),
("order", order)) + otherAttrs )
order += 1.0
if lbType != "calculation" or lbEntry.isRoot:
lbTreeWalk(lbType, parentElt, lbEntry.childStruct, roleRefs, locs, toPrefix, toName)
for hasLB, lbType, lbLB in ((hasPreLB, "presentation", preLB),
(hasDefLB, "definition", defLB),
(hasCalLB, "calculation", calLB)):
if hasLB:
for lbRefType, filename in linkbaseRefs:
if lbType == lbRefType:
# output presentation linkbase
lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base='', initialXml="""
<link:linkbase
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase
http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance"/>
""")
lbDoc.inDTS = True
addLinkbaseRef(lbRefType, filename, lbDoc)
lbElt = lbDoc.xmlRootElement
roleRefs = set()
if lbType == "definition":
roleRefs.update((("arcroleRef", XbrlConst.all, "http://www.xbrl.org/2005/xbrldt-2005.xsd#all"),
("arcroleRef", XbrlConst.dimensionDefault, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-default"),
("arcroleRef", XbrlConst.dimensionDomain, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-domain"),
("arcroleRef", XbrlConst.domainMember, "http://www.xbrl.org/2005/xbrldt-2005.xsd#domain-member"),
("arcroleRef", XbrlConst.hypercubeDimension, "http://www.xbrl.org/2005/xbrldt-2005.xsd#hypercube-dimension")))
lbTreeWalk(lbType, lbElt, lbLB, roleRefs)
firstLinkElt = None
for firstLinkElt in lbElt.iterchildren():
break
# add arcrole references
for roleref, roleURI, href in roleRefs:
XmlUtil.addChild(lbElt,
XbrlConst.link, roleref,
attributes=(("arcroleURI" if roleref == "arcroleRef" else "roleURI", roleURI),
("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", href)),
beforeSibling=firstLinkElt)
lbDoc.linkbaseDiscover(lbElt)
break
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
return dts
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in modelXbrl.contexts.values():
newCntx = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in modelXbrl.facts:
if fact.isItem:
attrs = {"contextRef": fact.contextID}
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
newFactForOldObjId[fact.objectIndex] = newFact
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
# footnote links
modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships"))
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
newLink = XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, linkqname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
if isinstance(linkChild, LocPrototype) and "{http://www.w3.org/1999/xlink}href" not in linkChild.attributes:
linkChild.attributes["{http://www.w3.org/1999/xlink}href"] = \
"#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
XmlUtil.addChild(newLink, linkChild.qname,
attributes=linkChild.attributes,
text=linkChild.textValue)
targetInstance.saveInstance(overrideFilepath=targetUrl)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def lbTreeWalk(lbType, parentElt, lbStruct, roleRefs, locs=None, fromPrefix=None, fromName=None):
order = 1.0
for lbEntry in lbStruct:
if lbEntry.isELR:
role = "unspecified"
if lbEntry.role and lbEntry.role.startswith("http://"): # have a role specified
role = lbEntry.role
elif lbEntry.name: #may be a definition
for linkroleUri, modelRoleTypes in dts.roleTypes.items():
definition = modelRoleTypes[0].definition
if lbEntry.name == definition:
role = linkroleUri
break
if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef
roleType = modelRoleTypes[0]
roleRef = ("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id)
roleRefs.add(roleRef)
linkElt = XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Link",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", role)))
locs = set()
lbTreeWalk(lbType, linkElt, lbEntry.childStruct, roleRefs, locs)
else:
toPrefix = lbEntry.prefix
toName = lbEntry.name
toHref = extensionHref(toPrefix, toName)
toLabel = toPrefix + "_" + toName
if toHref not in locs:
XmlUtil.addChild(parentElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", toHref),
("{http://www.w3.org/1999/xlink}label", toLabel)))
locs.add(toHref)
if not lbEntry.isRoot:
fromLabel = fromPrefix + "_" + fromName
if lbType == "calculation" and lbEntry.weight is not None:
otherAttrs = ( ("weight", lbEntry.weight), )
elif lbType == "presentation" and lbEntry.role is not None:
otherAttrs = ( ("preferredLabel", lbEntry.role), )
else:
otherAttrs = ( )
if lbEntry.arcrole == "_dimensions_": # pick proper consecutive arcrole
fromConcept = hrefConcept(fromPrefix, fromName)
toConcept = hrefConcept(toPrefix, toName)
if toConcept is not None and toConcept.isHypercubeItem:
arcrole = XbrlConst.all
elif toConcept is not None and toConcept.isDimensionItem:
arcrole = XbrlConst.hypercubeDimension
elif fromConcept is not None and fromConcept.isDimensionItem:
arcrole = XbrlConst.dimensionDomain
else:
arcrole = XbrlConst.domainMember
else:
arcrole = lbEntry.arcrole
XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Arc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", arcrole),
("{http://www.w3.org/1999/xlink}from", fromLabel),
("{http://www.w3.org/1999/xlink}to", toLabel),
("order", order)) + otherAttrs )
order += 1.0
if lbType != "calculation" or lbEntry.isRoot:
lbTreeWalk(lbType, parentElt, lbEntry.childStruct, roleRefs, locs, toPrefix, toName)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in modelXbrl.contexts.values():
newCntx = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem:
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)):
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src"):
filingFiles.add(attrValue)
except (XMLSyntaxError, UnicodeDecodeError):
pass
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
# footnote links
footnoteIdCount = {}
modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and
any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype) and HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
XmlUtil.addChild(newLink, linkChild.qname,
attributes=attributes,
text=linkChild.textValue)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src"):
filingFiles.add(attrValue)
targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
