def insertValidationResults(self):
logEntries = []
for handler in logging.getLogger("arelle").handlers:
if hasattr(handler, "dbHandlerLogEntries"):
logEntries = handler.dbHandlerLogEntries()
break
messages = []
messageRefs = [] # direct link to objects
for i, logEntry in enumerate(logEntries):
messageId = "message/{}".format(i+1)
self.report['messages'][messageId] = m = {
'code': logEntry['code'],
'level': logEntry['level'],
'value': logEntry['message']['text'],
'report': self.reportURI,
'messageId': messageId
}
# capture message ref's
for ref in logEntry['refs']:
modelObject = self.modelXbrl.modelObject(ref.get('objectId',''))
# for now just find a concept
aspectObj = None
if isinstance(modelObject, ModelFact):
factId = XmlUtil.elementFragmentIdentifier(modelObject)
dataPoint = self.report['dataPoints'][factId]
dataPoint.setdefault('messages', []).append(messageId)
elif isinstance(modelObject, ModelConcept):
# be sure there's a proxy
self.insertAspectProxies( (modelObject.qname,)) # need imediate use of proxy
self.aspectQnameProxy(modelObject.qname).setdefault('messages', []).append(messageId)
elif isinstance(modelObject, ModelRelationship):
''' TBD
sourceId = qnamePrefix_Name(modelObject.fromModelObject.qname)
toModelObject = modelObject.toModelObject
if isinstance(toModelObject, ModelConcept):
targetId = qnamePrefix_Name(toModelObject.qname)
elif isinstance(toModelObject, ModelResource):
targetId = toModelObject.modelDocument.basename + '#' + XmlUtil.elementFragmentIdentifier(toModelObject)
else:
continue
objUri = URIRef("{}/Relationship/{}/{}/{}/{}".format(
self.reportURI,
os.path.basename(modelObject.arcrole),
os.path.basename(modelObject.linkrole),
sourceId,
targetId) )
'''
else:
continue
if messages:
self.showStatus("insert validation messages")
def logArguments(self, codes, msg, codedArgs):
# determine logCode
messageCode = None
for argCode in codes if isinstance(codes,tuple) else (codes,):
if (isinstance(argCode, ModelValue.QName) or
(self.modelManager.disclosureSystem.EFM and argCode.startswith("EFM")) or
(self.modelManager.disclosureSystem.GFM and argCode.startswith("GFM")) or
(self.modelManager.disclosureSystem.HMRC and argCode.startswith("HMRC")) or
(self.modelManager.disclosureSystem.SBRNL and argCode.startswith("SBR.NL")) or
argCode[0:3] not in ("EFM", "GFM", "HMR", "SBR")):
messageCode = argCode
break
# determine message and extra arguments
fmtArgs = {}
extras = {"messageCode":messageCode}
for argName, argValue in codedArgs.items():
if argName in ("modelObject", "modelXbrl", "modelDocument"):
try:
entryUrl = self.modelDocument.uri
except AttributeError:
entryUrl = self.entryLoadingUrl
try:
objectUrl = argValue.modelDocument.uri
except AttributeError:
try:
objectUrl = self.modelDocument.uri
except AttributeError:
objectUrl = self.entryLoadingUrl
refs = []
for arg in (argValue if isinstance(argValue, (tuple,list)) else (argValue,)):
if arg is not None:
file = UrlUtil.relativeUri(entryUrl, objectUrl)
ref = {}
if isinstance(arg,ModelObject):
ref["href"] = file + "#" + XmlUtil.elementFragmentIdentifier(arg)
ref["sourceLine"] = arg.sourceline
ref["objectId"] = arg.objectId()
else:
ref["href"] = file
refs.append(ref)
extras["refs"] = refs
elif argName == "sourceLine":
if isinstance(argValue, _INT_TYPES): # must be sortable with int's in logger
extras["sourceLine"] = argValue
elif argName != "exc_info":
if isinstance(argValue, (ModelValue.QName, ModelObject, bool, FileNamedStringIO)):
fmtArgs[argName] = str(argValue)
elif isinstance(argValue, _INT_TYPES):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale, '%i', argValue)
elif isinstance(argValue,float):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale, '%f', argValue)
else:
fmtArgs[argName] = argValue
if "refs" not in extras:
try:
file = os.path.basename(self.modelDocument.uri)
except AttributeError:
try:
file = os.path.basename(self.entryLoadingUrl)
except:
file = ""
extras["refs"] = [{"href": file}]
return (messageCode,
(msg, fmtArgs) if fmtArgs else (msg,),
extras)
def logArguments(self, codes, msg, codedArgs):
# determine logCode
messageCode = None
for argCode in codes if isinstance(codes, tuple) else (codes, ):
if (isinstance(argCode, ModelValue.QName)
or (self.modelManager.disclosureSystem.EFM
and argCode.startswith("EFM"))
or (self.modelManager.disclosureSystem.GFM
and argCode.startswith("GFM"))
or (self.modelManager.disclosureSystem.HMRC
and argCode.startswith("HMRC"))
or (self.modelManager.disclosureSystem.SBRNL
and argCode.startswith("SBR.NL"))
or argCode[0:3] not in ("EFM", "GFM", "HMR", "SBR")):
messageCode = argCode
break
# determine message and extra arguments
fmtArgs = {}
extras = {"messageCode": messageCode}
for argName, argValue in codedArgs.items():
if argName in ("modelObject", "modelXbrl", "modelDocument"):
try:
entryUrl = self.modelDocument.uri
except AttributeError:
entryUrl = self.entryLoadingUrl
refs = []
for arg in (argValue if isinstance(argValue,
(tuple, list)) else
(argValue, )):
if arg is not None:
if isinstance(arg, _STR_BASE):
objectUrl = arg
else:
try:
objectUrl = arg.modelDocument.uri
except AttributeError:
try:
objectUrl = self.modelDocument.uri
except AttributeError:
objectUrl = self.entryLoadingUrl
file = UrlUtil.relativeUri(entryUrl, objectUrl)
ref = {}
if isinstance(arg, ModelObject):
ref["href"] = file + "#" + XmlUtil.elementFragmentIdentifier(
arg)
ref["sourceLine"] = arg.sourceline
ref["objectId"] = arg.objectId()
else:
ref["href"] = file
refs.append(ref)
extras["refs"] = refs
elif argName == "sourceLine":
if isinstance(
argValue,
_INT_TYPES): # must be sortable with int's in logger
extras["sourceLine"] = argValue
elif argName != "exc_info":
if isinstance(
argValue,
(ModelValue.QName, ModelObject, bool, FileNamedStringIO)):
fmtArgs[argName] = str(argValue)
elif isinstance(argValue, _INT_TYPES):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale,
'%i', argValue)
elif isinstance(argValue, float):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale,
'%f', argValue)
else:
fmtArgs[argName] = argValue
if "refs" not in extras:
try:
file = os.path.basename(self.modelDocument.uri)
except AttributeError:
try:
file = os.path.basename(self.entryLoadingUrl)
except:
file = ""
extras["refs"] = [{"href": file}]
return (messageCode, (msg, fmtArgs) if fmtArgs else (msg, ), extras)
def saveTargetDocument(modelXbrl, targetDocumentFilename,
targetDocumentSchemaRefs):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
targetDocumentFilename, modelXbrl.modelDocument.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(
_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(
targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement,
roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in modelXbrl.contexts.values():
newCntx = targetInstance.createContext(
context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0],
measures[1],
id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in modelXbrl.facts:
if fact.isItem:
attrs = {"contextRef": fact.contextID}
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
newFact = targetInstance.createFact(fact.qname,
attributes=attrs,
text=text,
parent=parent)
newFactForOldObjId[fact.objectIndex] = newFact
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
# footnote links
modelXbrl.modelManager.showStatus(
_("Creating and validating footnotes & relationships"))
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
any(lP.modelDocument.type == Type.INLINEXBRL
for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
newLink = XmlUtil.addChild(
targetInstance.modelDocument.xmlRootElement,
linkqname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
if isinstance(
linkChild, LocPrototype
) and "{http://www.w3.org/1999/xlink}href" not in linkChild.attributes:
linkChild.attributes["{http://www.w3.org/1999/xlink}href"] = \
"#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
XmlUtil.addChild(newLink,
linkChild.qname,
attributes=linkChild.attributes,
text=linkChild.textValue)
targetInstance.saveInstance(overrideFilepath=targetUrl)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in modelXbrl.contexts.values():
newCntx = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem:
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)):
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src"):
filingFiles.add(attrValue)
except (XMLSyntaxError, UnicodeDecodeError):
pass
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
# footnote links
footnoteIdCount = {}
modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and
any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(linkChild, LocPrototype) and HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
XmlUtil.addChild(newLink, linkChild.qname,
attributes=attributes,
text=linkChild.textValue)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src"):
filingFiles.add(attrValue)
targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in modelXbrl.contexts.values():
newCntx = targetInstance.createContext(context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod
else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in modelXbrl.facts:
if fact.isItem:
attrs = {"contextRef": fact.contextID}
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
newFactForOldObjId[fact.objectIndex] = newFact
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
# footnote links
modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships"))
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
newLink = XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, linkqname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
if isinstance(linkChild, LocPrototype) and "{http://www.w3.org/1999/xlink}href" not in linkChild.attributes:
linkChild.attributes["{http://www.w3.org/1999/xlink}href"] = \
"#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
XmlUtil.addChild(newLink, linkChild.qname,
attributes=linkChild.attributes,
text=linkChild.textValue)
targetInstance.saveInstance(overrideFilepath=targetUrl)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def modelObjectUri(modelObject):
return '#'.join((modelObjectDocumentUri(modelObject),
XmlUtil.elementFragmentIdentifier(modelObject)))
def walkTree(rels, parentRelId, seq, depth, relationshipSetKey, relationshipSet, visited, relSetId, doVertices):
for rel in rels:
if rel not in visited:
visited.add(rel)
if not doVertices:
_relProp = {'seq': seq,
'depth': depth,
'order': rel.orderDecimal,
'priority': rel.priority,
'relSetId': relSetId
}
if isinstance(rel.fromModelObject, ModelConcept):
if doVertices:
aspectQnamesUsed.add(rel.fromModelObject.qname)
sourceUri = True
else:
sourceQname = rel.fromModelObject.qname
sourceUri = self.aspectQnameProxyId(sourceQname)
sourceId = qnamePrefix_Name(rel.fromModelObject.qname)
else:
sourceUri = None # tbd
toModelObject = rel.toModelObject
if isinstance(toModelObject, ModelConcept):
if doVertices:
aspectQnamesUsed.add(toModelObject.qname)
targetUri = True
else:
targetUri = self.aspectQnameProxyId(toModelObject.qname)
targetId = qnamePrefix_Name(toModelObject.qname)
elif isinstance(toModelObject, ModelResource):
if doVertices:
resources.add(toModelObject)
targetUri = 0 # just can't be None, but doesn't matter on doVertices pass
else:
if rel.preferredLabel:
_relProp['preferredLabel'] = rel.preferredLabel
if rel.arcrole in (XbrlConst.all, XbrlConst.notAll):
_relProp['cubeClosed'] = rel.closed
elif rel.arcrole in (XbrlConst.dimensionDomain, XbrlConst.domainMember):
_relProp['aspectValueUsable'] = rel.usable
elif rel.arcrole == XbrlConst.summationItem:
_relProp['weight'] = rel.weightDecimal
if relationshipSet.arcrole == "XBRL-dimensions":
_relProp['arcrole'] = rel.arcrole
if toModelObject.role:
_relProp['resourceRole'] = toModelObject.role
targetUri = modelObjectUri(toModelObject)
targetId = toModelObject.modelDocument.basename + '#' + XmlUtil.elementFragmentIdentifier(toModelObject)
else:
targetUri = None # tbd
if sourceUri is not None and targetUri is not None:
targetRelSetId = relSetId
targetRelSetKey = relationshipSetKey
if relationshipSet.arcrole == "XBRL-dimensions" and rel.targetRole:
targetRelSet = self.modelXbrl.relationshipSet(relationshipSet.arcrole, rel.targetRole)
for i, relSetKey in enumerate(self.relationshipSets):
arcrole, ELR, linkqname, arcqname = relSetKey
if arcrole == "XBRL-dimensions" and ELR == rel.targetRole:
targetRelationshipSetId = relSetIds[relSetKey]
targetRelSetKey = relSetKey
break
if not doVertices:
_relProp['targetLinkrole'] = rel.targetRole
_relProp['targetRelSet'] = targetRelationshipSetId
else:
targetRelSetKey = relationshipSetKey
targetRelSet = relationshipSet
if doVertices:
relId = None
else:
_relProp['from'] = sourceUri
_relProp['fromQname'] = sourceQname
_relProp['to'] = targetUri
_arcrole = os.path.basename(rel.arcrole)
relId = "{}/{}/{}/{}".format(
_arcrole,
os.path.basename(rel.linkrole),
sourceId,
targetId)
_relProp['relId'] = relId
_relProp['relSetKey'] = relationshipSetKey
relE.append(_relProp)
seq += 1
seq = walkTree(targetRelSet.fromModelObject(toModelObject), relId, seq, depth+1, targetRelSetKey, targetRelSet, visited, targetRelSetId, doVertices)
visited.remove(rel)
return seq
def insertRelationshipSets(self):
self.showStatus("insert relationship sets")
aspectQnamesUsed = set()
for i, relationshipSetKey in enumerate(self.relationshipSets):
arcrole, linkrole, linkqname, arcqname = relationshipSetKey
if linkqname:
aspectQnamesUsed.add(linkqname)
if arcqname:
aspectQnamesUsed.add(arcqname)
self.insertAspectProxies(aspectQnamesUsed)
relationshipSets = self.report['relationshipSets']
relSetIds = {}
for i, relationshipSetKey in enumerate(self.relationshipSets):
arcrole, linkrole, linkqname, arcqname = relationshipSetKey
if arcrole not in ("XBRL-formulae", "Table-rendering", "XBRL-footnotes") and linkrole and linkqname and arcqname:
# skip paths and qnames for now (add later for non-SEC)
relSetId = "{}/{}".format(
os.path.basename(arcrole),
os.path.basename(linkrole))
relSetIds[relationshipSetKey] = relSetId
relationshipSets[relSetId] = relationshipSet = {
'arcrole': arcrole,
'linkrole': linkrole,
'arcname': self.aspectQnameProxyId(arcqname),
'linkname': self.aspectQnameProxyId(linkqname),
'report': self.reportURI,
'roots': [],
'relationships': []
}
# do tree walk to build relationships with depth annotated, no targetRole navigation
relE = [] # fromV, toV, label
resources = set()
aspectQnamesUsed = set()
resourceIDs = {} # index by object
def walkTree(rels, parentRelId, seq, depth, relationshipSetKey, relationshipSet, visited, relSetId, doVertices):
for rel in rels:
if rel not in visited:
visited.add(rel)
if not doVertices:
_relProp = {'seq': seq,
'depth': depth,
'order': rel.orderDecimal,
'priority': rel.priority,
'relSetId': relSetId
}
if isinstance(rel.fromModelObject, ModelConcept):
if doVertices:
aspectQnamesUsed.add(rel.fromModelObject.qname)
sourceUri = True
else:
sourceQname = rel.fromModelObject.qname
sourceUri = self.aspectQnameProxyId(sourceQname)
sourceId = qnamePrefix_Name(rel.fromModelObject.qname)
else:
sourceUri = None # tbd
toModelObject = rel.toModelObject
if isinstance(toModelObject, ModelConcept):
if doVertices:
aspectQnamesUsed.add(toModelObject.qname)
targetUri = True
else:
targetUri = self.aspectQnameProxyId(toModelObject.qname)
targetId = qnamePrefix_Name(toModelObject.qname)
elif isinstance(toModelObject, ModelResource):
if doVertices:
resources.add(toModelObject)
targetUri = 0 # just can't be None, but doesn't matter on doVertices pass
else:
if rel.preferredLabel:
_relProp['preferredLabel'] = rel.preferredLabel
if rel.arcrole in (XbrlConst.all, XbrlConst.notAll):
_relProp['cubeClosed'] = rel.closed
elif rel.arcrole in (XbrlConst.dimensionDomain, XbrlConst.domainMember):
_relProp['aspectValueUsable'] = rel.usable
elif rel.arcrole == XbrlConst.summationItem:
_relProp['weight'] = rel.weightDecimal
if relationshipSet.arcrole == "XBRL-dimensions":
_relProp['arcrole'] = rel.arcrole
if toModelObject.role:
_relProp['resourceRole'] = toModelObject.role
targetUri = modelObjectUri(toModelObject)
targetId = toModelObject.modelDocument.basename + '#' + XmlUtil.elementFragmentIdentifier(toModelObject)
else:
targetUri = None # tbd
if sourceUri is not None and targetUri is not None:
targetRelSetId = relSetId
targetRelSetKey = relationshipSetKey
if relationshipSet.arcrole == "XBRL-dimensions" and rel.targetRole:
targetRelSet = self.modelXbrl.relationshipSet(relationshipSet.arcrole, rel.targetRole)
for i, relSetKey in enumerate(self.relationshipSets):
arcrole, ELR, linkqname, arcqname = relSetKey
if arcrole == "XBRL-dimensions" and ELR == rel.targetRole:
targetRelationshipSetId = relSetIds[relSetKey]
targetRelSetKey = relSetKey
break
if not doVertices:
_relProp['targetLinkrole'] = rel.targetRole
_relProp['targetRelSet'] = targetRelationshipSetId
else:
targetRelSetKey = relationshipSetKey
targetRelSet = relationshipSet
if doVertices:
relId = None
else:
_relProp['from'] = sourceUri
_relProp['fromQname'] = sourceQname
_relProp['to'] = targetUri
_arcrole = os.path.basename(rel.arcrole)
relId = "{}/{}/{}/{}".format(
_arcrole,
os.path.basename(rel.linkrole),
sourceId,
targetId)
_relProp['relId'] = relId
_relProp['relSetKey'] = relationshipSetKey
relE.append(_relProp)
seq += 1
seq = walkTree(targetRelSet.fromModelObject(toModelObject), relId, seq, depth+1, targetRelSetKey, targetRelSet, visited, targetRelSetId, doVertices)
visited.remove(rel)
return seq
for doVertices in range(1,-1,-1): # pass 0 = vertices, pass 1 = edges
for i, relationshipSetKey in enumerate(self.relationshipSets):
arcrole, linkrole, linkqname, arcqname = relationshipSetKey
if arcrole not in ("XBRL-formulae", "Table-rendering", "XBRL-footnotes") and linkrole and linkqname and arcqname:
relSetId = relSetIds[relationshipSetKey]
relationshipSet = self.modelXbrl.relationshipSet(*relationshipSetKey)
seq = 1
for rootConcept in relationshipSet.rootConcepts:
seq = walkTree(relationshipSet.fromModelObject(rootConcept), None, seq, 1, relationshipSetKey, relationshipSet, set(), relSetId, doVertices)
if doVertices:
if resources:
for resource in resources:
resourceUri = modelObjectUri(resource)
r = {'url': resourceUri,
'value': resource.stringValue
}
if resource.xmlLang:
r['language'] = resource.xmlLang
if resource.role:
r['role'] = resource.role
self.documents[modelObjectDocumentUri(resource)]['resources'][
XmlUtil.elementFragmentIdentifier(resource)] = r
self.insertAspectProxies(aspectQnamesUsed)
else:
for j, rel in enumerate(relE):
relId = rel['relId']
relSetId = rel['relSetId']
relSet = relationshipSets[relSetId]
r = dict((k,v)
for k,v in rel.items()
if k not in ('relId', 'relPredicate', 'relSetId', 'relSetKey', 'fromQname'))
relSet['relationships'].append(r)
if rel.get('depth', 0) == 1:
relSet['roots'].append(r)
sourceQname = rel['fromQname']
if sourceQname in self.aspect_proxy:
self.aspect_proxy[sourceQname] \
.setdefault('relationships', {}) \
.setdefault(rel['relSetId'], []) \
.append(rel['to'])
# TBD: do we want to link resources to the dts (by role, class, or otherwise?)
resourceIDs.clear() # dereferemce objects
resources = None
def insertDataPoints(self):
# separate graph
# document-> dataTypeSet -> dataType
self.showStatus("insert DataPoints")
# note these initial aspects Qnames used also must be in conceptsUsed above
dimensions = [] # index by hash of dimension
dimensionIds = {} # index for dimension
if self.modelXbrl.modelDocument.type in (Type.INSTANCE, Type.INLINEXBRL):
contextAspectValueSelections = {} # contexts processed already
unitIDs = set() # units processed already
periodProxies = {}
entityIdentifierAspectProxies = {}
dataPoints = self.report['dataPoints']
for fact in self.modelXbrl.factsInInstance:
self.insertAspectProxies( (fact.qname,) )
factId = XmlUtil.elementFragmentIdentifier(fact)
dataPoints[factId] = dataPoint = {
'document': modelObjectDocumentUri(fact),
'id': factId,
'sourceLine': fact.sourceline,
'dataPointUrl': modelObjectUri(fact),
'baseItem': self.aspectQnameProxyId(fact.qname)
}
context = fact.context
concept = fact.concept
if context is not None:
if context.entityIdentifier not in entityIdentifierAspectProxies:
entityScheme, entityIdentifier = context.entityIdentifier
entityIdentifierAspectProxy = "{}/{}".format(
qnamePrefix_Name(XbrlConst.qnXbrliIdentifier),
entityIdentifier)
e = self.insertAspectProxy(XbrlConst.qnXbrliIdentifier, entityIdentifierAspectProxy)
e['scheme'] = entityScheme
e['identifier'] = entityIdentifier
entityIdentifierAspectProxies[context.entityIdentifier] = entityIdentifierAspectProxy
else:
entityIdentifierAspectProxy = entityIdentifierAspectProxies[context.entityIdentifier]
dataPoint['entityIdentifier'] = entityIdentifierAspectProxy
if context.isForeverPeriod:
period = "forever"
if context.isInstantPeriod:
endDate = XmlUtil.dateunionValue(context.instantDatetime, subtractOneDay=True).replace(':','_')
period = "instant/{}".format(endDate)
else:
startDate = XmlUtil.dateunionValue(context.startDatetime).replace(':','_')
endDate = XmlUtil.dateunionValue(context.endDatetime, subtractOneDay=True).replace(':','_')
period = "duration/{}/{}".format(startDate, endDate)
if period not in periodProxies:
periodProxy = "{}/{}".format(
qnamePrefix_Name(XbrlConst.qnXbrliPeriod),
period)
p = self.insertAspectProxy(XbrlConst.qnXbrliPeriod, periodProxy)
p['isForever'] = context.isForeverPeriod
p['isInstant'] = context.isInstantPeriod
if context.isStartEndPeriod:
d = context.startDatetime
if d.hour == 0 and d.minute == 0 and d.second == 0:
d = d.date()
p['startDate'] = d
if context.isStartEndPeriod or context.isInstantPeriod:
d = context.endDatetime
if d.hour == 0 and d.minute == 0 and d.second == 0:
d = (d - datetime.timedelta(1)).date()
p['endDate'] = d
periodProxies[period] = periodProxy
else:
periodProxy = periodProxies[period]
dataPoint['period'] = periodProxy
dataPoint['contextUrl'] = modelObjectUri(context)
dataPoint['contextId'] = context.id
if context.id not in contextAspectValueSelections:
contextAspectValueSelections[context.id] = contextAspectValueSelection = []
for dimVal in context.qnameDims.values():
dim = qnamePrefix_Name(dimVal.dimensionQname)
if dimVal.isExplicit:
self.insertAspectProxies( (dimVal.memberQname,) ) # need imediate use of proxy
v = self.aspectQnameProxyId(dimVal.memberQname)
else:
v = dimVal.typedMember.stringValue
dimProxy = "{}/{}".format(dim, v)
d = self.insertAspectProxy(dimVal.dimensionQname, dimProxy)
contextAspectValueSelection.append(dimProxy)
d['aspect'] = dim
if dimVal.isExplicit:
d['aspectValue'] = v
else:
d['typedValue'] = v
else:
contextAspectValueSelection = contextAspectValueSelections[context.id]
dataPoint['aspectValueSelections'] = contextAspectValueSelection
if fact.isNumeric:
if fact.precision == "INF":
dataPoint['precision'] = "INF"
elif fact.precision is not None:
dataPoint['precision'] = fact.precision
if fact.decimals == "INF":
dataPoint['decimals'] = "INF"
elif fact.decimals is not None:
dataPoint['decimals'] = fact.decimals
if fact.unit is not None:
unit = fact.unit
unitProxy = "{}/{}".format(
qnamePrefix_Name(XbrlConst.qnXbrliUnit),
unit.id)
dataPoint['unit'] = unitProxy
if unit.id not in unitIDs:
unitIDs.add(unit.id)
u = self.insertAspectProxy(XbrlConst.qnXbrliUnit, unitProxy)
u['unitId'] = unit.id
mults, divs = unit.measures
u['multiplyMeasures'] = [qnameUri(qn) for qn in mults]
if divs:
u['divideMeasures'] = [qnameUri(qn) for qn in divs]
if fact.xmlLang is None and fact.concept is not None and fact.concept.baseXsdType is not None:
dataPoint['value'] = fact.xValue
# The insert with base XSD type but no language
elif fact.xmlLang is not None:
# assuming string type with language
dataPoint['language'] = fact.xmlLang
dataPoint['value'] = fact.value
else:
# Otherwise insert as plain liternal with no language or datatype
dataPoint['value'] = fact.value
if fact.modelTupleFacts:
dataPoint['tuple'] = [XmlUtil.elementFragmentIdentifier(tupleFact)
for tupleFact in fact.modelTupleFacts]
def saveTargetDocument(modelXbrl,
targetDocumentFilename,
targetDocumentSchemaRefs,
outputZip=None,
filingFiles=None):
targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
targetDocumentFilename, modelXbrl.modelDocument.filepath)
targetUrlParts = targetUrl.rpartition(".")
targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
modelXbrl.modelManager.showStatus(
_("Extracting instance ") + os.path.basename(targetUrl))
targetInstance = ModelXbrl.create(modelXbrl.modelManager,
newDocumentType=Type.INSTANCE,
url=targetUrl,
schemaRefs=targetDocumentSchemaRefs,
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(
targetInstance) # need dimension defaults
# roleRef and arcroleRef (of each inline document)
for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
for roleRefElt in sourceRefs.values():
XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement,
roleRefElt.qname,
attributes=roleRefElt.items())
# contexts
for context in modelXbrl.contexts.values():
if context is not None:
newCntx = targetInstance.createContext(
context.entityIdentifier[0],
context.entityIdentifier[1],
'instant' if context.isInstantPeriod else
'duration' if context.isStartEndPeriod else 'forever',
context.startDatetime,
context.endDatetime,
None,
context.qnameDims, [], [],
id=context.id)
for unit in modelXbrl.units.values():
if unit is not None:
measures = unit.measures
newUnit = targetInstance.createUnit(measures[0],
measures[1],
id=unit.id)
modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
newFactForOldObjId = {}
def createFacts(facts, parent):
for fact in facts:
if fact.isItem:
attrs = {"contextRef": fact.contextID}
if fact.id:
attrs["id"] = fact.id
if fact.isNumeric:
attrs["unitRef"] = fact.unitID
if fact.get("decimals"):
attrs["decimals"] = fact.get("decimals")
if fact.get("precision"):
attrs["precision"] = fact.get("precision")
if fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact.xValue if fact.xValid else fact.textValue
newFact = targetInstance.createFact(fact.qname,
attributes=attrs,
text=text,
parent=parent)
newFactForOldObjId[fact.objectIndex] = newFact
if filingFiles and fact.concept is not None and fact.concept.isTextBlock:
# check for img and other filing references
for xmltext in [text] + CDATApattern.findall(text):
try:
for elt in XML(
"<body>\n{0}\n</body>\n".format(xmltext)):
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src"):
filingFiles.add(attrValue)
except (XMLSyntaxError, UnicodeDecodeError):
pass
elif fact.isTuple:
newTuple = targetInstance.createFact(fact.qname, parent=parent)
newFactForOldObjId[fact.objectIndex] = newTuple
createFacts(fact.modelTupleFacts, newTuple)
createFacts(modelXbrl.facts, None)
# footnote links
footnoteIdCount = {}
modelXbrl.modelManager.showStatus(
_("Creating and validating footnotes & relationships"))
HREF = "{http://www.w3.org/1999/xlink}href"
footnoteLinks = defaultdict(list)
for linkKey, linkPrototypes in modelXbrl.baseSets.items():
arcrole, linkrole, linkqname, arcqname = linkKey
if (linkrole and linkqname and arcqname and # fully specified roles
arcrole != "XBRL-footnotes" and any(
lP.modelDocument.type == Type.INLINEXBRL
for lP in linkPrototypes)):
for linkPrototype in linkPrototypes:
if linkPrototype not in footnoteLinks[linkrole]:
footnoteLinks[linkrole].append(linkPrototype)
for linkrole in sorted(footnoteLinks.keys()):
for linkPrototype in footnoteLinks[linkrole]:
newLink = XmlUtil.addChild(
targetInstance.modelDocument.xmlRootElement,
linkPrototype.qname,
attributes=linkPrototype.attributes)
for linkChild in linkPrototype:
attributes = linkChild.attributes
if isinstance(
linkChild,
LocPrototype) and HREF not in linkChild.attributes:
linkChild.attributes[HREF] = \
"#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
elif isinstance(linkChild, ModelInlineFootnote):
idUseCount = footnoteIdCount.get(linkChild.footnoteID,
0) + 1
if idUseCount > 1: # if footnote with id in other links bump the id number
attributes = linkChild.attributes.copy()
attributes["id"] = "{}_{}".format(
attributes["id"], idUseCount)
footnoteIdCount[linkChild.footnoteID] = idUseCount
XmlUtil.addChild(newLink,
linkChild.qname,
attributes=attributes,
text=linkChild.textValue)
if filingFiles and linkChild.textValue:
footnoteHtml = XML("<body/>")
copyHtml(linkChild, footnoteHtml)
for elt in footnoteHtml.iter():
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src"):
filingFiles.add(attrValue)
targetInstance.saveInstance(overrideFilepath=targetUrl,
outputZip=outputZip)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def logArguments(self, codes, msg, codedArgs):
""" Prepares arguments for logger function as per info() below.
If codes includes EFM, GFM, HMRC, or SBR-coded error then the code chosen (if a sequence)
corresponds to whether EFM, GFM, HMRC, or SBR validation is in effect.
"""
def propValues(properties):
# deref objects in properties
return [(p[0],str(p[1])) if len(p) == 2 else (p[0],str(p[1]),propValues(p[2]))
for p in properties if 2 <= len(p) <= 3]
# determine logCode
messageCode = None
for argCode in codes if isinstance(codes,tuple) else (codes,):
if (isinstance(argCode, ModelValue.QName) or
(self.modelManager.disclosureSystem.EFM and argCode.startswith("EFM")) or
(self.modelManager.disclosureSystem.GFM and argCode.startswith("GFM")) or
(self.modelManager.disclosureSystem.HMRC and argCode.startswith("HMRC")) or
(self.modelManager.disclosureSystem.SBRNL and argCode.startswith("SBR.NL")) or
argCode[0:3] not in ("EFM", "GFM", "HMR", "SBR")):
messageCode = argCode
break
# determine message and extra arguments
fmtArgs = {}
extras = {"messageCode":messageCode}
logHrefObjectProperties = getattr(self.logger, "logHrefObjectProperties", False)
for argName, argValue in codedArgs.items():
if argName in ("modelObject", "modelXbrl", "modelDocument"):
try:
entryUrl = self.modelDocument.uri
except AttributeError:
entryUrl = self.entryLoadingUrl
refs = []
for arg in (argValue if isinstance(argValue, (tuple,list,set)) else (argValue,)):
if arg is not None:
if isinstance(arg, _STR_BASE):
objectUrl = arg
else:
try:
objectUrl = arg.modelDocument.uri
except AttributeError:
try:
objectUrl = self.modelDocument.uri
except AttributeError:
objectUrl = self.entryLoadingUrl
file = UrlUtil.relativeUri(entryUrl, objectUrl)
ref = {}
if isinstance(arg,ModelObject):
ref["href"] = file + "#" + XmlUtil.elementFragmentIdentifier(arg)
ref["sourceLine"] = arg.sourceline
ref["objectId"] = arg.objectId()
if logHrefObjectProperties:
try:
ref["properties"] = propValues(arg.propertyView)
except AttributeError:
pass # is a default properties entry appropriate or needed?
else:
ref["href"] = file
try:
ref["sourceLine"] = arg.sourceline
except AttributeError:
pass # arg may not have sourceline, ignore if so
refs.append(ref)
extras["refs"] = refs
elif argName == "sourceLine":
if isinstance(argValue, _INT_TYPES): # must be sortable with int's in logger
extras["sourceLine"] = argValue
elif argName != "exc_info":
if isinstance(argValue, (ModelValue.QName, ModelObject, bool, FileNamedStringIO,
# might be a set of lxml objects not dereferencable at shutdown
tuple, list, set)):
fmtArgs[argName] = str(argValue)
elif argValue is None:
fmtArgs[argName] = "(none)"
elif isinstance(argValue, _INT_TYPES):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale, '%i', argValue)
elif isinstance(argValue,float):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale, '%f', argValue)
else:
fmtArgs[argName] = argValue
if "refs" not in extras:
try:
file = os.path.basename(self.modelDocument.uri)
except AttributeError:
try:
file = os.path.basename(self.entryLoadingUrl)
except:
file = ""
extras["refs"] = [{"href": file}]
return (messageCode,
(msg, fmtArgs) if fmtArgs else (msg,),
extras)
def logArguments(self, codes, msg, codedArgs):
# determine logCode
messageCode = None
for argCode in codes if isinstance(codes,tuple) else (codes,):
if ((self.modelManager.disclosureSystem.EFM and argCode.startswith("EFM")) or
(self.modelManager.disclosureSystem.GFM and argCode.startswith("GFM")) or
(self.modelManager.disclosureSystem.HMRC and argCode.startswith("HMRC")) or
(self.modelManager.disclosureSystem.SBRNL and argCode.startswith("SBR.NL")) or
argCode[0:3] not in ("EFM", "GFM", "HMR", "SBR")):
messageCode = argCode
break
# determine message and extra arguments
fmtArgs = {}
extras = {"messageCode":messageCode}
for argName, argValue in codedArgs.items():
if argName in ("modelObject", "modelXbrl", "modelDocument"):
try:
entryUrl = self.modelDocument.uri
except AttributeError:
entryUrl = self.entryLoadingUrl
try:
objectUrl = argValue.modelDocument.uri
except AttributeError:
try:
objectUrl = self.modelDocument.uri
except AttributeError:
objectUrl = self.entryLoadingUrl
file = UrlUtil.relativeUri(entryUrl, objectUrl)
extras["file"] = file
if isinstance(argValue,ModelObject):
extras["href"] = file + "#" + XmlUtil.elementFragmentIdentifier(argValue)
extras["sourceLine"] = argValue.sourceline
extras["objectId"] = argValue.objectId()
else:
extras["href"] = file
extras["sourceLine"] = ""
elif argName == "sourceLine":
extras["sourceLine"] = argValue
elif argName != "exc_info":
if isinstance(argValue, (ModelValue.QName, ModelObject, bool)):
fmtArgs[argName] = str(argValue)
elif isinstance(argValue,int):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale, '%i', argValue)
elif isinstance(argValue,float):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale, '%f', argValue)
else:
fmtArgs[argName] = argValue
if "href" not in extras:
try:
file = os.path.basename(self.modelDocument.uri)
except AttributeError:
try:
file = os.path.basename(self.entryLoadingUrl)
except:
file = ""
extras["file"] = file
extras["href"] = file
extras["sourceLine"] = ""
return (messageCode,
(msg, fmtArgs) if fmtArgs else (msg,),
extras)
