def generateCHComponent(dts, componentFile):
if dts.fileSource.isArchive:
return
import os, io
from arelle import XmlUtil, XbrlConst
file = io.StringIO('''
<nsmap>
<Component/>
</nsmap>
''')
from arelle.ModelObjectFactory import parser
parser, parserLookupName, parserLookupClass = parser(dts, None)
xmlDocument = etree.parse(file, parser=parser, base_url=componentFile)
file.close()
for componentElt in xmlDocument.iter(tag="Component"):
break
# use presentation relationships for broader and narrower concepts
arcrole = XbrlConst.parentChild
# sort URIs by definition
linkroleUris = []
relationshipSet = dts.relationshipSet(arcrole)
if relationshipSet:
for linkroleUri in relationshipSet.linkRoleUris:
modelRoleTypes = dts.roleTypes.get(linkroleUri)
if modelRoleTypes:
roledefinition = (modelRoleTypes[0].genLabel(strip=True)
or modelRoleTypes[0].definition
or linkroleUri)
else:
roledefinition = linkroleUri
linkroleUris.append((roledefinition, linkroleUri))
linkroleUris.sort()
# for each URI in definition order
for roledefinition, linkroleUri in linkroleUris:
elt = etree.SubElement(componentElt,
"Network",
attrib={
"identifier": linkroleUri,
"label": roledefinition
})
linkRelationshipSet = dts.relationshipSet(arcrole, linkroleUri)
for rootConcept in linkRelationshipSet.rootConcepts:
genConcept(dts, elt, rootConcept, None, arcrole,
linkRelationshipSet, set())
fh = open(componentFile, "w", encoding="utf-8")
XmlUtil.writexml(fh, xmlDocument, encoding="utf-8")
fh.close()
dts.info(
"info:saveCHComponentFile",
_("Component file for %(entryFile)s in file %(componentOutputFile)s."),
modelObject=dts,
entryFile=dts.uri,
componentOutputFile=componentFile)
def generateCHComponent(dts, componentFile):
if dts.fileSource.isArchive:
return
import os, io
from arelle import XmlUtil, XbrlConst
file = io.StringIO('''
<nsmap>
<Component/>
</nsmap>
'''
)
from arelle.ModelObjectFactory import parser
parser, parserLookupName, parserLookupClass = parser(dts,None)
xmlDocument = etree.parse(file,parser=parser,base_url=componentFile)
file.close()
for componentElt in xmlDocument.iter(tag="Component"):
break
# use presentation relationships for broader and narrower concepts
arcrole = XbrlConst.parentChild
# sort URIs by definition
linkroleUris = []
relationshipSet = dts.relationshipSet(arcrole)
if relationshipSet:
for linkroleUri in relationshipSet.linkRoleUris:
modelRoleTypes = dts.roleTypes.get(linkroleUri)
if modelRoleTypes:
roledefinition = (modelRoleTypes[0].genLabel(strip=True) or modelRoleTypes[0].definition or linkroleUri)
else:
roledefinition = linkroleUri
linkroleUris.append((roledefinition, linkroleUri))
linkroleUris.sort()
# for each URI in definition order
for roledefinition, linkroleUri in linkroleUris:
elt = etree.SubElement(componentElt, "Network", attrib={
"identifier": linkroleUri,
"label": roledefinition})
linkRelationshipSet = dts.relationshipSet(arcrole, linkroleUri)
for rootConcept in linkRelationshipSet.rootConcepts:
genConcept(dts, elt, rootConcept, None, arcrole, linkRelationshipSet, set())
fh = open(componentFile, "w", encoding="utf-8")
XmlUtil.writexml(fh, xmlDocument, encoding="utf-8")
fh.close()
dts.info("info:saveCHComponentFile",
_("Component file for %(entryFile)s in file %(componentOutputFile)s."),
modelObject=dts,
entryFile=dts.uri, componentOutputFile=componentFile)
def generateUpdatedTableLB(dts, updatedTableLinkbaseFile):
import os, io
from arelle import XmlUtil, XbrlConst
from arelle.ViewUtil import viewReferences, referenceURI
from arelle.ModelRenderingObject import ModelEuAxisCoord
if dts.fileSource.isArchive:
dts.error("genTblLB:outFileIsArchive",
_("Updated Table Linkbase file cannot be an archive: %(tableLBOutputFile)s."),
modelObject=dts, tableLBOutputFile=updatedTableLinkbaseFile)
return
tblAxisRelSet = dts.relationshipSet(XbrlConst.euTableAxis)
axisMbrRelSet = dts.relationshipSet(XbrlConst.euAxisMember)
if len(tblAxisRelSet.modelRelationships) == 0:
dts.error("genTblLB:noInputTables",
_("DTS does not contain Eurofiling 2010 tables and axes: %(entryFile)s."),
modelObject=dts, entryFile=dts.uri)
return
file = io.StringIO('''
<nsmap>
<link:linkbase
xmlns:label="http://xbrl.org/2008/label"
xmlns:gen="http://xbrl.org/2008/generic"
xmlns:df="http://xbrl.org/2008/filter/dimension"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:reference="http://xbrl.org/2008/reference"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:table="http://xbrl.org/2011/table"
xmlns:formula="http://xbrl.org/2008/formula"
xsi:schemaLocation="
http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd
http://xbrl.org/2008/generic http://www.xbrl.org/2008/generic-link.xsd
http://xbrl.org/2008/reference http://www.xbrl.org/2008/generic-reference.xsd
http://xbrl.org/2008/label http://www.xbrl.org/2008/generic-label.xsd
http://xbrl.org/2011/table http://www.xbrl.org/2011/table.xsd
http://xbrl.org/2008/filter/dimension http://www.xbrl.org/2008/dimension-filter.xsd">
<link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/table-filter" xlink:type="simple"
xlink:href="http://www.xbrl.org/2011/table.xsd#table-filter"/>
<link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/table-axis" xlink:type="simple"
xlink:href="http://www.xbrl.org/2011/table.xsd#table-axis"/>
<link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/axis-subtree" xlink:type="simple"
xlink:href="http://www.xbrl.org/2011/table.xsd#axis-subtree"/>
<link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/axis-filter" xlink:type="simple"
xlink:href="http://www.xbrl.org/2011/filter-axis.xsd#axis-filter"/>
</link:linkbase>
</nsmap>
<!-- Generated by Arelle(r) http://arelle.org -->
'''
)
from arelle.ModelObjectFactory import parser
parser, parserLookupName, parserLookupClass = parser(dts,None)
from lxml import etree
xmlDocument = etree.parse(file,parser=parser,base_url=updatedTableLinkbaseFile)
file.close()
nsmapElt = xmlDocument.getroot()
#xmlDocument.getroot().init(self) ## is this needed ??
for lbElement in xmlDocument.iter(tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
break
class DocObj: # fake ModelDocument for namespaces
def __init__(self):
self.xmlRootElement = lbElement
self.xmlDocument = xmlDocument
docObj = DocObj()
numELRs = 0
numTables = 0
def copyAttrs(fromElt, toElt, attrTags):
for attr in attrTags:
if fromElt.get(attr):
toElt.set(attr, fromElt.get(attr))
def generateTable(newLinkElt, newTblElt, srcTblElt, tblAxisRelSet, axisMbrRelSet, visited):
if srcTblElt is not None:
for rel in tblAxisRelSet.fromModelObject(srcTblElt):
srcAxisElt = rel.toModelObject
if isinstance(srcAxisElt, ModelEuAxisCoord):
visited.add(srcAxisElt)
newAxisElt = etree.SubElement(newLinkElt, "{http://xbrl.org/2011/table}ruleAxis")
copyAttrs(srcAxisElt, newAxisElt, ("id",
"{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}label"))
newAxisElt.set("abstract", "true") # always true on root element
newArcElt = etree.SubElement(newLinkElt, "{http://xbrl.org/2011/table}axisArc")
copyAttrs(rel, newArcElt, ("id",
"{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}from",
"{http://www.w3.org/1999/xlink}to",
"order"))
newArcElt.set("{http://www.w3.org/1999/xlink}arcrole", XbrlConst.tableBreakdown)
newArcElt.set("axisDisposition", rel.axisDisposition)
generateAxis(newLinkElt, newAxisElt, srcAxisElt, axisMbrRelSet, visited)
visited.discard(srcAxisElt)
def generateAxis(newLinkElt, newAxisParentElt, srcAxisElt, axisMbrRelSet, visited):
for rel in axisMbrRelSet.fromModelObject(srcAxisElt):
tgtAxisElt = rel.toModelObject
if isinstance(tgtAxisElt, ModelEuAxisCoord) and tgtAxisElt not in visited:
visited.add(tgtAxisElt)
newAxisElt = etree.SubElement(newLinkElt, "{http://xbrl.org/2011/table}ruleAxis")
copyAttrs(tgtAxisElt, newAxisElt, ("id",
"abstract",
"{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}label"))
if tgtAxisElt.primaryItemQname:
newRuleElt = etree.SubElement(newAxisElt, "{http://xbrl.org/2008/formula}concept")
newQnameElt = etree.SubElement(newRuleElt, "{http://xbrl.org/2008/formula}qname")
newQnameElt.text = XmlUtil.addQnameValue(docObj, tgtAxisElt.primaryItemQname)
for dimQname, memQname in tgtAxisElt.explicitDims:
newRuleElt = etree.SubElement(newAxisElt, "{http://xbrl.org/2008/formula}explicitDimension")
newRuleElt.set("dimension", XmlUtil.addQnameValue(docObj, dimQname))
newMbrElt = etree.SubElement(newRuleElt, "{http://xbrl.org/2008/formula}member")
newQnameElt = etree.SubElement(newMbrElt, "{http://xbrl.org/2008/formula}qname")
newQnameElt.text = XmlUtil.addQnameValue(docObj, memQname)
newArcElt = etree.SubElement(newLinkElt, "{http://xbrl.org/2011/table}axisArc")
copyAttrs(rel, newArcElt, ("id",
"{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}from",
"{http://www.w3.org/1999/xlink}to",
"order"))
newArcElt.set("{http://www.w3.org/1999/xlink}arcrole", XbrlConst.tableAxisSubtree)
generateAxis(newLinkElt, newAxisElt, tgtAxisElt, axisMbrRelSet, visited)
visited.discard(tgtAxisElt)
# sort URIs
linkroleUris = sorted([linkroleUri
for linkroleUri in tblAxisRelSet.linkRoleUris])
firstNewLinkElt = None
roleRefUris = set()
for linkroleUri in linkroleUris:
numELRs += 1
newLinkElt = etree.SubElement(lbElement, "{http://xbrl.org/2008/generic}link")
newLinkElt.set("{http://www.w3.org/1999/xlink}type", "extended")
newLinkElt.set("{http://www.w3.org/1999/xlink}role", linkroleUri)
if firstNewLinkElt is None: firstNewLinkElt = newLinkElt
# To do: add roleRef if needed
tblAxisRelSet = dts.relationshipSet(XbrlConst.euTableAxis, linkroleUri)
axisMbrRelSet = dts.relationshipSet(XbrlConst.euAxisMember, linkroleUri)
for srcTblElt in tblAxisRelSet.rootConcepts:
if srcTblElt.tag == "{http://www.eurofiling.info/2010/rendering}table":
numTables += 1
newTblElt = etree.SubElement(newLinkElt, "{http://xbrl.org/2011/table}table")
newTblElt.set("aspectModel", "dimensional")
copyAttrs(srcTblElt, newTblElt, ("id",
"{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}label"))
generateTable(newLinkElt, newTblElt, srcTblElt, tblAxisRelSet, axisMbrRelSet, set())
if linkroleUri not in roleRefUris:
srcRoleRefElt = XmlUtil.descendant(srcTblElt.getroottree(), XbrlConst.link, "roleRef", "roleURI", linkroleUri)
if srcRoleRefElt is not None:
roleRefUris.add(linkroleUri)
newRoleRefElt = etree.Element("{http://www.xbrl.org/2003/linkbase}roleRef")
copyAttrs(srcRoleRefElt, newRoleRefElt, ("roleURI",
"{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}href"))
firstNewLinkElt.addprevious(newRoleRefElt)
fh = open(updatedTableLinkbaseFile, "w", encoding="utf-8")
XmlUtil.writexml(fh, xmlDocument, encoding="utf-8")
fh.close()
dts.info("info:updateTableLinkbase",
_("Updated Table Linkbase of %(entryFile)s has %(numberOfLinkroles)s linkroles, %(numberOfTables)s tables in file %(tableLBOutputFile)s."),
modelObject=dts,
entryFile=dts.uri, numberOfLinkroles=numELRs, numberOfTables=numTables, tableLBOutputFile=updatedTableLinkbaseFile)
def generateSkos(dts, skosFile):
try:
import os, io
from arelle import XmlUtil, XbrlConst
from arelle.ViewUtil import viewReferences, referenceURI
skosNs = "http://www.w3.org/2004/02/skos/core#"
rdfNs = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
dts.modelManager.showStatus("initializing SKOS document")
file = io.StringIO('''
<!DOCTYPE rdf:RDF>
<nsmap>
<rdf:RDF xmlns="urn:cgi:classifier:CGI:XBRL:201204#" xml:base="urn:cgi:classifierScheme:CGI:XBRL:201204" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xmlns:owl2xml="http://www.w3.org/2006/12/owl2-xml#" xmlns:p1="#" xmlns:owl="http://www.w3.org/2002/07/owl#" xmlns:xsd="http://www.w3.org/2001/XMLSchema#" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:xbrl-201204="urn:cgi:classifier:CGI:XBRL:201204#" xmlns:skos="http://www.w3.org/2004/02/skos/core#">
<owl:Ontology rdf:about="" />
<!-- Annotation properties -->
<owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/date" />
<owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/source" />
<owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/title" />
<owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/description" />
<owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/contributor" />
<owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/creator" />
<owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/format" />
<owl:AnnotationProperty rdf:about="http://www.w3.org/2002/07/owl#versionInfo" />
<!-- Object Properties -->
<!-- http://www.w3.org/2004/02/skos/core#broader -->
<owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#broader" />
<!-- http://www.w3.org/2004/02/skos/core#changeNote -->
<owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#changeNote" />
<!-- http://www.w3.org/2004/02/skos/core#hasTopConcept -->
<owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#hasTopConcept" />
<!-- http://www.w3.org/2004/02/skos/core#inScheme -->
<owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#inScheme" />
<!-- http://www.w3.org/2004/02/skos/core#topConceptOf -->
<owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#topConceptOf" />
<!-- Data properties -->
<!-- http://www.w3.org/2004/02/skos/core#definition -->
<owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#definition" />
<!-- http://www.w3.org/2004/02/skos/core#editorialNote -->
<owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#editorialNote" />
<!-- http://www.w3.org/2004/02/skos/core#historyNote -->
<owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#historyNote" />
<!-- http://www.w3.org/2004/02/skos/core#notation -->
<owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#notation" />
<!-- http://www.w3.org/2004/02/skos/core#prefLabel -->
<owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#prefLabel" />
<!-- Classes -->
<!-- http://www.w3.org/2002/07/owl#Thing -->
<owl:Class rdf:about="http://www.w3.org/2002/07/owl#Thing" />
<!-- http://www.w3.org/2004/02/skos/core#Concept -->
<owl:Class rdf:about="http://www.w3.org/2004/02/skos/core#Concept" />
<!-- http://www.w3.org/2004/02/skos/core#ConceptScheme -->
<owl:Class rdf:about="http://www.w3.org/2004/02/skos/core#ConceptScheme" />
<!-- Individuals -->
</rdf:RDF></nsmap>
<!-- Generated by the Arelle(r) http://arelle.org -->
'''
)
from arelle.ModelObjectFactory import parser
parser, parserLookupName, parserLookupClass = parser(dts,None)
from lxml import etree
xmlDocument = etree.parse(file,parser=parser,base_url=skosFile)
file.close()
xmlRootElement = xmlDocument.getroot()
#xmlDocument.getroot().init(self) ## is this needed ??
for rdfElement in xmlDocument.iter(tag="{http://www.w3.org/1999/02/22-rdf-syntax-ns#}RDF"):
break
numSchemes = 0
numConcepts = 0
# use presentation relationships for broader and narrower concepts
relationshipSet = dts.relationshipSet(XbrlConst.parentChild)
def conceptUri(concept):
return concept.qname.namespaceURI + "#" + concept.qname.localName
def namespaceUri(qname):
return qname.namespaceURI + "#" + qname.prefix
priorSchemeSibling = None
schemeNamespaces = set()
dts.modelManager.showStatus("setting SKOS concepts from XBRL concepts")
for qn, concept in sorted(dts.qnameConcepts.items(), key=lambda item:str(item[0])):
if concept.modelDocument.targetNamespace not in (
XbrlConst.xbrli, XbrlConst.link, XbrlConst.xlink, XbrlConst.xl,
XbrlConst.xbrldt):
if qn.namespaceURI not in schemeNamespaces:
# add conceptScheme
numSchemes += 1
skosElt = etree.Element("{http://www.w3.org/2004/02/skos/core#}ConceptScheme")
skosElt.set("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}about", namespaceUri(qn))
elt = etree.SubElement(skosElt, "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}type")
elt.set("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource",
"http://www.w3.org/2002/07/owl#Thing")
elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}notation")
elt.text = str(qn.prefix)
schemeNamespaces.add(qn.namespaceURI)
if priorSchemeSibling is not None:
priorSchemeSibling.addnext(skosElt)
else:
rdfElement.append(skosElt)
priorSchemeSibling = skosElt
numConcepts += 1
skosElt = etree.SubElement(rdfElement, "{http://www.w3.org/2004/02/skos/core#}Concept")
skosElt.set("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}about", conceptUri(concept))
elt = etree.SubElement(skosElt, "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}type")
elt.set("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource",
"http://www.w3.org/2002/07/owl#Thing")
elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}notation")
elt.text = str(concept.qname)
definition = concept.label(preferredLabel=XbrlConst.documentationLabel, lang="en", strip=True, fallbackToQname=False)
if definition:
elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}definition")
elt.text = definition
else: # if no definition, look for any references
references = viewReferences(concept)
if references:
elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}definition")
elt.text = references
linkedReferenceURI = referenceURI(concept)
if linkedReferenceURI: # link to reference
elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}definition")
elt.text = linkedReferenceURI
labelsRelationshipSet = dts.relationshipSet(XbrlConst.conceptLabel)
if labelsRelationshipSet:
for modelLabelRel in labelsRelationshipSet.fromModelObject(concept):
label = modelLabelRel.toModelObject
if label.role == XbrlConst.standardLabel:
elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}prefLabel")
elt.set("{http://www.w3.org/XML/1998/namespace}lang", label.xmlLang)
elt.text = label.text.strip()
for rel in relationshipSet.fromModelObject(concept): # narrower
elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}narrower")
elt.set("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource", conceptUri(rel.toModelObject))
for rel in relationshipSet.toModelObject(concept): # broader
elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}broader")
elt.set("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource", conceptUri(rel.fromModelObject))
elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}inScheme")
elt.set("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource",
namespaceUri(qn))
dts.modelManager.showStatus("saving SKOS file")
fh = open(skosFile, "w", encoding="utf-8")
XmlUtil.writexml(fh, xmlDocument, encoding="utf-8")
fh.close()
dts.info("info:saveSKOS",
_("SKOS of %(entryFile)s has %(numberOfConcepts)s concepts in SKOS RDF file %(skosOutputFile)s."),
modelObject=dts,
entryFile=dts.uri, numberOfConcepts=numConcepts, skosOutputFile=skosFile)
dts.modelManager.showStatus("ready", 3000)
except Exception as ex:
dts.error("exception",
_("SKOS generation exception: %(error)s"), error=ex,
modelXbrl=dts,
exc_info=True)
def create(modelXbrl, type, uri, schemaRefs=None, isEntry=False):
normalizedUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, None)
if isEntry:
modelXbrl.uri = normalizedUri
modelXbrl.entryLoadingUrl = normalizedUri
modelXbrl.uriDir = os.path.dirname(normalizedUri)
for i in range(modelXbrl.modelManager.disclosureSystem.maxSubmissionSubdirectoryEntryNesting):
modelXbrl.uriDir = os.path.dirname(modelXbrl.uriDir)
filepath = modelXbrl.modelManager.cntlr.webCache.getfilename(normalizedUri)
# XML document has nsmap root element to replace nsmap as new xmlns entries are required
if type == Type.INSTANCE:
# modelXbrl.uriDir = os.path.dirname(normalizedUri)
Xml = ('<nsmap>'
'<xbrl xmlns="http://www.xbrl.org/2003/instance"'
' xmlns:link="http://www.xbrl.org/2003/linkbase"'
' xmlns:xlink="http://www.w3.org/1999/xlink">')
if schemaRefs:
for schemaRef in schemaRefs:
Xml += '<link:schemaRef xlink:type="simple" xlink:href="{0}"/>'.format(schemaRef.replace("\\","/"))
Xml += '</xbrl></nsmap>'
elif type == Type.SCHEMA:
Xml = ('<nsmap><schema xmlns="http://www.w3.org/2001/XMLSchema" /></nsmap>')
elif type == Type.RSSFEED:
Xml = '<nsmap><rss version="2.0" /></nsmap>'
elif type == Type.DTSENTRIES:
Xml = None
else:
type = Type.Unknown
Xml = '<nsmap/>'
if Xml:
import io
file = io.StringIO(Xml)
_parser, _parserLookupName, _parserLookupClass = parser(modelXbrl,filepath)
xmlDocument = etree.parse(file,parser=_parser,base_url=filepath)
file.close()
else:
xmlDocument = None
if type == Type.RSSFEED:
from arelle.ModelRssObject import ModelRssObject
modelDocument = ModelRssObject(modelXbrl, type, uri, filepath, xmlDocument)
else:
modelDocument = ModelDocument(modelXbrl, type, normalizedUri, filepath, xmlDocument)
if Xml:
modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement
modelDocument.parserLookupName = _parserLookupName
modelDocument.parserLookupClass = _parserLookupClass
modelDocument.documentEncoding = "utf-8"
rootNode = xmlDocument.getroot()
rootNode.init(modelDocument)
if xmlDocument:
for semanticRoot in rootNode.iterchildren():
if isinstance(semanticRoot, ModelObject):
modelDocument.xmlRootElement = semanticRoot
break
if type == Type.INSTANCE:
modelDocument.instanceDiscover(modelDocument.xmlRootElement)
elif type == Type.RSSFEED:
modelDocument.rssFeedDiscover(modelDocument.xmlRootElement)
elif type == Type.SCHEMA:
modelDocument.targetNamespace = None
modelDocument.isQualifiedElementFormDefault = False
modelDocument.isQualifiedAttributeFormDefault = False
return modelDocument
def load(modelXbrl, uri, base=None, referringElement=None, isEntry=False, isDiscovered=False, isIncluded=None, namespace=None, reloadCache=False):
if referringElement is None: # used for error messages
referringElement = modelXbrl
normalizedUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, base)
if isEntry:
modelXbrl.entryLoadingUrl = normalizedUri # for error loggiong during loading
modelXbrl.uri = normalizedUri
modelXbrl.uriDir = os.path.dirname(normalizedUri)
for i in range(modelXbrl.modelManager.disclosureSystem.maxSubmissionSubdirectoryEntryNesting):
modelXbrl.uriDir = os.path.dirname(modelXbrl.uriDir)
if modelXbrl.modelManager.validateDisclosureSystem and \
not normalizedUri.startswith(modelXbrl.uriDir) and \
not modelXbrl.modelManager.disclosureSystem.hrefValid(normalizedUri):
blocked = modelXbrl.modelManager.disclosureSystem.blockDisallowedReferences
modelXbrl.error(("EFM.6.22.02", "GFM.1.1.3", "SBR.NL.2.1.0.06"),
_("Prohibited file for filings %(blockedIndicator)s: %(url)s"),
modelObject=referringElement, url=normalizedUri, blockedIndicator=_(" blocked") if blocked else "")
if blocked:
return None
if normalizedUri in modelXbrl.modelManager.disclosureSystem.mappedFiles:
mappedUri = modelXbrl.modelManager.disclosureSystem.mappedFiles[normalizedUri]
else: # handle mapped paths
mappedUri = normalizedUri
for mapFrom, mapTo in modelXbrl.modelManager.disclosureSystem.mappedPaths:
if normalizedUri.startswith(mapFrom):
mappedUri = mapTo + normalizedUri[len(mapFrom):]
break
if isEntry:
modelXbrl.entryLoadingUrl = mappedUri # for error loggiong during loading
if modelXbrl.fileSource.isInArchive(mappedUri):
filepath = mappedUri
else:
filepath = modelXbrl.modelManager.cntlr.webCache.getfilename(mappedUri, reload=reloadCache)
if filepath:
uri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(filepath)
if filepath is None: # error such as HTTPerror is already logged
modelXbrl.error("FileNotLoadable",
_("File can not be loaded: %(fileName)s"),
modelObject=referringElement, fileName=mappedUri)
type = Type.Unknown
return None
modelDocument = modelXbrl.urlDocs.get(mappedUri)
if modelDocument:
return modelDocument
# load XML and determine type of model document
modelXbrl.modelManager.showStatus(_("parsing {0}").format(uri))
file = None
try:
if (modelXbrl.modelManager.validateDisclosureSystem and
modelXbrl.modelManager.disclosureSystem.validateFileText):
file, _encoding = ValidateFilingText.checkfile(modelXbrl,filepath)
else:
file, _encoding = modelXbrl.fileSource.file(filepath)
_parser, _parserLookupName, _parserLookupClass = parser(modelXbrl,filepath)
xmlDocument = etree.parse(file,parser=_parser,base_url=filepath)
file.close()
except (EnvironmentError, KeyError) as err: # missing zip file raises KeyError
modelXbrl.error("IOerror",
_("%(fileName)s: file error: %(error)s"),
modelObject=referringElement, fileName=os.path.basename(uri), error=str(err))
type = Type.Unknown
if file:
file.close()
return None
except (etree.LxmlError,
ValueError) as err: # ValueError raised on bad format of qnames, xmlns'es, or parameters
modelXbrl.error("xmlSchema:syntax",
_("%(error)s, %(fileName)s, %(sourceAction)s source element"),
modelObject=referringElement, fileName=os.path.basename(uri),
error=str(err), sourceAction=("including" if isIncluded else "importing"))
type = Type.Unknown
if file:
file.close()
return None
# identify document
#modelXbrl.modelManager.addToLog("discovery: {0}".format(
# os.path.basename(uri)))
modelXbrl.modelManager.showStatus(_("loading {0}").format(uri))
modelDocument = None
rootNode = xmlDocument.getroot()
if rootNode is not None:
ln = rootNode.localName
ns = rootNode.namespaceURI
# type classification
if ns == XbrlConst.xsd and ln == "schema":
type = Type.SCHEMA
elif ns == XbrlConst.link:
if ln == "linkbase":
type = Type.LINKBASE
elif ln == "xbrl":
type = Type.INSTANCE
elif ns == XbrlConst.xbrli:
if ln == "xbrl":
type = Type.INSTANCE
elif ns == XbrlConst.xhtml and \
(ln == "html" or ln == "xhtml"):
type = Type.Unknown
if XbrlConst.ixbrl in rootNode.nsmap.values():
type = Type.INLINEXBRL
elif ln == "report" and ns == XbrlConst.ver:
type = Type.VERSIONINGREPORT
elif ln == "testcases" or ln == "documentation":
type = Type.TESTCASESINDEX
elif ln == "testcase":
type = Type.TESTCASE
elif ln == "registry" and ns == XbrlConst.registry:
type = Type.REGISTRY
elif ln == "rss":
type = Type.RSSFEED
else:
type = Type.Unknown
nestedInline = None
for htmlElt in rootNode.iter(tag="{http://www.w3.org/1999/xhtml}html"):
nestedInline = htmlElt
break
if nestedInline is None:
for htmlElt in rootNode.iter(tag="{http://www.w3.org/1999/xhtml}xhtml"):
nestedInline = htmlElt
break
if nestedInline is not None:
if XbrlConst.ixbrl in nestedInline.nsmap.values():
type = Type.INLINEXBRL
rootNode = nestedInline
#create modelDocument object or subtype as identified
if type == Type.VERSIONINGREPORT:
from arelle.ModelVersReport import ModelVersReport
modelDocument = ModelVersReport(modelXbrl, type, mappedUri, filepath, xmlDocument)
elif type == Type.RSSFEED:
from arelle.ModelRssObject import ModelRssObject
modelDocument = ModelRssObject(modelXbrl, type, mappedUri, filepath, xmlDocument)
else:
modelDocument = ModelDocument(modelXbrl, type, mappedUri, filepath, xmlDocument)
rootNode.init(modelDocument)
modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement
modelDocument.parserLookupName = _parserLookupName
modelDocument.parserLookupClass = _parserLookupClass
modelDocument.xmlRootElement = rootNode
modelDocument.schemaLocationElements.add(rootNode)
modelDocument.documentEncoding = _encoding
if isEntry or isDiscovered:
modelDocument.inDTS = True
# discovery (parsing)
if type == Type.SCHEMA:
modelDocument.schemaDiscover(rootNode, isIncluded, namespace)
elif type == Type.LINKBASE:
modelDocument.linkbaseDiscover(rootNode)
elif type == Type.INSTANCE:
modelDocument.instanceDiscover(rootNode)
elif type == Type.INLINEXBRL:
modelDocument.inlineXbrlDiscover(rootNode)
elif type == Type.VERSIONINGREPORT:
modelDocument.versioningReportDiscover(rootNode)
elif type == Type.TESTCASESINDEX:
modelDocument.testcasesIndexDiscover(xmlDocument)
elif type == Type.TESTCASE:
modelDocument.testcaseDiscover(rootNode)
elif type == Type.REGISTRY:
modelDocument.registryDiscover(rootNode)
elif type == Type.VERSIONINGREPORT:
modelDocument.versioningReportDiscover(rootNode)
elif type == Type.RSSFEED:
modelDocument.rssFeedDiscover(rootNode)
return modelDocument
def generateHtmlEbaTablesetFiles(dts, indexFile, lang="en"):
try:
import os, io
from arelle import Version, XbrlConst, XmlUtil
from arelle.ViewFileRenderedGrid import viewRenderedGrid
from arelle.ModelRenderingObject import ModelEuTable, ModelTable
numTableFiles = 0
file = io.StringIO('''
<html xmlns="http://www.w3.org/1999/xhtml">
<head id="Left">
<link type="text/css" rel="stylesheet" href="http://arelle.org/files/EBA/style20121210/eba.css" />
</head>
<body class="LTR IE7 ENGB">
<ul class="CMSListMenuUL" id="Vertical2"/>
</body>
</html>
''')
from arelle.ModelObjectFactory import parser
parser, parserLookupName, parserLookupClass = parser(dts, None)
from lxml import etree
indexDocument = etree.parse(file, parser=parser, base_url=indexFile)
file.close()
#xmlDocument.getroot().init(self) ## is this needed ??
for listElt in indexDocument.iter(
tag="{http://www.w3.org/1999/xhtml}ul"):
break
class nonTkBooleanVar():
def __init__(self, value=True):
self.value = value
def set(self, value):
self.value = value
def get(self):
return self.value
class View():
def __init__(self, tableOrELR, ignoreDimValidity,
xAxisChildrenFirst, yAxisChildrenFirst):
self.tblELR = tableOrELR
# context menu boolean vars (non-tkinter boolean
self.ignoreDimValidity = nonTkBooleanVar(
value=ignoreDimValidity)
self.xAxisChildrenFirst = nonTkBooleanVar(
value=xAxisChildrenFirst)
self.yAxisChildrenFirst = nonTkBooleanVar(
value=yAxisChildrenFirst)
indexBase = indexFile.rpartition(".")[0]
groupTableRels = dts.modelXbrl.relationshipSet(XbrlConst.euGroupTable)
modelTables = []
tblCssExtras = '''
body {background-image:url('http://arelle.org/files/EBA/style20121210/lhsbackground.jpg')}
table {background:#fff}
'''
# order number is missing
def viewTable(modelTable):
if modelTable is None:
return
if isinstance(modelTable, (ModelEuTable, ModelTable)):
# status
dts.modelManager.cntlr.addToLog("viewing: " + modelTable.id)
# for table file name, use table ELR
tblFile = os.path.join(os.path.dirname(indexFile),
modelTable.id + ".html")
viewRenderedGrid(dts,
tblFile,
lang=lang,
sourceView=View(modelTable, False, False,
True),
cssExtras=tblCssExtras)
# generaate menu entry
elt = etree.SubElement(listElt,
"{http://www.w3.org/1999/xhtml}li")
elt.set("class", "CMSListMenuLI")
elt.set("id", modelTable.id)
elt = etree.SubElement(elt, "{http://www.w3.org/1999/xhtml}a")
elt.text = modelTable.genLabel(lang=lang, strip=True)
elt.set("class", "CMSListMenuLink")
elt.set("href", "javascript:void(0)")
elt.set(
"onClick",
"javascript:parent.body.location.href='{0}';".format(
modelTable.id + ".html"))
elt.text = modelTable.genLabel(lang=lang, strip=True)
else: # just a header
# generaate menu entry
elt = etree.SubElement(listElt,
"{http://www.w3.org/1999/xhtml}li")
elt.set("class", "CMSListMenuLink")
elt.set("id", modelTable.id)
elt.text = modelTable.label(lang=lang, strip=True)
for rel in groupTableRels.fromModelObject(modelTable):
viewTable(rel.toModelObject)
for rootConcept in groupTableRels.rootConcepts:
sourceline = 0
for rel in dts.modelXbrl.relationshipSet(
XbrlConst.euGroupTable).fromModelObject(rootConcept):
sourceline = rel.sourceline
break
modelTables.append((rootConcept, sourceline))
for modelTable, order in sorted(modelTables, key=lambda x: x[1]):
viewTable(modelTable)
with open(indexBase + "FormsFrame.html", "wt", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocument, encoding="utf-8")
with open(indexFile, "wt", encoding="utf-8") as fh:
fh.write('''
<html xmlns="http://www.w3.org/1999/xhtml">
<head id="Head1">
<title>European Banking Authority - EBA - FINREP Taxonomy</title>
<meta name="generator" content="Arelle(r) {0}" />
<meta name="provider" content="Aguilonius(r)" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" />
<meta http-equiv="pragma" content="no-cache" />
<meta http-equiv="content-style-type" content="text/css" />
<meta http-equiv="content-script-type" content="text/javascript" />
<link type="text/css" rel="stylesheet" href="http://arelle.org/files/EBA/style20121210/eba.css" />
</head>
<frameset border="0" frameborder="0" rows="90,*">
<frame name="head" src="{1}" scrolling="no" marginwidth="0" marginheight="10"/>
<frameset bordercolor="#0000cc" border="10" frameborder="no" framespacing="0" cols="360, *">
<frame src="{2}" name="menu" bordercolor="#0000cc"/>
<frame src="{3}" name="body" bordercolor="#0000cc"/>
</frameset>
</frameset>
'''.format(
Version.version,
os.path.basename(indexBase) + "TopFrame.html",
os.path.basename(indexBase) + "FormsFrame.html",
os.path.basename(indexBase) + "CenterLanding.html",
))
with open(indexBase + "TopFrame.html", "wt", encoding="utf-8") as fh:
fh.write('''
<html xmlns="http://www.w3.org/1999/xhtml">
<head id="Top">
<link type="text/css" rel="stylesheet" href="http://arelle.org/files/EBA/style20121210/eba.css" />
</head>
<body class="LTR IE7 ENGB">
<div id="topsection">
<div id="topsectionLeft" style="cursor:pointer;" onclick="location.href='http://www.eba.europa.eu/home.aspx';"></div>
<div id="topsectionRight"></div>
<div id="topnavigation">
<ul id="menuElem">
<li><a href="http://www.eba.europa.eu/topnav/Contacts.aspx">Contacts</a></li>
<li><a href="http://www.eba.europa.eu/topnav/Links.aspx">Links</a></li>
<li><a href="http://www.eba.europa.eu/topnav/Sitemap.aspx">Sitemap</a></li>
<li><a href="http://www.eba.europa.eu/topnav/Legal-Notice.aspx">Legal Notice</a></li>
</ul>
</div>
</body>
</html>
''')
with open(indexBase + "CenterLanding.html", "wt",
encoding="utf-8") as fh:
fh.write('''
<html xmlns="http://www.w3.org/1999/xhtml">
<head id="Center">
<link type="text/css" rel="stylesheet" href="http://http://arelle.org/files/EBA/style20121210/eba.css" />
</head>
<body class="LTR IE7 ENGB">
<div id="plc_lt_zoneContent_usercontrol_userControlElem_ContentPanel">
<div id="plc_lt_zoneContent_usercontrol_userControlElem_PanelTitle">
<div id="pagetitle" style="float:left;width:500px;">
<h1>Taxonomy Tables Viewer</h1>
</div>
</div>
</div>
<div style="clear:both;"></div>
<div id="contentcenter">
<p style="text-align: justify; margin-top: 0pt; margin-bottom: 0pt">Please select tables to view by clicking in the left column.</p>
</div>
</body>
</html>
''')
# to merge gif's and style sheets, use a zipfile sibling of the python plug-in file.
#import zipfile
#zf = zipfile.ZipFile(__file__.rpartition('.')[0] + "Files.zip", mode="r")
#zf.extractall(path=os.path.dirname(indexBase))
#zf.close()
dts.info(
"info:saveEBAtables",
_("Tables index file of %(entryFile)s has %(numberTableFiles)s table files with index file %(indexFile)s."
),
modelObject=dts,
entryFile=dts.uri,
numberTableFiles=numTableFiles,
indexFile=indexFile)
dts.modelManager.showStatus(_("Saved EBA HTML Table Files"), 5000)
except Exception as ex:
dts.error("exception",
_("HTML EBA Tableset files generation exception: %(error)s"),
error=ex,
modelXbrl=dts,
exc_info=True)
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, **kwargs):
# check if big instance and has header with an initial incomplete tree walk (just 2 elements
if not _streamingExtensionsCheck:
return None
# track whether modelXbrl has been validated by this streaming extension
modelXbrl._streamingExtensionValidated = False
def logSyntaxErrors(parsercontext):
for error in parsercontext.error_log:
modelXbrl.error("xmlSchema:syntax",
_("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"),
modelObject=modelXbrl, fileName=os.path.basename(filepath),
error=error.message, line=error.line, column=error.column, sourceAction="streaming")
#### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
#### note: iterparse wants a binary file, but file is text mode
_file, = modelXbrl.fileSource.file(filepath, binary=True)
startedAt = time.time()
modelXbrl.profileActivity()
''' this seems twice as slow as iterparse
class instInfoTarget():
def __init__(self, element_factory=None, parser=None):
self.newTree = True
self.streamingAspects = None
self.foundInstance = False
self.creationSoftwareComment = ''
self.currentEltTag = "(before xbrli:xbrl)"
self.numRootFacts = 0
def start(self, tag, attrib, nsmap=None):
if self.newTree:
if tag == "{http://www.xbrl.org/2003/instance}xbrl":
self.foundInstance = True
self.newTree = False
else: # break
raise NotInstanceDocumentException()
elif not tag.startswith("{http://www.xbrl.org/"):
self.numRootFacts += 1
if self.numRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
self.currentEltTag = tag
def end(self, tag):
pass
def data(self, data):
pass
def comment(self, text):
if not self.foundInstance: # accumulate comments before xbrli:xbrl
self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text
elif not self.creationSoftwareComment:
self.creationSoftwareComment = text # or first comment after xbrli:xbrl
def pi(self, target, data):
if target == "xbrl-streamable-instance":
if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl":
self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results
else:
modelXbrl.error("streamingExtensions:headerMisplaced",
_("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"),
modelObject=modelXbrl, target=target, element=self.currentEltTag)
def close(self):
if not self.creationSoftwareComment:
self.creationSoftwareComment = None
return True
instInfo = instInfoTarget()
infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo)
try:
etree.parse(_file, parser=infoParser, base_url=filepath)
except NotInstanceDocumentException:
pass
'''
foundErrors = False
foundInstance = False
streamingAspects = None
creationSoftwareComment = None
instInfoNumRootFacts = 0
numElts = 0
elt = None
instInfoContext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
for event, elt in instInfoContext:
if event == "start":
if elt.getparent() is not None:
if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
if not foundInstance:
foundInstance = True
pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance")
if pi is None:
break
else:
streamingAspects = dict(pi.attrib.copy())
if creationSoftwareComment is None:
creationSoftwareComment = precedingComment(elt)
if not elt.tag.startswith("{http://www.xbrl.org/"):
instInfoNumRootFacts += 1
if instInfoNumRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
elif not foundInstance:
break
elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl":
creationSoftwareComment = precedingComment(elt)
if precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None:
modelXbrl.error("streamingExtensions:headerMisplaced",
_("Header is misplaced: %(error)s, must follow xbrli:xbrl element"),
modelObject=elt)
elif event == "end":
elt.clear()
numElts += 1
if numElts % 1000 == 0 and elt.getparent() is not None:
while elt.getprevious() is not None and elt.getparent() is not None:
del elt.getparent()[0]
if elt is not None:
elt.clear()
_file.seek(0,io.SEEK_SET) # allow reparsing
if not foundInstance or streamingAspects is None:
del elt
_file.close()
return None
modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
startedAt = time.time()
try:
version = Decimal(streamingAspects.get("version"))
if int(version) != 1:
modelXbrl.error("streamingExtensions:unsupportedVersion",
_("Streaming version %(version)s, major version number must be 1"),
modelObject=elt, version=version)
foundErrors = True
except (InvalidOperation, OverflowError):
modelXbrl.error("streamingExtensions:versionError",
_("Version %(version)s, number must be 1.n"),
modelObject=elt, version=streamingAspects.get("version", "(none)"))
foundErrors = True
for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
try:
bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
raise InvalidOperation
elif bufAspect == "contextBuffer":
contextBufferLimit = bufLimit
elif bufAspect == "unitBuffer":
unitBufferLimit = bufLimit
elif bufAspect == "footnoteBuffer":
footnoteBufferLimit = bufLimit
except InvalidOperation:
modelXbrl.error("streamingExtensions:valueError",
_("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"),
modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect))
foundErrors = True
if _streamingExtensionsValidate:
incompatibleValidations = []
_validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
_disclosureSystem = modelXbrl.modelManager.disclosureSystem
if _validateDisclosureSystem and _disclosureSystem.EFM:
incompatibleValidations.append("EFM")
if _validateDisclosureSystem and _disclosureSystem.GFM:
incompatibleValidations.append("GFM")
if _validateDisclosureSystem and _disclosureSystem.EBA:
incompatibleValidations.append("EBA")
if _validateDisclosureSystem and _disclosureSystem.HMRC:
incompatibleValidations.append("EBA")
if modelXbrl.modelManager.validateCalcLB:
incompatibleValidations.append("calculation LB")
if incompatibleValidations:
modelXbrl.error("streamingExtensions:incompatibleValidation",
_("Streaming instance validation does not support %(incompatibleValidations)s validation"),
modelObject=modelXbrl, incompatibleValidations=', '.join(incompatibleValidations))
foundErrors = True
if instInfoContext.error_log:
foundErrors = True
logSyntaxErrors(instInfoContext)
del instInfoContext # dereference
for pluginMethod in pluginClassMethods("Streaming.BlockStreaming"):
_blockingPluginName = pluginMethod(modelXbrl)
if _blockingPluginName: # name of blocking plugin is returned
modelXbrl.error("streamingExtensions:incompatiblePlugIn",
_("Streaming instance not supported by plugin %(blockingPlugin)s"),
modelObject=modelXbrl, blockingPlugin=_blockingPluginName)
foundErrors = True
if foundErrors:
_file.close()
return None
_encoding = XmlUtil.encoding(_file.read(512))
_file.seek(0,io.SEEK_SET) # allow reparsing
if _streamingExtensionsValidate:
validator = Validate(modelXbrl)
instValidator = validator.instValidator
eltMdlObjs = {}
contextBuffer = []
unitBuffer = []
footnoteBuffer = []
factBuffer = []
numFacts = 1
_streamingValidateFactsPlugin = any(True for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"))
class modelLoaderTarget():
def __init__(self, element_factory=None, parser=None):
self.newTree = True
self.currentMdlObj = None
self.beforeInstanceStream = True
self.beforeStartStreamingPlugin = True
self.numRootFacts = 1
modelXbrl.streamingParentModelObject = None
modelXbrl.isStreamingMode = True
def start(self, tag, attrib, nsmap=None):
modelXbrl.streamingParentModelObject = self.currentMdlObj # pass parent to makeelement for ModelObjectFactory
mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
mdlObj.sourceline = 1
if self.newTree:
self.newTree = False
self.currentMdlObj = mdlObj
modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
modelXbrl.modelDocument = modelDocument # needed for incremental validation
mdlObj.init(modelDocument)
modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement
modelDocument.parserLookupName = _parserLookupName
modelDocument.parserLookupClass = _parserLookupClass
modelDocument.xmlRootElement = mdlObj
modelDocument.schemaLocationElements.add(mdlObj)
modelDocument.documentEncoding = _encoding
modelDocument._creationSoftwareComment = creationSoftwareComment
modelXbrl.info("streamingExtensions:streaming",
_("Stream processing this instance."),
modelObject = modelDocument)
else:
self.currentMdlObj.append(mdlObj)
self.currentMdlObj = mdlObj
mdlObj._init()
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if (self.beforeInstanceStream and (
(ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
(ns == XbrlConst.xbrli and ln in ("context", "unit")) or
(ns not in (XbrlConst.link, XbrlConst.xbrli)))):
self.beforeInstanceStream = False
if _streamingExtensionsValidate:
instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters())
else: # need default dimensions
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
elif not self.beforeInstanceStream and self.beforeStartStreamingPlugin:
for pluginMethod in pluginClassMethods("Streaming.Start"):
pluginMethod(modelXbrl)
self.beforeStartStreamingPlugin = False
return mdlObj
def end(self, tag):
modelDocument = modelXbrl.modelDocument
mdlObj = self.currentMdlObj
parentMdlObj = mdlObj.getparent()
self.currentMdlObj = parentMdlObj
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
else:
if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj,)
instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
instValidator.checkContextsDimensions(contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
# drop before additing as dropped may have same id as added
unit = unitBuffer.pop(0)
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
unit = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkUnits( (mdlObj,) )
elif ln == "xbrl": # end of document
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
for pluginMethod in pluginClassMethods("Streaming.Finish"):
pluginMethod(modelXbrl)
elif ns == XbrlConst.link:
if ln == "footnoteLink":
XmlValidate.validate(modelXbrl, mdlObj)
footnoteLinks = (mdlObj,)
modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elif ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(mdlObj)
elif not modelXbrl.skipDTS:
if ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
self.numRootFacts += 1
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.factDiscover(mdlObj, modelXbrl.facts)
if _streamingExtensionsValidate or _streamingValidateFactsPlugin:
factsToCheck = (mdlObj,) # validate current fact by itself
if _streamingExtensionsValidate:
instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
instValidator.checkFactsDimensions(factsToCheck)
if _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
factsToCheck = modelXbrl.facts.copy()
factsHaveBeenProcessed = True
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
if not pluginMethod(modelXbrl, factsToCheck):
factsHaveBeenProcessed = False
if factsHaveBeenProcessed:
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(fact)]
else:
dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed
del parentMdlObj[parentMdlObj.index(mdlObj)]
del factsToCheck # dereference fact or batch of facts
if self.numRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts,
100.0 * self.numRootFacts / instInfoNumRootFacts),
minTimeToShow=20.0)
return mdlObj
def data(self, data):
self.currentMdlObj.text = data
def comment(self, text):
pass
def pi(self, target, data):
pass
def close(self):
del modelXbrl.streamingParentModelObject
return None
_parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget())
etree.parse(_file, parser=_parser, base_url=filepath)
logSyntaxErrors(_parser)
if _streamingExtensionsValidate and validator is not None:
_file.close()
del instValidator
validator.close()
# track that modelXbrl has been validated by this streaming extension
modelXbrl._streamingExtensionValidated = True
modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
return modelXbrl.modelDocument
def generateSkos(dts, skosFile):
try:
import os, io
from arelle import XmlUtil, XbrlConst
from arelle.ViewUtil import viewReferences, referenceURI
skosNs = "http://www.w3.org/2004/02/skos/core#"
rdfNs = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
dts.modelManager.showStatus("initializing SKOS document")
file = io.StringIO('''
<!DOCTYPE rdf:RDF>
<nsmap>
<rdf:RDF xmlns="urn:cgi:classifier:CGI:XBRL:201204#" xml:base="urn:cgi:classifierScheme:CGI:XBRL:201204" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xmlns:owl2xml="http://www.w3.org/2006/12/owl2-xml#" xmlns:p1="#" xmlns:owl="http://www.w3.org/2002/07/owl#" xmlns:xsd="http://www.w3.org/2001/XMLSchema#" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:xbrl-201204="urn:cgi:classifier:CGI:XBRL:201204#" xmlns:skos="http://www.w3.org/2004/02/skos/core#">
<owl:Ontology rdf:about="" />
<!-- Annotation properties -->
<owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/date" />
<owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/source" />
<owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/title" />
<owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/description" />
<owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/contributor" />
<owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/creator" />
<owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/format" />
<owl:AnnotationProperty rdf:about="http://www.w3.org/2002/07/owl#versionInfo" />
<!-- Object Properties -->
<!-- http://www.w3.org/2004/02/skos/core#broader -->
<owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#broader" />
<!-- http://www.w3.org/2004/02/skos/core#changeNote -->
<owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#changeNote" />
<!-- http://www.w3.org/2004/02/skos/core#hasTopConcept -->
<owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#hasTopConcept" />
<!-- http://www.w3.org/2004/02/skos/core#inScheme -->
<owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#inScheme" />
<!-- http://www.w3.org/2004/02/skos/core#topConceptOf -->
<owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#topConceptOf" />
<!-- Data properties -->
<!-- http://www.w3.org/2004/02/skos/core#definition -->
<owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#definition" />
<!-- http://www.w3.org/2004/02/skos/core#editorialNote -->
<owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#editorialNote" />
<!-- http://www.w3.org/2004/02/skos/core#historyNote -->
<owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#historyNote" />
<!-- http://www.w3.org/2004/02/skos/core#notation -->
<owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#notation" />
<!-- http://www.w3.org/2004/02/skos/core#prefLabel -->
<owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#prefLabel" />
<!-- Classes -->
<!-- http://www.w3.org/2002/07/owl#Thing -->
<owl:Class rdf:about="http://www.w3.org/2002/07/owl#Thing" />
<!-- http://www.w3.org/2004/02/skos/core#Concept -->
<owl:Class rdf:about="http://www.w3.org/2004/02/skos/core#Concept" />
<!-- http://www.w3.org/2004/02/skos/core#ConceptScheme -->
<owl:Class rdf:about="http://www.w3.org/2004/02/skos/core#ConceptScheme" />
<!-- Individuals -->
</rdf:RDF></nsmap>
<!-- Generated by the Arelle(r) http://arelle.org -->
''')
from arelle.ModelObjectFactory import parser
parser, parserLookupName, parserLookupClass = parser(dts, None)
from lxml import etree
xmlDocument = etree.parse(file, parser=parser, base_url=skosFile)
file.close()
xmlRootElement = xmlDocument.getroot()
#xmlDocument.getroot().init(self) ## is this needed ??
for rdfElement in xmlDocument.iter(
tag="{http://www.w3.org/1999/02/22-rdf-syntax-ns#}RDF"):
break
numSchemes = 0
numConcepts = 0
# use presentation relationships for broader and narrower concepts
relationshipSet = dts.relationshipSet(XbrlConst.parentChild)
def conceptUri(concept):
return concept.qname.namespaceURI + "#" + concept.qname.localName
def namespaceUri(qname):
return qname.namespaceURI + "#" + qname.prefix
priorSchemeSibling = None
schemeNamespaces = set()
dts.modelManager.showStatus("setting SKOS concepts from XBRL concepts")
for qn, concept in sorted(dts.qnameConcepts.items(),
key=lambda item: str(item[0])):
if concept.modelDocument.targetNamespace not in (XbrlConst.xbrli,
XbrlConst.link,
XbrlConst.xlink,
XbrlConst.xl,
XbrlConst.xbrldt):
if qn.namespaceURI not in schemeNamespaces:
# add conceptScheme
numSchemes += 1
skosElt = etree.Element(
"{http://www.w3.org/2004/02/skos/core#}ConceptScheme")
skosElt.set(
"{http://www.w3.org/1999/02/22-rdf-syntax-ns#}about",
namespaceUri(qn))
elt = etree.SubElement(
skosElt,
"{http://www.w3.org/1999/02/22-rdf-syntax-ns#}type")
elt.set(
"{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource",
"http://www.w3.org/2002/07/owl#Thing")
elt = etree.SubElement(
skosElt,
"{http://www.w3.org/2004/02/skos/core#}notation")
elt.text = str(qn.prefix)
schemeNamespaces.add(qn.namespaceURI)
if priorSchemeSibling is not None:
priorSchemeSibling.addnext(skosElt)
else:
rdfElement.append(skosElt)
priorSchemeSibling = skosElt
numConcepts += 1
skosElt = etree.SubElement(
rdfElement,
"{http://www.w3.org/2004/02/skos/core#}Concept")
skosElt.set(
"{http://www.w3.org/1999/02/22-rdf-syntax-ns#}about",
conceptUri(concept))
elt = etree.SubElement(
skosElt,
"{http://www.w3.org/1999/02/22-rdf-syntax-ns#}type")
elt.set(
"{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource",
"http://www.w3.org/2002/07/owl#Thing")
elt = etree.SubElement(
skosElt, "{http://www.w3.org/2004/02/skos/core#}notation")
elt.text = str(concept.qname)
definition = concept.label(
preferredLabel=XbrlConst.documentationLabel,
lang="en",
strip=True,
fallbackToQname=False)
if definition:
elt = etree.SubElement(
skosElt,
"{http://www.w3.org/2004/02/skos/core#}definition")
elt.text = definition
else: # if no definition, look for any references
references = viewReferences(concept)
if references:
elt = etree.SubElement(
skosElt,
"{http://www.w3.org/2004/02/skos/core#}definition")
elt.text = references
linkedReferenceURI = referenceURI(concept)
if linkedReferenceURI: # link to reference
elt = etree.SubElement(
skosElt,
"{http://www.w3.org/2004/02/skos/core#}definition")
elt.text = linkedReferenceURI
labelsRelationshipSet = dts.relationshipSet(
XbrlConst.conceptLabel)
if labelsRelationshipSet:
for modelLabelRel in labelsRelationshipSet.fromModelObject(
concept):
label = modelLabelRel.toModelObject
if label.role == XbrlConst.standardLabel:
elt = etree.SubElement(
skosElt,
"{http://www.w3.org/2004/02/skos/core#}prefLabel"
)
elt.set(
"{http://www.w3.org/XML/1998/namespace}lang",
label.xmlLang)
elt.text = label.text.strip()
for rel in relationshipSet.fromModelObject(
concept): # narrower
elt = etree.SubElement(
skosElt,
"{http://www.w3.org/2004/02/skos/core#}narrower")
elt.set(
"{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource",
conceptUri(rel.toModelObject))
for rel in relationshipSet.toModelObject(concept): # broader
elt = etree.SubElement(
skosElt,
"{http://www.w3.org/2004/02/skos/core#}broader")
elt.set(
"{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource",
conceptUri(rel.fromModelObject))
elt = etree.SubElement(
skosElt, "{http://www.w3.org/2004/02/skos/core#}inScheme")
elt.set(
"{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource",
namespaceUri(qn))
dts.modelManager.showStatus("saving SKOS file")
fh = open(skosFile, "w", encoding="utf-8")
XmlUtil.writexml(fh, xmlDocument, encoding="utf-8")
fh.close()
dts.info(
"info:saveSKOS",
_("SKOS of %(entryFile)s has %(numberOfConcepts)s concepts in SKOS RDF file %(skosOutputFile)s."
),
modelObject=dts,
entryFile=dts.uri,
numberOfConcepts=numConcepts,
skosOutputFile=skosFile)
dts.modelManager.showStatus("ready", 3000)
except Exception as ex:
dts.error("exception",
_("SKOS generation exception: %(error)s"),
error=ex,
modelXbrl=dts,
exc_info=True)
def generateUpdatedTableLB(dts, updatedTableLinkbaseFile):
import os, io
from arelle import XmlUtil, XbrlConst
from arelle.ViewUtil import viewReferences, referenceURI
from arelle.ModelRenderingObject import ModelEuAxisCoord
if dts.fileSource.isArchive:
dts.error(
"genTblLB:outFileIsArchive",
_("Updated Table Linkbase file cannot be an archive: %(tableLBOutputFile)s."
),
modelObject=dts,
tableLBOutputFile=updatedTableLinkbaseFile)
return
tblAxisRelSet = dts.relationshipSet(XbrlConst.euTableAxis)
axisMbrRelSet = dts.relationshipSet(XbrlConst.euAxisMember)
if len(tblAxisRelSet.modelRelationships) == 0:
dts.error(
"genTblLB:noInputTables",
_("DTS does not contain Eurofiling 2010 tables and axes: %(entryFile)s."
),
modelObject=dts,
entryFile=dts.uri)
return
file = io.StringIO('''
<nsmap>
<link:linkbase
xmlns:label="http://xbrl.org/2008/label"
xmlns:gen="http://xbrl.org/2008/generic"
xmlns:df="http://xbrl.org/2008/filter/dimension"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:reference="http://xbrl.org/2008/reference"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:table="http://xbrl.org/2011/table"
xmlns:formula="http://xbrl.org/2008/formula"
xsi:schemaLocation="
http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd
http://xbrl.org/2008/generic http://www.xbrl.org/2008/generic-link.xsd
http://xbrl.org/2008/reference http://www.xbrl.org/2008/generic-reference.xsd
http://xbrl.org/2008/label http://www.xbrl.org/2008/generic-label.xsd
http://xbrl.org/2011/table http://www.xbrl.org/2011/table.xsd
http://xbrl.org/2008/filter/dimension http://www.xbrl.org/2008/dimension-filter.xsd">
<link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/table-filter" xlink:type="simple"
xlink:href="http://www.xbrl.org/2011/table.xsd#table-filter"/>
<link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/table-axis" xlink:type="simple"
xlink:href="http://www.xbrl.org/2011/table.xsd#table-axis"/>
<link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/axis-subtree" xlink:type="simple"
xlink:href="http://www.xbrl.org/2011/table.xsd#axis-subtree"/>
<link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/axis-filter" xlink:type="simple"
xlink:href="http://www.xbrl.org/2011/filter-axis.xsd#axis-filter"/>
</link:linkbase>
</nsmap>
<!-- Generated by Arelle(r) http://arelle.org -->
''')
from arelle.ModelObjectFactory import parser
parser, parserLookupName, parserLookupClass = parser(dts, None)
from lxml import etree
xmlDocument = etree.parse(file,
parser=parser,
base_url=updatedTableLinkbaseFile)
file.close()
nsmapElt = xmlDocument.getroot()
#xmlDocument.getroot().init(self) ## is this needed ??
for lbElement in xmlDocument.iter(
tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
break
class DocObj: # fake ModelDocument for namespaces
def __init__(self):
self.xmlRootElement = lbElement
self.xmlDocument = xmlDocument
docObj = DocObj()
numELRs = 0
numTables = 0
def copyAttrs(fromElt, toElt, attrTags):
for attr in attrTags:
if fromElt.get(attr):
toElt.set(attr, fromElt.get(attr))
def generateTable(newLinkElt, newTblElt, srcTblElt, tblAxisRelSet,
axisMbrRelSet, visited):
if srcTblElt is not None:
for rel in tblAxisRelSet.fromModelObject(srcTblElt):
srcAxisElt = rel.toModelObject
if isinstance(srcAxisElt, ModelEuAxisCoord):
visited.add(srcAxisElt)
newAxisElt = etree.SubElement(
newLinkElt, "{http://xbrl.org/2011/table}ruleAxis")
copyAttrs(srcAxisElt, newAxisElt,
("id", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}label"))
newAxisElt.set("abstract",
"true") # always true on root element
newArcElt = etree.SubElement(
newLinkElt, "{http://xbrl.org/2011/table}axisArc")
copyAttrs(rel, newArcElt,
("id", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}from",
"{http://www.w3.org/1999/xlink}to", "order"))
newArcElt.set("{http://www.w3.org/1999/xlink}arcrole",
XbrlConst.tableBreakdown)
newArcElt.set("axisDisposition", rel.axisDisposition)
generateAxis(newLinkElt, newAxisElt, srcAxisElt,
axisMbrRelSet, visited)
visited.discard(srcAxisElt)
def generateAxis(newLinkElt, newAxisParentElt, srcAxisElt, axisMbrRelSet,
visited):
for rel in axisMbrRelSet.fromModelObject(srcAxisElt):
tgtAxisElt = rel.toModelObject
if isinstance(tgtAxisElt,
ModelEuAxisCoord) and tgtAxisElt not in visited:
visited.add(tgtAxisElt)
newAxisElt = etree.SubElement(
newLinkElt, "{http://xbrl.org/2011/table}ruleAxis")
copyAttrs(
tgtAxisElt, newAxisElt,
("id", "abstract", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}label"))
if tgtAxisElt.primaryItemQname:
newRuleElt = etree.SubElement(
newAxisElt, "{http://xbrl.org/2008/formula}concept")
newQnameElt = etree.SubElement(
newRuleElt, "{http://xbrl.org/2008/formula}qname")
newQnameElt.text = XmlUtil.addQnameValue(
docObj, tgtAxisElt.primaryItemQname)
for dimQname, memQname in tgtAxisElt.explicitDims:
newRuleElt = etree.SubElement(
newAxisElt,
"{http://xbrl.org/2008/formula}explicitDimension")
newRuleElt.set("dimension",
XmlUtil.addQnameValue(docObj, dimQname))
newMbrElt = etree.SubElement(
newRuleElt, "{http://xbrl.org/2008/formula}member")
newQnameElt = etree.SubElement(
newMbrElt, "{http://xbrl.org/2008/formula}qname")
newQnameElt.text = XmlUtil.addQnameValue(docObj, memQname)
newArcElt = etree.SubElement(
newLinkElt, "{http://xbrl.org/2011/table}axisArc")
copyAttrs(rel, newArcElt,
("id", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}from",
"{http://www.w3.org/1999/xlink}to", "order"))
newArcElt.set("{http://www.w3.org/1999/xlink}arcrole",
XbrlConst.tableAxisSubtree)
generateAxis(newLinkElt, newAxisElt, tgtAxisElt, axisMbrRelSet,
visited)
visited.discard(tgtAxisElt)
# sort URIs
linkroleUris = sorted(
[linkroleUri for linkroleUri in tblAxisRelSet.linkRoleUris])
firstNewLinkElt = None
roleRefUris = set()
for linkroleUri in linkroleUris:
numELRs += 1
newLinkElt = etree.SubElement(lbElement,
"{http://xbrl.org/2008/generic}link")
newLinkElt.set("{http://www.w3.org/1999/xlink}type", "extended")
newLinkElt.set("{http://www.w3.org/1999/xlink}role", linkroleUri)
if firstNewLinkElt is None: firstNewLinkElt = newLinkElt
# To do: add roleRef if needed
tblAxisRelSet = dts.relationshipSet(XbrlConst.euTableAxis, linkroleUri)
axisMbrRelSet = dts.relationshipSet(XbrlConst.euAxisMember,
linkroleUri)
for srcTblElt in tblAxisRelSet.rootConcepts:
if srcTblElt.tag == "{http://www.eurofiling.info/2010/rendering}table":
numTables += 1
newTblElt = etree.SubElement(
newLinkElt, "{http://xbrl.org/2011/table}table")
newTblElt.set("aspectModel", "dimensional")
copyAttrs(srcTblElt, newTblElt,
("id", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}label"))
generateTable(newLinkElt, newTblElt, srcTblElt, tblAxisRelSet,
axisMbrRelSet, set())
if linkroleUri not in roleRefUris:
srcRoleRefElt = XmlUtil.descendant(srcTblElt.getroottree(),
XbrlConst.link,
"roleRef", "roleURI",
linkroleUri)
if srcRoleRefElt is not None:
roleRefUris.add(linkroleUri)
newRoleRefElt = etree.Element(
"{http://www.xbrl.org/2003/linkbase}roleRef")
copyAttrs(
srcRoleRefElt, newRoleRefElt,
("roleURI", "{http://www.w3.org/1999/xlink}type",
"{http://www.w3.org/1999/xlink}href"))
firstNewLinkElt.addprevious(newRoleRefElt)
fh = open(updatedTableLinkbaseFile, "w", encoding="utf-8")
XmlUtil.writexml(fh, xmlDocument, encoding="utf-8")
fh.close()
dts.info(
"info:updateTableLinkbase",
_("Updated Table Linkbase of %(entryFile)s has %(numberOfLinkroles)s linkroles, %(numberOfTables)s tables in file %(tableLBOutputFile)s."
),
modelObject=dts,
entryFile=dts.uri,
numberOfLinkroles=numELRs,
numberOfTables=numTables,
tableLBOutputFile=updatedTableLinkbaseFile)
def generateHtmlEbaTablesetFiles(dts, indexFile, lang="en"):
try:
import os, io
from arelle import Version, XbrlConst, XmlUtil
from arelle.ViewFileRenderedGrid import viewRenderedGrid
from arelle.ModelRenderingObject import ModelEuTable, ModelTable
numTableFiles = 0
file = io.StringIO('''
<html xmlns="http://www.w3.org/1999/xhtml">
<head id="Left">
<link type="text/css" rel="stylesheet" href="http://arelle.org/files/EBA/style20121210/eba.css" />
</head>
<body class="LTR IE7 ENGB">
<ul class="CMSListMenuUL" id="Vertical2"/>
</body>
</html>
'''
)
from arelle.ModelObjectFactory import parser
parser, parserLookupName, parserLookupClass = parser(dts,None)
from lxml import etree
indexDocument = etree.parse(file,parser=parser,base_url=indexFile)
file.close()
#xmlDocument.getroot().init(self) ## is this needed ??
for listElt in indexDocument.iter(tag="{http://www.w3.org/1999/xhtml}ul"):
break
class nonTkBooleanVar():
def __init__(self, value=True):
self.value = value
def set(self, value):
self.value = value
def get(self):
return self.value
class View():
def __init__(self, tableOrELR, ignoreDimValidity, xAxisChildrenFirst, yAxisChildrenFirst):
self.tblELR = tableOrELR
# context menu boolean vars (non-tkinter boolean
self.ignoreDimValidity = nonTkBooleanVar(value=ignoreDimValidity)
self.xAxisChildrenFirst = nonTkBooleanVar(value=xAxisChildrenFirst)
self.yAxisChildrenFirst = nonTkBooleanVar(value=yAxisChildrenFirst)
indexBase = indexFile.rpartition(".")[0]
groupTableRels = dts.modelXbrl.relationshipSet(XbrlConst.euGroupTable)
modelTables = []
tblCssExtras='''
body {background-image:url('http://arelle.org/files/EBA/style20121210/lhsbackground.jpg')}
table {background:#fff}
'''
# order number is missing
def viewTable(modelTable):
if isinstance(modelTable, (ModelEuTable, ModelTable)):
# status
dts.modelManager.cntlr.addToLog("viewing: " + modelTable.id)
# for table file name, use table ELR
tblFile = os.path.join(os.path.dirname(indexFile), modelTable.id + ".html")
viewRenderedGrid(dts,
tblFile,
lang=lang,
sourceView=View(modelTable, False, False, True),
cssExtras=tblCssExtras)
# generaate menu entry
elt = etree.SubElement(listElt, "{http://www.w3.org/1999/xhtml}li")
elt.set("class", "CMSListMenuLI")
elt.set("id", modelTable.id)
elt = etree.SubElement(elt, "{http://www.w3.org/1999/xhtml}a")
elt.text = modelTable.genLabel(lang=lang, strip=True)
elt.set("class", "CMSListMenuLink")
elt.set("href", "javascript:void(0)")
elt.set("onClick", "javascript:parent.body.location.href='{0}';".format(modelTable.id + ".html"))
elt.text = modelTable.genLabel(lang=lang, strip=True)
else: # just a header
# generaate menu entry
elt = etree.SubElement(listElt, "{http://www.w3.org/1999/xhtml}li")
elt.set("class", "CMSListMenuLink")
elt.set("id", modelTable.id)
elt.text = modelTable.label(lang=lang, strip=True)
for rel in groupTableRels.fromModelObject(modelTable):
viewTable(rel.toModelObject)
for rootConcept in groupTableRels.rootConcepts:
sourceline = 0
for rel in dts.modelXbrl.relationshipSet(XbrlConst.euGroupTable).fromModelObject(rootConcept):
sourceline = rel.sourceline
break
modelTables.append((rootConcept, sourceline))
for modelTable, order in sorted(modelTables, key=lambda x: x[1]):
viewTable(modelTable)
with open(indexBase + "FormsFrame.html", "wt", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocument, encoding="utf-8")
with open(indexFile, "wt", encoding="utf-8") as fh:
fh.write(
'''
<html xmlns="http://www.w3.org/1999/xhtml">
<head id="Head1">
<title>European Banking Authority - EBA - FINREP Taxonomy</title>
<meta name="generator" content="Arelle(r) {0}" />
<meta name="provider" content="Aguilonius(r)" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" />
<meta http-equiv="pragma" content="no-cache" />
<meta http-equiv="content-style-type" content="text/css" />
<meta http-equiv="content-script-type" content="text/javascript" />
<link type="text/css" rel="stylesheet" href="http://arelle.org/files/EBA/style20121210/eba.css" />
</head>
<frameset border="0" frameborder="0" rows="90,*">
<frame name="head" src="{1}" scrolling="no" marginwidth="0" marginheight="10"/>
<frameset bordercolor="#0000cc" border="10" frameborder="no" framespacing="0" cols="360, *">
<frame src="{2}" name="menu" bordercolor="#0000cc"/>
<frame src="{3}" name="body" bordercolor="#0000cc"/>
</frameset>
</frameset>
'''.format(Version.version,
os.path.basename(indexBase) + "TopFrame.html",
os.path.basename(indexBase) + "FormsFrame.html",
os.path.basename(indexBase) + "CenterLanding.html",
))
with open(indexBase + "TopFrame.html", "wt", encoding="utf-8") as fh:
fh.write(
'''
<html xmlns="http://www.w3.org/1999/xhtml">
<head id="Top">
<link type="text/css" rel="stylesheet" href="http://arelle.org/files/EBA/style20121210/eba.css" />
</head>
<body class="LTR IE7 ENGB">
<div id="topsection">
<div id="topsectionLeft" style="cursor:pointer;" onclick="location.href='http://www.eba.europa.eu/home.aspx';"></div>
<div id="topsectionRight"></div>
<div id="topnavigation">
<ul id="menuElem">
<li><a href="http://www.eba.europa.eu/topnav/Contacts.aspx">Contacts</a></li>
<li><a href="http://www.eba.europa.eu/topnav/Links.aspx">Links</a></li>
<li><a href="http://www.eba.europa.eu/topnav/Sitemap.aspx">Sitemap</a></li>
<li><a href="http://www.eba.europa.eu/topnav/Legal-Notice.aspx">Legal Notice</a></li>
</ul>
</div>
</body>
</html>
''')
with open(indexBase + "CenterLanding.html", "wt", encoding="utf-8") as fh:
fh.write(
'''
<html xmlns="http://www.w3.org/1999/xhtml">
<head id="Center">
<link type="text/css" rel="stylesheet" href="http://http://arelle.org/files/EBA/style20121210/eba.css" />
</head>
<body class="LTR IE7 ENGB">
<div id="plc_lt_zoneContent_usercontrol_userControlElem_ContentPanel">
<div id="plc_lt_zoneContent_usercontrol_userControlElem_PanelTitle">
<div id="pagetitle" style="float:left;width:500px;">
<h1>Taxonomy Tables Viewer</h1>
</div>
</div>
</div>
<div style="clear:both;"></div>
<div id="contentcenter">
<p style="text-align: justify; margin-top: 0pt; margin-bottom: 0pt">Please select tables to view by clicking in the left column.</p>
</div>
</body>
</html>
''')
# to merge gif's and style sheets, use a zipfile sibling of the python plug-in file.
#import zipfile
#zf = zipfile.ZipFile(__file__.rpartition('.')[0] + "Files.zip", mode="r")
#zf.extractall(path=os.path.dirname(indexBase))
#zf.close()
dts.info("info:saveEBAtables",
_("Tables index file of %(entryFile)s has %(numberTableFiles)s table files with index file %(indexFile)s."),
modelObject=dts,
entryFile=dts.uri, numberTableFiles=numTableFiles, indexFile=indexFile)
dts.modelManager.showStatus(_("Saved EBA HTML Table Files"), 5000)
except Exception as ex:
dts.error("exception",
_("HTML EBA Tableset files generation exception: %(error)s"), error=ex,
modelXbrl=dts,
exc_info=True)
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath):
# check if big instance and has header with an initial incomplete tree walk (just 2 elements
def logSyntaxErrors(parsercontext):
for error in parsercontext.error_log:
modelXbrl.error("xmlSchema:syntax",
_("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"),
modelObject=modelDocument, fileName=os.path.basename(filepath),
error=error.message, line=error.line, column=error.column, sourceAction="streaming")
#### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
#### note: iterparse wants a binary file, but file is text mode
_file, = modelXbrl.fileSource.file(filepath, binary=True)
startedAt = time.time()
modelXbrl.profileActivity()
parsercontext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
foundInstance = False
foundErrors = False
streamingAspects = None
numRootFacts1 = 0
numElts = 0
elt = None
for event, elt in parsercontext:
if event == "start":
if elt.getparent() is not None:
if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
if not foundInstance:
foundInstance = True
pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance")
if pi is None:
break
else:
streamingAspects = dict(pi.attrib.copy())
if not elt.tag.startswith("{http://www.xbrl.org/"):
numRootFacts1 += 1
if numRootFacts1 % 1000 == 0:
modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
elif not foundInstance:
break
elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl" and precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None:
modelXbrl.error("streamingExtensions:headerMisplaced",
_("Header is misplaced: %(error)s, must follow xbrli:xbrl element"),
modelObject=elt)
elif event == "end":
elt.clear()
numElts += 1
if numElts % 1000 == 0 and elt.getparent() is not None:
while elt.getprevious() is not None and elt.getparent() is not None:
del elt.getparent()[0]
if elt is not None:
elt.clear()
_file.seek(0,io.SEEK_SET) # allow reparsing
if not foundInstance or streamingAspects is None:
del elt, parsercontext
_file.close()
return None
modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
startedAt = time.time()
try:
version = Decimal(streamingAspects.get("version"))
if int(version) != 1:
modelXbrl.error("streamingExtensions:unsupportedVersion",
_("Streaming version %(version)s, major version number must be 1"),
modelObject=elt, version=version)
foundErrors = True
except (InvalidOperation, OverflowError):
modelXbrl.error("streamingExtensions:versionError",
_("Version %(version)s, number must be 1.n"),
modelObject=elt, version=streamingAspects.get("version", "(none)"))
foundErrors = True
for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
try:
bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
raise InvalidOperation
elif bufAspect == "contextBuffer":
contextBufferLimit = bufLimit
elif bufAspect == "unitBuffer":
unitBufferLimit = bufLimit
elif bufAspect == "footnoteBuffer":
footnoteBufferLimit = bufLimit
except InvalidOperation:
modelXbrl.error("streamingExtensions:valueError",
_("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"),
modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect))
foundErrors = True
if parsercontext.error_log:
foundErrors = True
logSyntaxErrors(parsercontext)
if foundErrors:
_file.close()
return None
parsercontext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
_parser, _parserLookupName, _parserLookupClass = parser(modelXbrl,filepath)
eltMdlObjs = {}
beforeInstanceStream = True
validator = None
contextBuffer = []
unitBuffer = []
footnoteBuffer = []
factBuffer = []
numFacts = numRootFacts2 = 1
for event, elt in parsercontext:
if event == "start":
mdlObj = _parser.makeelement(elt.tag, attrib=elt.attrib, nsmap=elt.nsmap)
mdlObj.sourceline = elt.sourceline
eltMdlObjs[elt] = mdlObj
if elt.getparent() is None:
modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, etree.ElementTree(mdlObj))
modelDocument.xmlRootElement = mdlObj
modelXbrl.modelDocument = modelDocument # needed for incremental validation
mdlObj.init(modelDocument)
modelXbrl.info("streamingExtensions:streaming",
_("Stream processing this instance."),
modelObject = modelDocument)
else:
eltMdlObjs[elt.getparent()].append(mdlObj)
mdlObj._init()
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if (beforeInstanceStream and (
(ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
(ns == XbrlConst.xbrli and ln in ("context", "unit")) or
(ns not in (XbrlConst.link, XbrlConst.xbrli)))):
beforeInstanceStream = False
if _streamingExtensionsValidate:
validator = Validate(modelXbrl)
validator.instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters())
else: # need default dimensions
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
mdlObj = None # deref
elif event == "end":
mdlObj = eltMdlObjs.pop(elt)
if elt.text: # text available after child nodes processed
mdlObj.text = elt.text
ns = mdlObj.namespaceURI
ln = mdlObj.localName
parentMdlObj = mdlObj.getparent()
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
modelDocument.contextDiscover(mdlObj)
else:
if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj,)
validator.instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
validator.instValidator.checkContextsDimensions(contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
# drop before additing as dropped may have same id as added
unit = unitBuffer.pop(0)
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
unit = None
modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
validator.instValidator.checkUnits( (mdlObj,) )
elif ln == "xbrl": # end of document
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
elt.clear()
elif ns == XbrlConst.link:
if ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(mdlObj)
elif ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
elif ln == "footnoteLink":
footnoteLinks = (mdlObj,)
modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
validator.instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elt.clear()
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
numRootFacts2 += 1
modelDocument.factDiscover(mdlObj, modelXbrl.facts)
XmlValidate.validate(modelXbrl, mdlObj)
if _streamingExtensionsValidate:
factsToCheck = (mdlObj,)
validator.instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
validator.instValidator.checkFactsDimensions(factsToCheck)
del factsToCheck
dropFact(modelXbrl, mdlObj, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(mdlObj)]
if numRootFacts2 % 1000 == 0:
modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(numRootFacts2, numRootFacts1, 100.0 * numRootFacts2 / numRootFacts1),
minTimeToShow=20.0)
# get rid of root element from iterparse's tree
elt.clear()
while elt.getprevious() is not None: # cleans up any prior siblings
del elt.getparent()[0]
mdlObj = None # deref
logSyntaxErrors(parsercontext)
del parsercontext
if validator is not None:
validator.close()
_file.close()
modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
return modelDocument
def create(modelXbrl, type, uri, schemaRefs=None, isEntry=False):
normalizedUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
uri, None)
if isEntry:
modelXbrl.uri = normalizedUri
modelXbrl.entryLoadingUrl = normalizedUri
modelXbrl.uriDir = os.path.dirname(normalizedUri)
for i in range(modelXbrl.modelManager.disclosureSystem.
maxSubmissionSubdirectoryEntryNesting):
modelXbrl.uriDir = os.path.dirname(modelXbrl.uriDir)
filepath = modelXbrl.modelManager.cntlr.webCache.getfilename(normalizedUri)
# XML document has nsmap root element to replace nsmap as new xmlns entries are required
if type == Type.INSTANCE:
# modelXbrl.uriDir = os.path.dirname(normalizedUri)
Xml = ('<nsmap>'
'<xbrl xmlns="http://www.xbrl.org/2003/instance"'
' xmlns:link="http://www.xbrl.org/2003/linkbase"'
' xmlns:xlink="http://www.w3.org/1999/xlink">')
if schemaRefs:
for schemaRef in schemaRefs:
Xml += '<link:schemaRef xlink:type="simple" xlink:href="{0}"/>'.format(
schemaRef.replace("\\", "/"))
Xml += '</xbrl></nsmap>'
elif type == Type.SCHEMA:
Xml = (
'<nsmap><schema xmlns="http://www.w3.org/2001/XMLSchema" /></nsmap>'
)
elif type == Type.RSSFEED:
Xml = '<nsmap><rss version="2.0" /></nsmap>'
elif type == Type.DTSENTRIES:
Xml = None
else:
type = Type.UnknownXML
Xml = '<nsmap/>'
if Xml:
import io
file = io.StringIO(Xml)
_parser, _parserLookupName, _parserLookupClass = parser(
modelXbrl, filepath)
xmlDocument = etree.parse(file, parser=_parser, base_url=filepath)
file.close()
else:
xmlDocument = None
if type == Type.RSSFEED:
from arelle.ModelRssObject import ModelRssObject
modelDocument = ModelRssObject(modelXbrl, type, uri, filepath,
xmlDocument)
else:
modelDocument = ModelDocument(modelXbrl, type, normalizedUri, filepath,
xmlDocument)
if Xml:
modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement
modelDocument.parserLookupName = _parserLookupName
modelDocument.parserLookupClass = _parserLookupClass
modelDocument.documentEncoding = "utf-8"
rootNode = xmlDocument.getroot()
rootNode.init(modelDocument)
if xmlDocument:
for semanticRoot in rootNode.iterchildren():
if isinstance(semanticRoot, ModelObject):
modelDocument.xmlRootElement = semanticRoot
break
if type == Type.INSTANCE:
modelDocument.instanceDiscover(modelDocument.xmlRootElement)
elif type == Type.RSSFEED:
modelDocument.rssFeedDiscover(modelDocument.xmlRootElement)
elif type == Type.SCHEMA:
modelDocument.targetNamespace = None
modelDocument.isQualifiedElementFormDefault = False
modelDocument.isQualifiedAttributeFormDefault = False
modelDocument.definesUTR = False
return modelDocument
def load(modelXbrl,
uri,
base=None,
referringElement=None,
isEntry=False,
isDiscovered=False,
isIncluded=None,
namespace=None,
reloadCache=False):
if referringElement is None: # used for error messages
referringElement = modelXbrl
normalizedUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
uri, base)
if isEntry:
modelXbrl.entryLoadingUrl = normalizedUri # for error loggiong during loading
modelXbrl.uri = normalizedUri
modelXbrl.uriDir = os.path.dirname(normalizedUri)
for i in range(modelXbrl.modelManager.disclosureSystem.
maxSubmissionSubdirectoryEntryNesting):
modelXbrl.uriDir = os.path.dirname(modelXbrl.uriDir)
if modelXbrl.modelManager.validateDisclosureSystem and \
not normalizedUri.startswith(modelXbrl.uriDir) and \
not modelXbrl.modelManager.disclosureSystem.hrefValid(normalizedUri):
blocked = modelXbrl.modelManager.disclosureSystem.blockDisallowedReferences
modelXbrl.error(
("EFM.6.22.02", "GFM.1.1.3", "SBR.NL.2.1.0.06"
if normalizedUri.startswith("http") else "SBR.NL.2.2.0.17"),
_("Prohibited file for filings %(blockedIndicator)s: %(url)s"),
modelObject=referringElement,
url=normalizedUri,
blockedIndicator=_(" blocked") if blocked else "")
if blocked:
return None
if normalizedUri in modelXbrl.modelManager.disclosureSystem.mappedFiles:
mappedUri = modelXbrl.modelManager.disclosureSystem.mappedFiles[
normalizedUri]
else: # handle mapped paths
mappedUri = normalizedUri
for mapFrom, mapTo in modelXbrl.modelManager.disclosureSystem.mappedPaths:
if normalizedUri.startswith(mapFrom):
mappedUri = mapTo + normalizedUri[len(mapFrom):]
break
if isEntry:
modelXbrl.entryLoadingUrl = mappedUri # for error loggiong during loading
if modelXbrl.fileSource.isInArchive(mappedUri):
filepath = mappedUri
else:
filepath = modelXbrl.modelManager.cntlr.webCache.getfilename(
mappedUri, reload=reloadCache)
if filepath:
uri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(filepath)
if filepath is None: # error such as HTTPerror is already logged
modelXbrl.error("FileNotLoadable",
_("File can not be loaded: %(fileName)s"),
modelObject=referringElement,
fileName=mappedUri)
return None
modelDocument = modelXbrl.urlDocs.get(mappedUri)
if modelDocument:
return modelDocument
# load XML and determine type of model document
modelXbrl.modelManager.showStatus(_("parsing {0}").format(uri))
file = None
try:
if (modelXbrl.modelManager.validateDisclosureSystem
and modelXbrl.modelManager.disclosureSystem.validateFileText):
file, _encoding = ValidateFilingText.checkfile(modelXbrl, filepath)
else:
file, _encoding = modelXbrl.fileSource.file(filepath)
_parser, _parserLookupName, _parserLookupClass = parser(
modelXbrl, filepath)
xmlDocument = etree.parse(file, parser=_parser, base_url=filepath)
file.close()
except (EnvironmentError,
KeyError) as err: # missing zip file raises KeyError
if file:
file.close()
# retry in case of well known schema locations
if not isIncluded and namespace and namespace in XbrlConst.standardNamespaceSchemaLocations and uri != XbrlConst.standardNamespaceSchemaLocations[
namespace]:
return load(modelXbrl,
XbrlConst.standardNamespaceSchemaLocations[namespace],
base, referringElement, isEntry, isDiscovered,
isIncluded, namespace, reloadCache)
modelXbrl.error("IOerror",
_("%(fileName)s: file error: %(error)s"),
modelObject=referringElement,
fileName=os.path.basename(uri),
error=str(err))
return None
except (
etree.LxmlError, ValueError
) as err: # ValueError raised on bad format of qnames, xmlns'es, or parameters
if file:
file.close()
if not isEntry and str(
err) == "Start tag expected, '<' not found, line 1, column 1":
return ModelDocument(modelXbrl, Type.UnknownNonXML, mappedUri,
filepath, None)
else:
modelXbrl.error(
"xmlSchema:syntax",
_("%(error)s, %(fileName)s, %(sourceAction)s source element"),
modelObject=referringElement,
fileName=os.path.basename(uri),
error=str(err),
sourceAction=("including" if isIncluded else "importing"))
return None
# identify document
#modelXbrl.modelManager.addToLog("discovery: {0}".format(
# os.path.basename(uri)))
modelXbrl.modelManager.showStatus(_("loading {0}").format(uri))
modelDocument = None
rootNode = xmlDocument.getroot()
if rootNode is not None:
ln = rootNode.localName
ns = rootNode.namespaceURI
# type classification
if ns == XbrlConst.xsd and ln == "schema":
type = Type.SCHEMA
elif ns == XbrlConst.link:
if ln == "linkbase":
type = Type.LINKBASE
elif ln == "xbrl":
type = Type.INSTANCE
elif ns == XbrlConst.xbrli:
if ln == "xbrl":
type = Type.INSTANCE
elif ns == XbrlConst.xhtml and \
(ln == "html" or ln == "xhtml"):
type = Type.UnknownXML
if XbrlConst.ixbrl in rootNode.nsmap.values():
type = Type.INLINEXBRL
elif ln == "report" and ns == XbrlConst.ver:
type = Type.VERSIONINGREPORT
elif ln == "testcases" or ln == "documentation":
type = Type.TESTCASESINDEX
elif ln == "testcase":
type = Type.TESTCASE
elif ln == "registry" and ns == XbrlConst.registry:
type = Type.REGISTRY
elif ln == "rss":
type = Type.RSSFEED
elif ln == "ptvl":
type = Type.ARCSINFOSET
elif ln == "facts":
type = Type.FACTDIMSINFOSET
else:
type = Type.UnknownXML
nestedInline = None
for htmlElt in rootNode.iter(
tag="{http://www.w3.org/1999/xhtml}html"):
nestedInline = htmlElt
break
if nestedInline is None:
for htmlElt in rootNode.iter(
tag="{http://www.w3.org/1999/xhtml}xhtml"):
nestedInline = htmlElt
break
if nestedInline is not None:
if XbrlConst.ixbrl in nestedInline.nsmap.values():
type = Type.INLINEXBRL
rootNode = nestedInline
#create modelDocument object or subtype as identified
if type == Type.VERSIONINGREPORT:
from arelle.ModelVersReport import ModelVersReport
modelDocument = ModelVersReport(modelXbrl, type, mappedUri,
filepath, xmlDocument)
elif type == Type.RSSFEED:
from arelle.ModelRssObject import ModelRssObject
modelDocument = ModelRssObject(modelXbrl, type, mappedUri,
filepath, xmlDocument)
else:
modelDocument = ModelDocument(modelXbrl, type, mappedUri, filepath,
xmlDocument)
rootNode.init(modelDocument)
modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement
modelDocument.parserLookupName = _parserLookupName
modelDocument.parserLookupClass = _parserLookupClass
modelDocument.xmlRootElement = rootNode
modelDocument.schemaLocationElements.add(rootNode)
modelDocument.documentEncoding = _encoding
if isEntry or isDiscovered:
modelDocument.inDTS = True
# discovery (parsing)
if type == Type.SCHEMA:
modelDocument.schemaDiscover(rootNode, isIncluded, namespace)
elif type == Type.LINKBASE:
modelDocument.linkbaseDiscover(rootNode)
elif type == Type.INSTANCE:
modelDocument.instanceDiscover(rootNode)
elif type == Type.INLINEXBRL:
modelDocument.inlineXbrlDiscover(rootNode)
elif type == Type.VERSIONINGREPORT:
modelDocument.versioningReportDiscover(rootNode)
elif type == Type.TESTCASESINDEX:
modelDocument.testcasesIndexDiscover(xmlDocument)
elif type == Type.TESTCASE:
modelDocument.testcaseDiscover(rootNode)
elif type == Type.REGISTRY:
modelDocument.registryDiscover(rootNode)
elif type == Type.VERSIONINGREPORT:
modelDocument.versioningReportDiscover(rootNode)
elif type == Type.RSSFEED:
modelDocument.rssFeedDiscover(rootNode)
return modelDocument
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, **kwargs):
# check if big instance and has header with an initial incomplete tree walk (just 2 elements
if not _streamingExtensionsCheck:
return None
# track whether modelXbrl has been validated by this streaming extension
modelXbrl._streamingExtensionValidated = False
def logSyntaxErrors(parsercontext):
for error in parsercontext.error_log:
modelXbrl.error("xmlSchema:syntax",
_("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"),
modelObject=modelXbrl, fileName=os.path.basename(filepath),
error=error.message, line=error.line, column=error.column, sourceAction="streaming")
#### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
#### note: iterparse wants a binary file, but file is text mode
_file, = modelXbrl.fileSource.file(filepath, binary=True)
startedAt = time.time()
modelXbrl.profileActivity()
''' this seems twice as slow as iterparse
class instInfoTarget():
def __init__(self, element_factory=None, parser=None):
self.newTree = True
self.streamingAspects = None
self.foundInstance = False
self.creationSoftwareComment = ''
self.currentEltTag = "(before xbrli:xbrl)"
self.numRootFacts = 0
def start(self, tag, attrib, nsmap=None):
if self.newTree:
if tag == "{http://www.xbrl.org/2003/instance}xbrl":
self.foundInstance = True
self.newTree = False
else: # break
raise NotInstanceDocumentException()
elif not tag.startswith("{http://www.xbrl.org/"):
self.numRootFacts += 1
if self.numRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
self.currentEltTag = tag
def end(self, tag):
pass
def data(self, data):
pass
def comment(self, text):
if not self.foundInstance: # accumulate comments before xbrli:xbrl
self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text
elif not self.creationSoftwareComment:
self.creationSoftwareComment = text # or first comment after xbrli:xbrl
def pi(self, target, data):
if target == "xbrl-streamable-instance":
if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl":
self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results
else:
modelXbrl.error("streamingExtensions:headerMisplaced",
_("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"),
modelObject=modelXbrl, target=target, element=self.currentEltTag)
def close(self):
if not self.creationSoftwareComment:
self.creationSoftwareComment = None
return True
instInfo = instInfoTarget()
infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo)
try:
etree.parse(_file, parser=infoParser, base_url=filepath)
except NotInstanceDocumentException:
pass
'''
foundErrors = False
foundInstance = False
streamingAspects = None
creationSoftwareComment = None
instInfoNumRootFacts = 0
numElts = 0
elt = None
instInfoContext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
for event, elt in instInfoContext:
if event == "start":
if elt.getparent() is not None:
if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
if not foundInstance:
foundInstance = True
pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance")
if pi is None:
break
else:
streamingAspects = dict(pi.attrib.copy())
if creationSoftwareComment is None:
creationSoftwareComment = precedingComment(elt)
if not elt.tag.startswith("{http://www.xbrl.org/"):
instInfoNumRootFacts += 1
if instInfoNumRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
elif not foundInstance:
break
elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl":
creationSoftwareComment = precedingComment(elt)
if precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None:
modelXbrl.error("streamingExtensions:headerMisplaced",
_("Header is misplaced: %(error)s, must follow xbrli:xbrl element"),
modelObject=elt)
elif event == "end":
elt.clear()
numElts += 1
if numElts % 1000 == 0 and elt.getparent() is not None:
while elt.getprevious() is not None and elt.getparent() is not None:
del elt.getparent()[0]
if elt is not None:
elt.clear()
_file.seek(0,io.SEEK_SET) # allow reparsing
if not foundInstance or streamingAspects is None:
del elt
_file.close()
return None
modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
startedAt = time.time()
try:
version = Decimal(streamingAspects.get("version"))
if int(version) != 1:
modelXbrl.error("streamingExtensions:unsupportedVersion",
_("Streaming version %(version)s, major version number must be 1"),
modelObject=elt, version=version)
foundErrors = True
except (InvalidOperation, OverflowError):
modelXbrl.error("streamingExtensions:versionError",
_("Version %(version)s, number must be 1.n"),
modelObject=elt, version=streamingAspects.get("version", "(none)"))
foundErrors = True
for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
try:
bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
raise InvalidOperation
elif bufAspect == "contextBuffer":
contextBufferLimit = bufLimit
elif bufAspect == "unitBuffer":
unitBufferLimit = bufLimit
elif bufAspect == "footnoteBuffer":
footnoteBufferLimit = bufLimit
except InvalidOperation:
modelXbrl.error("streamingExtensions:valueError",
_("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"),
modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect))
foundErrors = True
if _streamingExtensionsValidate:
incompatibleValidations = []
_validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
_disclosureSystem = modelXbrl.modelManager.disclosureSystem
if _validateDisclosureSystem and _disclosureSystem.EFM:
incompatibleValidations.append("EFM")
if _validateDisclosureSystem and _disclosureSystem.GFM:
incompatibleValidations.append("GFM")
if _validateDisclosureSystem and _disclosureSystem.EBA:
incompatibleValidations.append("EBA")
if _validateDisclosureSystem and _disclosureSystem.HMRC:
incompatibleValidations.append("EBA")
if modelXbrl.modelManager.validateCalcLB:
incompatibleValidations.append("calculation LB")
if incompatibleValidations:
modelXbrl.error("streamingExtensions:incompatibleValidation",
_("Streaming instance validation does not support %(incompatibleValidations)s validation"),
modelObject=modelXbrl, incompatibleValidations=', '.join(incompatibleValidations))
foundErrors = True
if instInfoContext.error_log:
foundErrors = True
logSyntaxErrors(instInfoContext)
del instInfoContext # dereference
if foundErrors:
_file.close()
return None
_encoding = XmlUtil.encoding(_file.read(512))
_file.seek(0,io.SEEK_SET) # allow reparsing
if _streamingExtensionsValidate:
validator = Validate(modelXbrl)
instValidator = validator.instValidator
eltMdlObjs = {}
contextBuffer = []
unitBuffer = []
footnoteBuffer = []
factBuffer = []
numFacts = 1
class modelLoaderTarget():
def __init__(self, element_factory=None, parser=None):
self.newTree = True
self.currentMdlObj = None
self.beforeInstanceStream = True
self.numRootFacts = 1
def start(self, tag, attrib, nsmap=None):
mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
mdlObj.sourceline = 1
if self.newTree:
self.newTree = False
self.currentMdlObj = mdlObj
modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
modelXbrl.modelDocument = modelDocument # needed for incremental validation
mdlObj.init(modelDocument)
modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement
modelDocument.parserLookupName = _parserLookupName
modelDocument.parserLookupClass = _parserLookupClass
modelDocument.xmlRootElement = mdlObj
modelDocument.schemaLocationElements.add(mdlObj)
modelDocument.documentEncoding = _encoding
modelDocument._creationSoftwareComment = creationSoftwareComment
modelXbrl.info("streamingExtensions:streaming",
_("Stream processing this instance."),
modelObject = modelDocument)
else:
self.currentMdlObj.append(mdlObj)
self.currentMdlObj = mdlObj
mdlObj._init()
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if (self.beforeInstanceStream and (
(ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
(ns == XbrlConst.xbrli and ln in ("context", "unit")) or
(ns not in (XbrlConst.link, XbrlConst.xbrli)))):
self.beforeInstanceStream = False
if _streamingExtensionsValidate:
instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters())
else: # need default dimensions
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
return mdlObj
def end(self, tag):
modelDocument = modelXbrl.modelDocument
mdlObj = self.currentMdlObj
parentMdlObj = mdlObj.getparent()
self.currentMdlObj = parentMdlObj
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
else:
if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj,)
instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
instValidator.checkContextsDimensions(contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
# drop before additing as dropped may have same id as added
unit = unitBuffer.pop(0)
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
unit = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkUnits( (mdlObj,) )
elif ln == "xbrl": # end of document
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
elif ns == XbrlConst.link:
if ln == "footnoteLink":
XmlValidate.validate(modelXbrl, mdlObj)
footnoteLinks = (mdlObj,)
modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elif ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(mdlObj)
elif not modelXbrl.skipDTS:
if ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
self.numRootFacts += 1
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.factDiscover(mdlObj, modelXbrl.facts)
if _streamingExtensionsValidate:
factsToCheck = (mdlObj,)
instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
instValidator.checkFactsDimensions(factsToCheck)
del factsToCheck
dropFact(modelXbrl, mdlObj, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(mdlObj)]
if self.numRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts,
100.0 * self.numRootFacts / instInfoNumRootFacts),
minTimeToShow=20.0)
return mdlObj
def data(self, data):
self.currentMdlObj.text = data
def comment(self, text):
pass
def pi(self, target, data):
pass
def close(self):
return None
_parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget())
etree.parse(_file, parser=_parser, base_url=filepath)
logSyntaxErrors(_parser)
_file.close()
if _streamingExtensionsValidate and validator is not None:
del instValidator
validator.close()
# track that modelXbrl has been validated by this streaming extension
modelXbrl._streamingExtensionValidated = True
modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
return modelXbrl.modelDocument