Example #1
0
def generateCHComponent(dts, componentFile):
    if dts.fileSource.isArchive:
        return
    import os, io
    from arelle import XmlUtil, XbrlConst
    file = io.StringIO('''
<nsmap>
  <Component/>
</nsmap>
''')
    from arelle.ModelObjectFactory import parser
    parser, parserLookupName, parserLookupClass = parser(dts, None)
    xmlDocument = etree.parse(file, parser=parser, base_url=componentFile)
    file.close()
    for componentElt in xmlDocument.iter(tag="Component"):
        break

    # use presentation relationships for broader and narrower concepts
    arcrole = XbrlConst.parentChild
    # sort URIs by definition
    linkroleUris = []
    relationshipSet = dts.relationshipSet(arcrole)
    if relationshipSet:
        for linkroleUri in relationshipSet.linkRoleUris:
            modelRoleTypes = dts.roleTypes.get(linkroleUri)
            if modelRoleTypes:
                roledefinition = (modelRoleTypes[0].genLabel(strip=True)
                                  or modelRoleTypes[0].definition
                                  or linkroleUri)
            else:
                roledefinition = linkroleUri
            linkroleUris.append((roledefinition, linkroleUri))
        linkroleUris.sort()

        # for each URI in definition order
        for roledefinition, linkroleUri in linkroleUris:
            elt = etree.SubElement(componentElt,
                                   "Network",
                                   attrib={
                                       "identifier": linkroleUri,
                                       "label": roledefinition
                                   })
            linkRelationshipSet = dts.relationshipSet(arcrole, linkroleUri)
            for rootConcept in linkRelationshipSet.rootConcepts:
                genConcept(dts, elt, rootConcept, None, arcrole,
                           linkRelationshipSet, set())

    fh = open(componentFile, "w", encoding="utf-8")
    XmlUtil.writexml(fh, xmlDocument, encoding="utf-8")
    fh.close()

    dts.info(
        "info:saveCHComponentFile",
        _("Component file for %(entryFile)s in file %(componentOutputFile)s."),
        modelObject=dts,
        entryFile=dts.uri,
        componentOutputFile=componentFile)
def generateCHComponent(dts, componentFile):
    if dts.fileSource.isArchive:
        return
    import os, io
    from arelle import XmlUtil, XbrlConst
    file = io.StringIO('''
<nsmap>
  <Component/>
</nsmap>
'''
     )
    from arelle.ModelObjectFactory import parser
    parser, parserLookupName, parserLookupClass = parser(dts,None)
    xmlDocument = etree.parse(file,parser=parser,base_url=componentFile)
    file.close()
    for componentElt in  xmlDocument.iter(tag="Component"):
        break
    
    # use presentation relationships for broader and narrower concepts
    arcrole = XbrlConst.parentChild
    # sort URIs by definition
    linkroleUris = []
    relationshipSet = dts.relationshipSet(arcrole)
    if relationshipSet:
        for linkroleUri in relationshipSet.linkRoleUris:
            modelRoleTypes = dts.roleTypes.get(linkroleUri)
            if modelRoleTypes:
                roledefinition = (modelRoleTypes[0].genLabel(strip=True) or modelRoleTypes[0].definition or linkroleUri)                    
            else:
                roledefinition = linkroleUri
            linkroleUris.append((roledefinition, linkroleUri))
        linkroleUris.sort()
    
        # for each URI in definition order
        for roledefinition, linkroleUri in linkroleUris:
            elt = etree.SubElement(componentElt, "Network", attrib={
                                        "identifier": linkroleUri,
                                        "label": roledefinition})
            linkRelationshipSet = dts.relationshipSet(arcrole, linkroleUri)
            for rootConcept in linkRelationshipSet.rootConcepts:
                genConcept(dts, elt, rootConcept, None, arcrole, linkRelationshipSet, set())

    fh = open(componentFile, "w", encoding="utf-8")
    XmlUtil.writexml(fh, xmlDocument, encoding="utf-8")
    fh.close()
    
    dts.info("info:saveCHComponentFile",
             _("Component file for %(entryFile)s in file %(componentOutputFile)s."),
             modelObject=dts,
             entryFile=dts.uri, componentOutputFile=componentFile)
Example #3
0
def generateUpdatedTableLB(dts, updatedTableLinkbaseFile):
    import os, io
    from arelle import XmlUtil, XbrlConst
    from arelle.ViewUtil import viewReferences, referenceURI
    from arelle.ModelRenderingObject import ModelEuAxisCoord
    
    if dts.fileSource.isArchive:
        dts.error("genTblLB:outFileIsArchive",
                 _("Updated Table Linkbase file cannot be an archive: %(tableLBOutputFile)s."),
                 modelObject=dts, tableLBOutputFile=updatedTableLinkbaseFile)
        return
    tblAxisRelSet = dts.relationshipSet(XbrlConst.euTableAxis)
    axisMbrRelSet = dts.relationshipSet(XbrlConst.euAxisMember)
    if len(tblAxisRelSet.modelRelationships) == 0:
        dts.error("genTblLB:noInputTables",
                 _("DTS does not contain Eurofiling 2010 tables and axes: %(entryFile)s."),
                 modelObject=dts, entryFile=dts.uri)
        return

    file = io.StringIO('''
<nsmap>
<link:linkbase 
xmlns:label="http://xbrl.org/2008/label"
xmlns:gen="http://xbrl.org/2008/generic"
xmlns:df="http://xbrl.org/2008/filter/dimension"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:reference="http://xbrl.org/2008/reference"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:table="http://xbrl.org/2011/table"
xmlns:formula="http://xbrl.org/2008/formula"
xsi:schemaLocation="             
http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd
http://xbrl.org/2008/generic http://www.xbrl.org/2008/generic-link.xsd
http://xbrl.org/2008/reference http://www.xbrl.org/2008/generic-reference.xsd
http://xbrl.org/2008/label http://www.xbrl.org/2008/generic-label.xsd
http://xbrl.org/2011/table http://www.xbrl.org/2011/table.xsd
http://xbrl.org/2008/filter/dimension http://www.xbrl.org/2008/dimension-filter.xsd">
  <link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/table-filter" xlink:type="simple" 
   xlink:href="http://www.xbrl.org/2011/table.xsd#table-filter"/>
  <link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/table-axis" xlink:type="simple"
   xlink:href="http://www.xbrl.org/2011/table.xsd#table-axis"/>
  <link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/axis-subtree" xlink:type="simple"
   xlink:href="http://www.xbrl.org/2011/table.xsd#axis-subtree"/>
  <link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/axis-filter" xlink:type="simple"
   xlink:href="http://www.xbrl.org/2011/filter-axis.xsd#axis-filter"/>
</link:linkbase>
</nsmap>
<!--  Generated by Arelle(r) http://arelle.org --> 
'''
     )
    from arelle.ModelObjectFactory import parser
    parser, parserLookupName, parserLookupClass = parser(dts,None)
    from lxml import etree
    xmlDocument = etree.parse(file,parser=parser,base_url=updatedTableLinkbaseFile)
    file.close()
    nsmapElt = xmlDocument.getroot()
    #xmlDocument.getroot().init(self)  ## is this needed ??
    for lbElement in  xmlDocument.iter(tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
        break
    
    class DocObj:  # fake ModelDocument for namespaces
        def __init__(self):
            self.xmlRootElement = lbElement
            self.xmlDocument = xmlDocument    
    docObj = DocObj()
    
    numELRs = 0
    numTables = 0
    
    def copyAttrs(fromElt, toElt, attrTags):
        for attr in attrTags:
            if fromElt.get(attr):
                toElt.set(attr, fromElt.get(attr))

    def generateTable(newLinkElt, newTblElt, srcTblElt, tblAxisRelSet, axisMbrRelSet, visited):
        if srcTblElt is not None:
            for rel in tblAxisRelSet.fromModelObject(srcTblElt):
                srcAxisElt = rel.toModelObject
                if isinstance(srcAxisElt, ModelEuAxisCoord):
                    visited.add(srcAxisElt)
                    newAxisElt = etree.SubElement(newLinkElt, "{http://xbrl.org/2011/table}ruleAxis")
                    copyAttrs(srcAxisElt, newAxisElt, ("id", 
                                                       "{http://www.w3.org/1999/xlink}type",
                                                       "{http://www.w3.org/1999/xlink}label"))
                    newAxisElt.set("abstract", "true") # always true on root element
                    newArcElt = etree.SubElement(newLinkElt, "{http://xbrl.org/2011/table}axisArc")
                    copyAttrs(rel, newArcElt, ("id", 
                                               "{http://www.w3.org/1999/xlink}type",
                                               "{http://www.w3.org/1999/xlink}from",
                                               "{http://www.w3.org/1999/xlink}to",
                                               "order"))
                    newArcElt.set("{http://www.w3.org/1999/xlink}arcrole", XbrlConst.tableBreakdown)
                    newArcElt.set("axisDisposition", rel.axisDisposition)
                    generateAxis(newLinkElt, newAxisElt, srcAxisElt, axisMbrRelSet, visited)
                    visited.discard(srcAxisElt)

    def generateAxis(newLinkElt, newAxisParentElt, srcAxisElt, axisMbrRelSet, visited):
        for rel in axisMbrRelSet.fromModelObject(srcAxisElt):
            tgtAxisElt = rel.toModelObject
            if isinstance(tgtAxisElt, ModelEuAxisCoord) and tgtAxisElt not in visited:
                visited.add(tgtAxisElt)
                newAxisElt = etree.SubElement(newLinkElt, "{http://xbrl.org/2011/table}ruleAxis")
                copyAttrs(tgtAxisElt, newAxisElt, ("id", 
                                                   "abstract",
                                                   "{http://www.w3.org/1999/xlink}type",
                                                   "{http://www.w3.org/1999/xlink}label"))
                if tgtAxisElt.primaryItemQname:
                    newRuleElt = etree.SubElement(newAxisElt, "{http://xbrl.org/2008/formula}concept")
                    newQnameElt = etree.SubElement(newRuleElt, "{http://xbrl.org/2008/formula}qname")
                    newQnameElt.text = XmlUtil.addQnameValue(docObj, tgtAxisElt.primaryItemQname)
                for dimQname, memQname in tgtAxisElt.explicitDims:
                    newRuleElt = etree.SubElement(newAxisElt, "{http://xbrl.org/2008/formula}explicitDimension")
                    newRuleElt.set("dimension", XmlUtil.addQnameValue(docObj, dimQname))
                    newMbrElt = etree.SubElement(newRuleElt, "{http://xbrl.org/2008/formula}member")
                    newQnameElt = etree.SubElement(newMbrElt, "{http://xbrl.org/2008/formula}qname")
                    newQnameElt.text = XmlUtil.addQnameValue(docObj, memQname)
                newArcElt = etree.SubElement(newLinkElt, "{http://xbrl.org/2011/table}axisArc")
                copyAttrs(rel, newArcElt, ("id", 
                                           "{http://www.w3.org/1999/xlink}type",
                                           "{http://www.w3.org/1999/xlink}from",
                                           "{http://www.w3.org/1999/xlink}to",
                                           "order"))
                newArcElt.set("{http://www.w3.org/1999/xlink}arcrole", XbrlConst.tableAxisSubtree)
                generateAxis(newLinkElt, newAxisElt, tgtAxisElt, axisMbrRelSet, visited)
                visited.discard(tgtAxisElt)
        
    # sort URIs
    linkroleUris = sorted([linkroleUri
                           for linkroleUri in tblAxisRelSet.linkRoleUris])
    
    firstNewLinkElt = None
    roleRefUris = set()
    for linkroleUri in linkroleUris:
        numELRs += 1
        newLinkElt = etree.SubElement(lbElement, "{http://xbrl.org/2008/generic}link")
        newLinkElt.set("{http://www.w3.org/1999/xlink}type", "extended")
        newLinkElt.set("{http://www.w3.org/1999/xlink}role", linkroleUri)
        if firstNewLinkElt is None: firstNewLinkElt = newLinkElt
        # To do: add roleRef if needed
        tblAxisRelSet = dts.relationshipSet(XbrlConst.euTableAxis, linkroleUri)
        axisMbrRelSet = dts.relationshipSet(XbrlConst.euAxisMember, linkroleUri)
        for srcTblElt in tblAxisRelSet.rootConcepts:
            if srcTblElt.tag == "{http://www.eurofiling.info/2010/rendering}table":
                numTables += 1
                newTblElt = etree.SubElement(newLinkElt, "{http://xbrl.org/2011/table}table")
                newTblElt.set("aspectModel", "dimensional")
                copyAttrs(srcTblElt, newTblElt, ("id", 
                                                 "{http://www.w3.org/1999/xlink}type",
                                                 "{http://www.w3.org/1999/xlink}label"))
                generateTable(newLinkElt, newTblElt, srcTblElt, tblAxisRelSet, axisMbrRelSet, set())
                
                if linkroleUri not in roleRefUris:
                    srcRoleRefElt = XmlUtil.descendant(srcTblElt.getroottree(), XbrlConst.link, "roleRef", "roleURI", linkroleUri)
                    if srcRoleRefElt is not None:
                        roleRefUris.add(linkroleUri)
                        newRoleRefElt = etree.Element("{http://www.xbrl.org/2003/linkbase}roleRef")
                        copyAttrs(srcRoleRefElt, newRoleRefElt, ("roleURI", 
                                                                 "{http://www.w3.org/1999/xlink}type",
                                                                 "{http://www.w3.org/1999/xlink}href"))
                        firstNewLinkElt.addprevious(newRoleRefElt)
            
    fh = open(updatedTableLinkbaseFile, "w", encoding="utf-8")
    XmlUtil.writexml(fh, xmlDocument, encoding="utf-8")
    fh.close()
    
    dts.info("info:updateTableLinkbase",
             _("Updated Table Linkbase of %(entryFile)s has %(numberOfLinkroles)s linkroles, %(numberOfTables)s tables in file %(tableLBOutputFile)s."),
             modelObject=dts,
             entryFile=dts.uri, numberOfLinkroles=numELRs, numberOfTables=numTables, tableLBOutputFile=updatedTableLinkbaseFile)
Example #4
0
def generateSkos(dts, skosFile):
    try: 
        import os, io
        from arelle import XmlUtil, XbrlConst
        from arelle.ViewUtil import viewReferences, referenceURI
        skosNs = "http://www.w3.org/2004/02/skos/core#"
        rdfNs = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
        dts.modelManager.showStatus("initializing SKOS document")
        file = io.StringIO('''
  <!DOCTYPE rdf:RDF> 
  <nsmap>
  <rdf:RDF xmlns="urn:cgi:classifier:CGI:XBRL:201204#" xml:base="urn:cgi:classifierScheme:CGI:XBRL:201204" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xmlns:owl2xml="http://www.w3.org/2006/12/owl2-xml#" xmlns:p1="#" xmlns:owl="http://www.w3.org/2002/07/owl#" xmlns:xsd="http://www.w3.org/2001/XMLSchema#" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:xbrl-201204="urn:cgi:classifier:CGI:XBRL:201204#" xmlns:skos="http://www.w3.org/2004/02/skos/core#">
  <owl:Ontology rdf:about="" /> 
  <!-- Annotation properties --> 
  <owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/date" /> 
  <owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/source" /> 
  <owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/title" /> 
  <owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/description" /> 
  <owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/contributor" /> 
  <owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/creator" /> 
  <owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/format" /> 
  <owl:AnnotationProperty rdf:about="http://www.w3.org/2002/07/owl#versionInfo" /> 
  <!-- Object Properties --> 
  <!--  http://www.w3.org/2004/02/skos/core#broader --> 
  <owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#broader" /> 
  <!--  http://www.w3.org/2004/02/skos/core#changeNote --> 
  <owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#changeNote" /> 
  <!--  http://www.w3.org/2004/02/skos/core#hasTopConcept --> 
  <owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#hasTopConcept" /> 
  <!--  http://www.w3.org/2004/02/skos/core#inScheme  --> 
  <owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#inScheme" /> 
  <!--  http://www.w3.org/2004/02/skos/core#topConceptOf  --> 
  <owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#topConceptOf" /> 
  <!-- Data properties --> 
  <!--  http://www.w3.org/2004/02/skos/core#definition --> 
  <owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#definition" /> 
  <!--  http://www.w3.org/2004/02/skos/core#editorialNote --> 
  <owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#editorialNote" /> 
  <!--  http://www.w3.org/2004/02/skos/core#historyNote --> 
  <owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#historyNote" /> 
  <!--  http://www.w3.org/2004/02/skos/core#notation --> 
  <owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#notation" /> 
  <!--  http://www.w3.org/2004/02/skos/core#prefLabel --> 
  <owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#prefLabel" /> 
  <!-- Classes --> 
  <!--  http://www.w3.org/2002/07/owl#Thing --> 
  <owl:Class rdf:about="http://www.w3.org/2002/07/owl#Thing" /> 
  <!--  http://www.w3.org/2004/02/skos/core#Concept --> 
  <owl:Class rdf:about="http://www.w3.org/2004/02/skos/core#Concept" /> 
  <!--  http://www.w3.org/2004/02/skos/core#ConceptScheme --> 
  <owl:Class rdf:about="http://www.w3.org/2004/02/skos/core#ConceptScheme" /> 
  <!-- Individuals --> 
</rdf:RDF></nsmap>
 <!--  Generated by the Arelle(r) http://arelle.org --> 
'''
         )
        from arelle.ModelObjectFactory import parser
        parser, parserLookupName, parserLookupClass = parser(dts,None)
        from lxml import etree
        xmlDocument = etree.parse(file,parser=parser,base_url=skosFile)
        file.close()
        xmlRootElement = xmlDocument.getroot()
        #xmlDocument.getroot().init(self)  ## is this needed ??
        for rdfElement in  xmlDocument.iter(tag="{http://www.w3.org/1999/02/22-rdf-syntax-ns#}RDF"):
            break
        numSchemes = 0
        numConcepts = 0
        
        # use presentation relationships for broader and narrower concepts
        relationshipSet = dts.relationshipSet(XbrlConst.parentChild)
        
        def conceptUri(concept):
            return concept.qname.namespaceURI + "#" + concept.qname.localName
        
        def namespaceUri(qname):
            return qname.namespaceURI + "#" + qname.prefix
        
        priorSchemeSibling = None
        schemeNamespaces = set()
        
        dts.modelManager.showStatus("setting SKOS concepts from XBRL concepts")
        for qn, concept in sorted(dts.qnameConcepts.items(), key=lambda item:str(item[0])):
            if concept.modelDocument.targetNamespace not in (
                     XbrlConst.xbrli, XbrlConst.link, XbrlConst.xlink, XbrlConst.xl,
                     XbrlConst.xbrldt):
                if qn.namespaceURI not in schemeNamespaces:
                    # add conceptScheme
                    numSchemes += 1
                    skosElt = etree.Element("{http://www.w3.org/2004/02/skos/core#}ConceptScheme")
                    skosElt.set("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}about", namespaceUri(qn))
                    elt = etree.SubElement(skosElt, "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}type")
                    elt.set("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource", 
                            "http://www.w3.org/2002/07/owl#Thing")
                    elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}notation")
                    elt.text = str(qn.prefix)
                    schemeNamespaces.add(qn.namespaceURI)
                    if priorSchemeSibling is not None:
                        priorSchemeSibling.addnext(skosElt)
                    else:
                        rdfElement.append(skosElt)
                    priorSchemeSibling = skosElt
                
                numConcepts += 1
                skosElt = etree.SubElement(rdfElement, "{http://www.w3.org/2004/02/skos/core#}Concept")
                skosElt.set("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}about", conceptUri(concept))
                elt = etree.SubElement(skosElt, "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}type")
                elt.set("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource", 
                        "http://www.w3.org/2002/07/owl#Thing")
                elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}notation")
                elt.text = str(concept.qname)
                definition = concept.label(preferredLabel=XbrlConst.documentationLabel, lang="en", strip=True, fallbackToQname=False)
                if definition:
                    elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}definition")
                    elt.text = definition
                else:   # if no definition, look for any references
                    references = viewReferences(concept)
                    if references:
                        elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}definition")
                        elt.text = references
                    linkedReferenceURI = referenceURI(concept)
                    if linkedReferenceURI:    # link to reference
                        elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}definition")
                        elt.text = linkedReferenceURI
                labelsRelationshipSet = dts.relationshipSet(XbrlConst.conceptLabel)
                if labelsRelationshipSet:
                    for modelLabelRel in labelsRelationshipSet.fromModelObject(concept):
                        label = modelLabelRel.toModelObject
                        if label.role == XbrlConst.standardLabel:
                            elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}prefLabel")
                            elt.set("{http://www.w3.org/XML/1998/namespace}lang", label.xmlLang)
                            elt.text = label.text.strip()
                for rel in relationshipSet.fromModelObject(concept): # narrower
                    elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}narrower")
                    elt.set("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource", conceptUri(rel.toModelObject))
                for rel in relationshipSet.toModelObject(concept): # broader
                    elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}broader")
                    elt.set("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource", conceptUri(rel.fromModelObject))
                    
                elt = etree.SubElement(skosElt, "{http://www.w3.org/2004/02/skos/core#}inScheme")
                elt.set("{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource", 
                        namespaceUri(qn))
                
        dts.modelManager.showStatus("saving SKOS file")
        fh = open(skosFile, "w", encoding="utf-8")
        XmlUtil.writexml(fh, xmlDocument, encoding="utf-8")
        fh.close()
        
        dts.info("info:saveSKOS",
                 _("SKOS of %(entryFile)s has %(numberOfConcepts)s concepts in SKOS RDF file %(skosOutputFile)s."),
                 modelObject=dts,
                 entryFile=dts.uri, numberOfConcepts=numConcepts, skosOutputFile=skosFile)
        dts.modelManager.showStatus("ready", 3000)
    except Exception as ex:
        dts.error("exception",
            _("SKOS generation exception: %(error)s"), error=ex,
            modelXbrl=dts,
            exc_info=True)
Example #5
0
def create(modelXbrl, type, uri, schemaRefs=None, isEntry=False):
    normalizedUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, None)
    if isEntry:
        modelXbrl.uri = normalizedUri
        modelXbrl.entryLoadingUrl = normalizedUri
        modelXbrl.uriDir = os.path.dirname(normalizedUri)
        for i in range(modelXbrl.modelManager.disclosureSystem.maxSubmissionSubdirectoryEntryNesting):
            modelXbrl.uriDir = os.path.dirname(modelXbrl.uriDir)
    filepath = modelXbrl.modelManager.cntlr.webCache.getfilename(normalizedUri)
    # XML document has nsmap root element to replace nsmap as new xmlns entries are required
    if type == Type.INSTANCE:
        # modelXbrl.uriDir = os.path.dirname(normalizedUri)
        Xml = ('<nsmap>'
               '<xbrl xmlns="http://www.xbrl.org/2003/instance"'
               ' xmlns:link="http://www.xbrl.org/2003/linkbase"'
               ' xmlns:xlink="http://www.w3.org/1999/xlink">')
        if schemaRefs:
            for schemaRef in schemaRefs:
                Xml += '<link:schemaRef xlink:type="simple" xlink:href="{0}"/>'.format(schemaRef.replace("\\","/"))
        Xml += '</xbrl></nsmap>'
    elif type == Type.SCHEMA:
        Xml = ('<nsmap><schema xmlns="http://www.w3.org/2001/XMLSchema" /></nsmap>')
    elif type == Type.RSSFEED:
        Xml = '<nsmap><rss version="2.0" /></nsmap>'
    elif type == Type.DTSENTRIES:
        Xml = None
    else:
        type = Type.Unknown
        Xml = '<nsmap/>'
    if Xml:
        import io
        file = io.StringIO(Xml)
        _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl,filepath)
        xmlDocument = etree.parse(file,parser=_parser,base_url=filepath)
        file.close()
    else:
        xmlDocument = None
    if type == Type.RSSFEED:
        from arelle.ModelRssObject import ModelRssObject 
        modelDocument = ModelRssObject(modelXbrl, type, uri, filepath, xmlDocument)
    else:
        modelDocument = ModelDocument(modelXbrl, type, normalizedUri, filepath, xmlDocument)
    if Xml:
        modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement 
        modelDocument.parserLookupName = _parserLookupName
        modelDocument.parserLookupClass = _parserLookupClass
        modelDocument.documentEncoding = "utf-8"
        rootNode = xmlDocument.getroot()
        rootNode.init(modelDocument)
        if xmlDocument:
            for semanticRoot in rootNode.iterchildren():
                if isinstance(semanticRoot, ModelObject):
                    modelDocument.xmlRootElement = semanticRoot
                    break
    if type == Type.INSTANCE:
        modelDocument.instanceDiscover(modelDocument.xmlRootElement)
    elif type == Type.RSSFEED:
        modelDocument.rssFeedDiscover(modelDocument.xmlRootElement)
    elif type == Type.SCHEMA:
        modelDocument.targetNamespace = None
        modelDocument.isQualifiedElementFormDefault = False
        modelDocument.isQualifiedAttributeFormDefault = False
    return modelDocument
Example #6
0
def load(modelXbrl, uri, base=None, referringElement=None, isEntry=False, isDiscovered=False, isIncluded=None, namespace=None, reloadCache=False):
    if referringElement is None: # used for error messages
        referringElement = modelXbrl
    normalizedUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, base)
    if isEntry:
        modelXbrl.entryLoadingUrl = normalizedUri   # for error loggiong during loading
        modelXbrl.uri = normalizedUri
        modelXbrl.uriDir = os.path.dirname(normalizedUri)
        for i in range(modelXbrl.modelManager.disclosureSystem.maxSubmissionSubdirectoryEntryNesting):
            modelXbrl.uriDir = os.path.dirname(modelXbrl.uriDir)
    if modelXbrl.modelManager.validateDisclosureSystem and \
       not normalizedUri.startswith(modelXbrl.uriDir) and \
       not modelXbrl.modelManager.disclosureSystem.hrefValid(normalizedUri):
        blocked = modelXbrl.modelManager.disclosureSystem.blockDisallowedReferences
        modelXbrl.error(("EFM.6.22.02", "GFM.1.1.3", "SBR.NL.2.1.0.06"),
                _("Prohibited file for filings %(blockedIndicator)s: %(url)s"),
                modelObject=referringElement, url=normalizedUri, blockedIndicator=_(" blocked") if blocked else "")
        if blocked:
            return None
    if normalizedUri in modelXbrl.modelManager.disclosureSystem.mappedFiles:
        mappedUri = modelXbrl.modelManager.disclosureSystem.mappedFiles[normalizedUri]
    else:  # handle mapped paths
        mappedUri = normalizedUri
        for mapFrom, mapTo in modelXbrl.modelManager.disclosureSystem.mappedPaths:
            if normalizedUri.startswith(mapFrom):
                mappedUri = mapTo + normalizedUri[len(mapFrom):]
                break
    if isEntry:
        modelXbrl.entryLoadingUrl = mappedUri   # for error loggiong during loading
    if modelXbrl.fileSource.isInArchive(mappedUri):
        filepath = mappedUri
    else:
        filepath = modelXbrl.modelManager.cntlr.webCache.getfilename(mappedUri, reload=reloadCache)
        if filepath:
            uri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(filepath)
    if filepath is None: # error such as HTTPerror is already logged
        modelXbrl.error("FileNotLoadable",
                _("File can not be loaded: %(fileName)s"),
                modelObject=referringElement, fileName=mappedUri)
        type = Type.Unknown
        return None
    
    modelDocument = modelXbrl.urlDocs.get(mappedUri)
    if modelDocument:
        return modelDocument
    
    # load XML and determine type of model document
    modelXbrl.modelManager.showStatus(_("parsing {0}").format(uri))
    file = None
    try:
        if (modelXbrl.modelManager.validateDisclosureSystem and 
            modelXbrl.modelManager.disclosureSystem.validateFileText):
            file, _encoding = ValidateFilingText.checkfile(modelXbrl,filepath)
        else:
            file, _encoding = modelXbrl.fileSource.file(filepath)
        _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl,filepath)
        xmlDocument = etree.parse(file,parser=_parser,base_url=filepath)
        file.close()
    except (EnvironmentError, KeyError) as err:  # missing zip file raises KeyError
        modelXbrl.error("IOerror",
                _("%(fileName)s: file error: %(error)s"),
                modelObject=referringElement, fileName=os.path.basename(uri), error=str(err))
        type = Type.Unknown
        if file:
            file.close()
        return None
    except (etree.LxmlError,
            ValueError) as err:  # ValueError raised on bad format of qnames, xmlns'es, or parameters
        modelXbrl.error("xmlSchema:syntax",
                _("%(error)s, %(fileName)s, %(sourceAction)s source element"),
                modelObject=referringElement, fileName=os.path.basename(uri), 
                error=str(err), sourceAction=("including" if isIncluded else "importing"))
        type = Type.Unknown
        if file:
            file.close()
        return None
    
    # identify document
    #modelXbrl.modelManager.addToLog("discovery: {0}".format(
    #            os.path.basename(uri)))
    modelXbrl.modelManager.showStatus(_("loading {0}").format(uri))
    modelDocument = None
    
    rootNode = xmlDocument.getroot()
    if rootNode is not None:
        ln = rootNode.localName
        ns = rootNode.namespaceURI
        
        # type classification
        if ns == XbrlConst.xsd and ln == "schema":
            type = Type.SCHEMA
        elif ns == XbrlConst.link:
            if ln == "linkbase":
                type = Type.LINKBASE
            elif ln == "xbrl":
                type = Type.INSTANCE
        elif ns == XbrlConst.xbrli:
            if ln == "xbrl":
                type = Type.INSTANCE
        elif ns == XbrlConst.xhtml and \
             (ln == "html" or ln == "xhtml"):
            type = Type.Unknown
            if XbrlConst.ixbrl in rootNode.nsmap.values():
                type = Type.INLINEXBRL
        elif ln == "report" and ns == XbrlConst.ver:
            type = Type.VERSIONINGREPORT
        elif ln == "testcases" or ln == "documentation":
            type = Type.TESTCASESINDEX
        elif ln == "testcase":
            type = Type.TESTCASE
        elif ln == "registry" and ns == XbrlConst.registry:
            type = Type.REGISTRY
        elif ln == "rss":
            type = Type.RSSFEED
        else:
            type = Type.Unknown
            nestedInline = None
            for htmlElt in rootNode.iter(tag="{http://www.w3.org/1999/xhtml}html"):
                nestedInline = htmlElt
                break
            if nestedInline is None:
                for htmlElt in rootNode.iter(tag="{http://www.w3.org/1999/xhtml}xhtml"):
                    nestedInline = htmlElt
                    break
            if nestedInline is not None:
                if XbrlConst.ixbrl in nestedInline.nsmap.values():
                    type = Type.INLINEXBRL
                    rootNode = nestedInline

        #create modelDocument object or subtype as identified
        if type == Type.VERSIONINGREPORT:
            from arelle.ModelVersReport import ModelVersReport
            modelDocument = ModelVersReport(modelXbrl, type, mappedUri, filepath, xmlDocument)
        elif type == Type.RSSFEED:
            from arelle.ModelRssObject import ModelRssObject 
            modelDocument = ModelRssObject(modelXbrl, type, mappedUri, filepath, xmlDocument)
        else:
            modelDocument = ModelDocument(modelXbrl, type, mappedUri, filepath, xmlDocument)
        rootNode.init(modelDocument)
        modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement 
        modelDocument.parserLookupName = _parserLookupName
        modelDocument.parserLookupClass = _parserLookupClass
        modelDocument.xmlRootElement = rootNode
        modelDocument.schemaLocationElements.add(rootNode)
        modelDocument.documentEncoding = _encoding

        if isEntry or isDiscovered:
            modelDocument.inDTS = True
        
        # discovery (parsing)
        if type == Type.SCHEMA:
            modelDocument.schemaDiscover(rootNode, isIncluded, namespace)
        elif type == Type.LINKBASE:
            modelDocument.linkbaseDiscover(rootNode)
        elif type == Type.INSTANCE:
            modelDocument.instanceDiscover(rootNode)
        elif type == Type.INLINEXBRL:
            modelDocument.inlineXbrlDiscover(rootNode)
        elif type == Type.VERSIONINGREPORT:
            modelDocument.versioningReportDiscover(rootNode)
        elif type == Type.TESTCASESINDEX:
            modelDocument.testcasesIndexDiscover(xmlDocument)
        elif type == Type.TESTCASE:
            modelDocument.testcaseDiscover(rootNode)
        elif type == Type.REGISTRY:
            modelDocument.registryDiscover(rootNode)
        elif type == Type.VERSIONINGREPORT:
            modelDocument.versioningReportDiscover(rootNode)
        elif type == Type.RSSFEED:
            modelDocument.rssFeedDiscover(rootNode)
    return modelDocument
Example #7
0
def generateHtmlEbaTablesetFiles(dts, indexFile, lang="en"):
    try:
        import os, io
        from arelle import Version, XbrlConst, XmlUtil
        from arelle.ViewFileRenderedGrid import viewRenderedGrid
        from arelle.ModelRenderingObject import ModelEuTable, ModelTable

        numTableFiles = 0

        file = io.StringIO('''
<html xmlns="http://www.w3.org/1999/xhtml">
<head id="Left">
  <link type="text/css" rel="stylesheet" href="http://arelle.org/files/EBA/style20121210/eba.css" /> 
</head>
<body class="LTR IE7 ENGB">
    <ul class="CMSListMenuUL" id="Vertical2"/>
</body>
</html>
''')
        from arelle.ModelObjectFactory import parser
        parser, parserLookupName, parserLookupClass = parser(dts, None)
        from lxml import etree
        indexDocument = etree.parse(file, parser=parser, base_url=indexFile)
        file.close()
        #xmlDocument.getroot().init(self)  ## is this needed ??
        for listElt in indexDocument.iter(
                tag="{http://www.w3.org/1999/xhtml}ul"):
            break

        class nonTkBooleanVar():
            def __init__(self, value=True):
                self.value = value

            def set(self, value):
                self.value = value

            def get(self):
                return self.value

        class View():
            def __init__(self, tableOrELR, ignoreDimValidity,
                         xAxisChildrenFirst, yAxisChildrenFirst):
                self.tblELR = tableOrELR
                # context menu boolean vars (non-tkinter boolean
                self.ignoreDimValidity = nonTkBooleanVar(
                    value=ignoreDimValidity)
                self.xAxisChildrenFirst = nonTkBooleanVar(
                    value=xAxisChildrenFirst)
                self.yAxisChildrenFirst = nonTkBooleanVar(
                    value=yAxisChildrenFirst)

        indexBase = indexFile.rpartition(".")[0]
        groupTableRels = dts.modelXbrl.relationshipSet(XbrlConst.euGroupTable)
        modelTables = []
        tblCssExtras = '''
body {background-image:url('http://arelle.org/files/EBA/style20121210/lhsbackground.jpg')}
table {background:#fff}
'''

        # order number is missing
        def viewTable(modelTable):
            if modelTable is None:
                return
            if isinstance(modelTable, (ModelEuTable, ModelTable)):
                # status
                dts.modelManager.cntlr.addToLog("viewing: " + modelTable.id)
                # for table file name, use table ELR
                tblFile = os.path.join(os.path.dirname(indexFile),
                                       modelTable.id + ".html")
                viewRenderedGrid(dts,
                                 tblFile,
                                 lang=lang,
                                 sourceView=View(modelTable, False, False,
                                                 True),
                                 cssExtras=tblCssExtras)

                # generaate menu entry
                elt = etree.SubElement(listElt,
                                       "{http://www.w3.org/1999/xhtml}li")
                elt.set("class", "CMSListMenuLI")
                elt.set("id", modelTable.id)
                elt = etree.SubElement(elt, "{http://www.w3.org/1999/xhtml}a")
                elt.text = modelTable.genLabel(lang=lang, strip=True)
                elt.set("class", "CMSListMenuLink")
                elt.set("href", "javascript:void(0)")
                elt.set(
                    "onClick",
                    "javascript:parent.body.location.href='{0}';".format(
                        modelTable.id + ".html"))
                elt.text = modelTable.genLabel(lang=lang, strip=True)

            else:  # just a header
                # generaate menu entry
                elt = etree.SubElement(listElt,
                                       "{http://www.w3.org/1999/xhtml}li")
                elt.set("class", "CMSListMenuLink")
                elt.set("id", modelTable.id)
                elt.text = modelTable.label(lang=lang, strip=True)

            for rel in groupTableRels.fromModelObject(modelTable):
                viewTable(rel.toModelObject)

        for rootConcept in groupTableRels.rootConcepts:
            sourceline = 0
            for rel in dts.modelXbrl.relationshipSet(
                    XbrlConst.euGroupTable).fromModelObject(rootConcept):
                sourceline = rel.sourceline
                break
            modelTables.append((rootConcept, sourceline))

        for modelTable, order in sorted(modelTables, key=lambda x: x[1]):
            viewTable(modelTable)

        with open(indexBase + "FormsFrame.html", "wt", encoding="utf-8") as fh:
            XmlUtil.writexml(fh, indexDocument, encoding="utf-8")

        with open(indexFile, "wt", encoding="utf-8") as fh:
            fh.write('''
<html xmlns="http://www.w3.org/1999/xhtml">
<head id="Head1">
  <title>European Banking Authority - EBA  - FINREP Taxonomy</title>
  <meta name="generator" content="Arelle(r) {0}" /> 
  <meta name="provider" content="Aguilonius(r)" />
  <meta http-equiv="content-type" content="text/html; charset=UTF-8" /> 
  <meta http-equiv="pragma" content="no-cache" /> 
  <meta http-equiv="content-style-type" content="text/css" /> 
  <meta http-equiv="content-script-type" content="text/javascript" /> 
  <link type="text/css" rel="stylesheet" href="http://arelle.org/files/EBA/style20121210/eba.css" /> 
</head>
<frameset border="0" frameborder="0" rows="90,*">
   <frame name="head" src="{1}" scrolling="no" marginwidth="0" marginheight="10"/>
   <frameset  bordercolor="#0000cc" border="10" frameborder="no" framespacing="0" cols="360, *">
      <frame src="{2}" name="menu" bordercolor="#0000cc"/>
      <frame src="{3}" name="body" bordercolor="#0000cc"/>
   </frameset>
</frameset>
'''.format(
                Version.version,
                os.path.basename(indexBase) + "TopFrame.html",
                os.path.basename(indexBase) + "FormsFrame.html",
                os.path.basename(indexBase) + "CenterLanding.html",
            ))

        with open(indexBase + "TopFrame.html", "wt", encoding="utf-8") as fh:
            fh.write('''
<html xmlns="http://www.w3.org/1999/xhtml">
<head id="Top">
  <link type="text/css" rel="stylesheet" href="http://arelle.org/files/EBA/style20121210/eba.css" /> 
</head>
  <body class="LTR IE7 ENGB">
   <div id="topsection">
      <div id="topsectionLeft" style="cursor:pointer;" onclick="location.href='http://www.eba.europa.eu/home.aspx';"></div>
      <div id="topsectionRight"></div>
      <div id="topnavigation">
      <ul id="menuElem">
        <li><a href="http://www.eba.europa.eu/topnav/Contacts.aspx">Contacts</a></li>
        <li><a href="http://www.eba.europa.eu/topnav/Links.aspx">Links</a></li>
        <li><a href="http://www.eba.europa.eu/topnav/Sitemap.aspx">Sitemap</a></li>
        <li><a href="http://www.eba.europa.eu/topnav/Legal-Notice.aspx">Legal Notice</a></li>
      </ul>
    </div>
  </body>
</html>
''')

        with open(indexBase + "CenterLanding.html", "wt",
                  encoding="utf-8") as fh:
            fh.write('''
<html xmlns="http://www.w3.org/1999/xhtml">
<head id="Center">
  <link type="text/css" rel="stylesheet" href="http://http://arelle.org/files/EBA/style20121210/eba.css" /> 
</head>
<body class="LTR IE7 ENGB">
  <div id="plc_lt_zoneContent_usercontrol_userControlElem_ContentPanel">
    <div id="plc_lt_zoneContent_usercontrol_userControlElem_PanelTitle">
      <div id="pagetitle" style="float:left;width:500px;">
        <h1>Taxonomy Tables Viewer</h1>
      </div>
    </div>
  </div>
  <div style="clear:both;"></div>
  <div id="contentcenter">
    <p style="text-align: justify; margin-top: 0pt; margin-bottom: 0pt">Please select tables to view by clicking in the left column.</p>
  </div>
</body>
</html>
''')

        # to merge gif's and style sheets, use a zipfile sibling of the python plug-in file.
        #import zipfile
        #zf = zipfile.ZipFile(__file__.rpartition('.')[0] + "Files.zip", mode="r")
        #zf.extractall(path=os.path.dirname(indexBase))
        #zf.close()

        dts.info(
            "info:saveEBAtables",
            _("Tables index file of %(entryFile)s has %(numberTableFiles)s table files with index file %(indexFile)s."
              ),
            modelObject=dts,
            entryFile=dts.uri,
            numberTableFiles=numTableFiles,
            indexFile=indexFile)

        dts.modelManager.showStatus(_("Saved EBA HTML Table Files"), 5000)
    except Exception as ex:
        dts.error("exception",
                  _("HTML EBA Tableset files generation exception: %(error)s"),
                  error=ex,
                  modelXbrl=dts,
                  exc_info=True)
Example #8
0
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, **kwargs):
    # check if big instance and has header with an initial incomplete tree walk (just 2 elements
    if not _streamingExtensionsCheck:
        return None
    
    # track whether modelXbrl has been validated by this streaming extension
    modelXbrl._streamingExtensionValidated = False
        
    def logSyntaxErrors(parsercontext):
        for error in parsercontext.error_log:
            modelXbrl.error("xmlSchema:syntax",
                    _("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"),
                    modelObject=modelXbrl, fileName=os.path.basename(filepath), 
                    error=error.message, line=error.line, column=error.column, sourceAction="streaming")
    #### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
    #### note: iterparse wants a binary file, but file is text mode
    _file, = modelXbrl.fileSource.file(filepath, binary=True)
    startedAt = time.time()
    modelXbrl.profileActivity()
    ''' this seems twice as slow as iterparse
    class instInfoTarget():
        def __init__(self, element_factory=None, parser=None):
            self.newTree = True
            self.streamingAspects = None
            self.foundInstance = False
            self.creationSoftwareComment = ''
            self.currentEltTag = "(before xbrli:xbrl)"
            self.numRootFacts = 0
        def start(self, tag, attrib, nsmap=None):
            if self.newTree:
                if tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    self.foundInstance = True
                    self.newTree = False
                else: # break 
                    raise NotInstanceDocumentException()
            elif not tag.startswith("{http://www.xbrl.org/"):
                self.numRootFacts += 1
                if self.numRootFacts % 1000 == 0:
                    modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
            self.currentEltTag = tag
        def end(self, tag):
            pass
        def data(self, data):
            pass
        def comment(self, text):
            if not self.foundInstance: # accumulate comments before xbrli:xbrl
                self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text
            elif not self.creationSoftwareComment:
                self.creationSoftwareComment = text # or first comment after xbrli:xbrl
        def pi(self, target, data):
            if target == "xbrl-streamable-instance":
                if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl":
                    self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results
                else:
                    modelXbrl.error("streamingExtensions:headerMisplaced",
                            _("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"),
                            modelObject=modelXbrl, target=target, element=self.currentEltTag)
        def close(self):
            if not self.creationSoftwareComment:
                self.creationSoftwareComment = None
            return True
    instInfo = instInfoTarget()
    infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo)
    try:
        etree.parse(_file, parser=infoParser, base_url=filepath)
    except NotInstanceDocumentException:
        pass
    '''
    foundErrors = False
    foundInstance = False
    streamingAspects = None
    creationSoftwareComment = None
    instInfoNumRootFacts = 0
    numElts = 0
    elt = None
    instInfoContext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
    for event, elt in instInfoContext:
        if event == "start":
            if elt.getparent() is not None:
                if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    if not foundInstance:
                        foundInstance = True
                        pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance")
                        if pi is None:
                            break
                        else:
                            streamingAspects = dict(pi.attrib.copy())
                            if creationSoftwareComment is None:
                                creationSoftwareComment = precedingComment(elt)
                    if not elt.tag.startswith("{http://www.xbrl.org/"):
                        instInfoNumRootFacts += 1
                        if instInfoNumRootFacts % 1000 == 0:
                            modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
                elif not foundInstance:       
                    break
            elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl":
                creationSoftwareComment = precedingComment(elt)
                if precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None:
                    modelXbrl.error("streamingExtensions:headerMisplaced",
                            _("Header is misplaced: %(error)s, must follow xbrli:xbrl element"),
                            modelObject=elt)
        elif event == "end":
            elt.clear()
            numElts += 1
            if numElts % 1000 == 0 and elt.getparent() is not None:
                while elt.getprevious() is not None and elt.getparent() is not None:
                    del elt.getparent()[0]
    if elt is not None:
        elt.clear()
    
    _file.seek(0,io.SEEK_SET) # allow reparsing
    if not foundInstance or streamingAspects is None:
        del elt
        _file.close()
        return None
    modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
    startedAt = time.time()
    try:
        version = Decimal(streamingAspects.get("version"))
        if int(version) != 1:
            modelXbrl.error("streamingExtensions:unsupportedVersion",
                    _("Streaming version %(version)s, major version number must be 1"),
                    modelObject=elt, version=version)
            foundErrors = True
    except (InvalidOperation, OverflowError):
        modelXbrl.error("streamingExtensions:versionError",
                _("Version %(version)s, number must be 1.n"),
                modelObject=elt, version=streamingAspects.get("version", "(none)"))
        foundErrors = True
    for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
        try:
            bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
            if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
                raise InvalidOperation
            elif bufAspect == "contextBuffer":
                contextBufferLimit = bufLimit
            elif bufAspect == "unitBuffer":
                unitBufferLimit = bufLimit
            elif bufAspect == "footnoteBuffer":
                footnoteBufferLimit = bufLimit
        except InvalidOperation:
            modelXbrl.error("streamingExtensions:valueError",
                    _("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"),
                    modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect))
            foundErrors = True
    if _streamingExtensionsValidate:
        incompatibleValidations = []
        _validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
        _disclosureSystem = modelXbrl.modelManager.disclosureSystem
        if _validateDisclosureSystem and _disclosureSystem.EFM:
            incompatibleValidations.append("EFM")
        if _validateDisclosureSystem and _disclosureSystem.GFM:
            incompatibleValidations.append("GFM")
        if _validateDisclosureSystem and _disclosureSystem.EBA:
            incompatibleValidations.append("EBA")
        if _validateDisclosureSystem and _disclosureSystem.HMRC:
            incompatibleValidations.append("EBA")
        if modelXbrl.modelManager.validateCalcLB:
            incompatibleValidations.append("calculation LB")
        if incompatibleValidations:
            modelXbrl.error("streamingExtensions:incompatibleValidation",
                    _("Streaming instance validation does not support %(incompatibleValidations)s validation"),
                    modelObject=modelXbrl, incompatibleValidations=', '.join(incompatibleValidations))
            foundErrors = True
    if instInfoContext.error_log:
        foundErrors = True
    logSyntaxErrors(instInfoContext)
    del instInfoContext # dereference

    for pluginMethod in pluginClassMethods("Streaming.BlockStreaming"):
        _blockingPluginName = pluginMethod(modelXbrl)
        if _blockingPluginName: # name of blocking plugin is returned
            modelXbrl.error("streamingExtensions:incompatiblePlugIn",
                    _("Streaming instance not supported by plugin %(blockingPlugin)s"),
                    modelObject=modelXbrl, blockingPlugin=_blockingPluginName)
            foundErrors = True
    
    if foundErrors:
        _file.close()
        return None

    _encoding = XmlUtil.encoding(_file.read(512))
    _file.seek(0,io.SEEK_SET) # allow reparsing

    if _streamingExtensionsValidate:
        validator = Validate(modelXbrl)
        instValidator = validator.instValidator

    eltMdlObjs = {}
    contextBuffer = []
    unitBuffer = []
    footnoteBuffer = []
    factBuffer = []
    numFacts = 1
    
    _streamingValidateFactsPlugin = any(True for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"))

    
    class modelLoaderTarget():
        def __init__(self, element_factory=None, parser=None):
            self.newTree = True
            self.currentMdlObj = None
            self.beforeInstanceStream = True
            self.beforeStartStreamingPlugin = True
            self.numRootFacts = 1
            modelXbrl.streamingParentModelObject = None
            modelXbrl.isStreamingMode = True
        def start(self, tag, attrib, nsmap=None):
            modelXbrl.streamingParentModelObject = self.currentMdlObj # pass parent to makeelement for ModelObjectFactory
            mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
            mdlObj.sourceline = 1
            if self.newTree:
                self.newTree = False
                self.currentMdlObj = mdlObj
                modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
                modelXbrl.modelDocument = modelDocument # needed for incremental validation
                mdlObj.init(modelDocument)
                modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement 
                modelDocument.parserLookupName = _parserLookupName
                modelDocument.parserLookupClass = _parserLookupClass
                modelDocument.xmlRootElement = mdlObj
                modelDocument.schemaLocationElements.add(mdlObj)
                modelDocument.documentEncoding = _encoding
                modelDocument._creationSoftwareComment = creationSoftwareComment
                modelXbrl.info("streamingExtensions:streaming",
                               _("Stream processing this instance."),
                               modelObject = modelDocument)
            else:
                self.currentMdlObj.append(mdlObj)
                self.currentMdlObj = mdlObj
                mdlObj._init()
                ns = mdlObj.namespaceURI
                ln = mdlObj.localName
                if (self.beforeInstanceStream and (
                    (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
                    (ns == XbrlConst.xbrli and ln in ("context", "unit")) or
                    (ns not in (XbrlConst.link, XbrlConst.xbrli)))):
                    self.beforeInstanceStream = False
                    if _streamingExtensionsValidate:
                        instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters())
                    else: # need default dimensions
                        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
                elif not self.beforeInstanceStream and self.beforeStartStreamingPlugin:
                    for pluginMethod in pluginClassMethods("Streaming.Start"):
                        pluginMethod(modelXbrl)
                    self.beforeStartStreamingPlugin = False
            return mdlObj
        def end(self, tag):
            modelDocument = modelXbrl.modelDocument
            mdlObj = self.currentMdlObj
            parentMdlObj = mdlObj.getparent()
            self.currentMdlObj = parentMdlObj
            ns = mdlObj.namespaceURI
            ln = mdlObj.localName
            if ns == XbrlConst.xbrli:
                if ln == "context":
                    if mdlObj.get("sticky"):
                        del mdlObj.attrib["sticky"]
                        XmlValidate.validate(modelXbrl, mdlObj)
                        modelDocument.contextDiscover(mdlObj)
                    else:
                        if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
                            # drop before adding as dropped may have same id as added
                            cntx = contextBuffer.pop(0)
                            dropContext(modelXbrl, cntx)
                            del parentMdlObj[parentMdlObj.index(cntx)]
                            cntx = None
                        XmlValidate.validate(modelXbrl, mdlObj)
                        modelDocument.contextDiscover(mdlObj)
                        if contextBufferLimit.is_finite():
                            contextBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        contextsToCheck = (mdlObj,)
                        instValidator.checkContexts(contextsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkContextsDimensions(contextsToCheck)
                        del contextsToCheck # dereference
                elif ln == "unit":
                    if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
                        # drop before additing as dropped may have same id as added
                        unit = unitBuffer.pop(0)
                        dropUnit(modelXbrl, unit)
                        del parentMdlObj[parentMdlObj.index(unit)]
                        unit = None 
                    XmlValidate.validate(modelXbrl, mdlObj)
                    modelDocument.unitDiscover(mdlObj)
                    if unitBufferLimit.is_finite():
                        unitBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkUnits( (mdlObj,) )
                elif ln == "xbrl": # end of document
                    # check remaining footnote refs
                    for footnoteLink in footnoteBuffer:
                        checkFootnoteHrefs(modelXbrl, footnoteLink)
                    for pluginMethod in pluginClassMethods("Streaming.Finish"):
                        pluginMethod(modelXbrl)
            elif ns == XbrlConst.link:
                if ln == "footnoteLink":
                    XmlValidate.validate(modelXbrl, mdlObj)
                    footnoteLinks = (mdlObj,)
                    modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
                    if footnoteBufferLimit.is_finite():
                        footnoteBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkLinks(footnoteLinks)
                        if len(footnoteBuffer) > footnoteBufferLimit:
                            # check that hrefObjects for locators were all satisfied
                                # drop before addition as dropped may have same id as added
                            footnoteLink = footnoteBuffer.pop(0)
                            checkFootnoteHrefs(modelXbrl, footnoteLink)
                            dropFootnoteLink(modelXbrl, footnoteLink)
                            del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            footnoteLink = None
                    footnoteLinks = None
                elif ln in ("schemaRef", "linkbaseRef"):
                    modelDocument.discoverHref(mdlObj)
                elif not modelXbrl.skipDTS:
                    if ln in ("roleRef", "arcroleRef"):
                        modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
            elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
                self.numRootFacts += 1
                XmlValidate.validate(modelXbrl, mdlObj)
                modelDocument.factDiscover(mdlObj, modelXbrl.facts)
                if _streamingExtensionsValidate or _streamingValidateFactsPlugin:
                    factsToCheck = (mdlObj,)  # validate current fact by itself
                    if _streamingExtensionsValidate:
                        instValidator.checkFacts(factsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkFactsDimensions(factsToCheck)
                    if _streamingValidateFactsPlugin:
                        # plugin attempts to process batch of all root facts not yet processed (not just current one)
                        factsToCheck = modelXbrl.facts.copy()
                        factsHaveBeenProcessed = True
                        # can block facts deletion if required data not yet available, such as numeric unit for DpmDB
                        for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
                            if not pluginMethod(modelXbrl, factsToCheck):
                                factsHaveBeenProcessed = False
                        if factsHaveBeenProcessed:
                            for fact in factsToCheck:
                                dropFact(modelXbrl, fact, modelXbrl.facts)
                                del parentMdlObj[parentMdlObj.index(fact)]
                    else:
                        dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed
                        del parentMdlObj[parentMdlObj.index(mdlObj)]
                    del factsToCheck # dereference fact or batch of facts
                if self.numRootFacts % 1000 == 0:
                    modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, 
                                                                                              100.0 * self.numRootFacts / instInfoNumRootFacts), 
                                              minTimeToShow=20.0)
            return mdlObj
        def data(self, data):
            self.currentMdlObj.text = data
        def comment(self, text):
            pass
        def pi(self, target, data):
            pass
        def close(self):
            del modelXbrl.streamingParentModelObject
            return None
        
    _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget())
    etree.parse(_file, parser=_parser, base_url=filepath)
    logSyntaxErrors(_parser)
    if _streamingExtensionsValidate and validator is not None:
        _file.close()
        del instValidator
        validator.close()
        # track that modelXbrl has been validated by this streaming extension
        modelXbrl._streamingExtensionValidated = True
        
    modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
    return modelXbrl.modelDocument
Example #9
0
def generateSkos(dts, skosFile):
    try:
        import os, io
        from arelle import XmlUtil, XbrlConst
        from arelle.ViewUtil import viewReferences, referenceURI
        skosNs = "http://www.w3.org/2004/02/skos/core#"
        rdfNs = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
        dts.modelManager.showStatus("initializing SKOS document")
        file = io.StringIO('''
  <!DOCTYPE rdf:RDF> 
  <nsmap>
  <rdf:RDF xmlns="urn:cgi:classifier:CGI:XBRL:201204#" xml:base="urn:cgi:classifierScheme:CGI:XBRL:201204" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xmlns:owl2xml="http://www.w3.org/2006/12/owl2-xml#" xmlns:p1="#" xmlns:owl="http://www.w3.org/2002/07/owl#" xmlns:xsd="http://www.w3.org/2001/XMLSchema#" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:xbrl-201204="urn:cgi:classifier:CGI:XBRL:201204#" xmlns:skos="http://www.w3.org/2004/02/skos/core#">
  <owl:Ontology rdf:about="" /> 
  <!-- Annotation properties --> 
  <owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/date" /> 
  <owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/source" /> 
  <owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/title" /> 
  <owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/description" /> 
  <owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/contributor" /> 
  <owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/creator" /> 
  <owl:AnnotationProperty rdf:about="http://purl.org/dc/elements/1.1/format" /> 
  <owl:AnnotationProperty rdf:about="http://www.w3.org/2002/07/owl#versionInfo" /> 
  <!-- Object Properties --> 
  <!--  http://www.w3.org/2004/02/skos/core#broader --> 
  <owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#broader" /> 
  <!--  http://www.w3.org/2004/02/skos/core#changeNote --> 
  <owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#changeNote" /> 
  <!--  http://www.w3.org/2004/02/skos/core#hasTopConcept --> 
  <owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#hasTopConcept" /> 
  <!--  http://www.w3.org/2004/02/skos/core#inScheme  --> 
  <owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#inScheme" /> 
  <!--  http://www.w3.org/2004/02/skos/core#topConceptOf  --> 
  <owl:ObjectProperty rdf:about="http://www.w3.org/2004/02/skos/core#topConceptOf" /> 
  <!-- Data properties --> 
  <!--  http://www.w3.org/2004/02/skos/core#definition --> 
  <owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#definition" /> 
  <!--  http://www.w3.org/2004/02/skos/core#editorialNote --> 
  <owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#editorialNote" /> 
  <!--  http://www.w3.org/2004/02/skos/core#historyNote --> 
  <owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#historyNote" /> 
  <!--  http://www.w3.org/2004/02/skos/core#notation --> 
  <owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#notation" /> 
  <!--  http://www.w3.org/2004/02/skos/core#prefLabel --> 
  <owl:DatatypeProperty rdf:about="http://www.w3.org/2004/02/skos/core#prefLabel" /> 
  <!-- Classes --> 
  <!--  http://www.w3.org/2002/07/owl#Thing --> 
  <owl:Class rdf:about="http://www.w3.org/2002/07/owl#Thing" /> 
  <!--  http://www.w3.org/2004/02/skos/core#Concept --> 
  <owl:Class rdf:about="http://www.w3.org/2004/02/skos/core#Concept" /> 
  <!--  http://www.w3.org/2004/02/skos/core#ConceptScheme --> 
  <owl:Class rdf:about="http://www.w3.org/2004/02/skos/core#ConceptScheme" /> 
  <!-- Individuals --> 
</rdf:RDF></nsmap>
 <!--  Generated by the Arelle(r) http://arelle.org --> 
''')
        from arelle.ModelObjectFactory import parser
        parser, parserLookupName, parserLookupClass = parser(dts, None)
        from lxml import etree
        xmlDocument = etree.parse(file, parser=parser, base_url=skosFile)
        file.close()
        xmlRootElement = xmlDocument.getroot()
        #xmlDocument.getroot().init(self)  ## is this needed ??
        for rdfElement in xmlDocument.iter(
                tag="{http://www.w3.org/1999/02/22-rdf-syntax-ns#}RDF"):
            break
        numSchemes = 0
        numConcepts = 0

        # use presentation relationships for broader and narrower concepts
        relationshipSet = dts.relationshipSet(XbrlConst.parentChild)

        def conceptUri(concept):
            return concept.qname.namespaceURI + "#" + concept.qname.localName

        def namespaceUri(qname):
            return qname.namespaceURI + "#" + qname.prefix

        priorSchemeSibling = None
        schemeNamespaces = set()

        dts.modelManager.showStatus("setting SKOS concepts from XBRL concepts")
        for qn, concept in sorted(dts.qnameConcepts.items(),
                                  key=lambda item: str(item[0])):
            if concept.modelDocument.targetNamespace not in (XbrlConst.xbrli,
                                                             XbrlConst.link,
                                                             XbrlConst.xlink,
                                                             XbrlConst.xl,
                                                             XbrlConst.xbrldt):
                if qn.namespaceURI not in schemeNamespaces:
                    # add conceptScheme
                    numSchemes += 1
                    skosElt = etree.Element(
                        "{http://www.w3.org/2004/02/skos/core#}ConceptScheme")
                    skosElt.set(
                        "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}about",
                        namespaceUri(qn))
                    elt = etree.SubElement(
                        skosElt,
                        "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}type")
                    elt.set(
                        "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource",
                        "http://www.w3.org/2002/07/owl#Thing")
                    elt = etree.SubElement(
                        skosElt,
                        "{http://www.w3.org/2004/02/skos/core#}notation")
                    elt.text = str(qn.prefix)
                    schemeNamespaces.add(qn.namespaceURI)
                    if priorSchemeSibling is not None:
                        priorSchemeSibling.addnext(skosElt)
                    else:
                        rdfElement.append(skosElt)
                    priorSchemeSibling = skosElt

                numConcepts += 1
                skosElt = etree.SubElement(
                    rdfElement,
                    "{http://www.w3.org/2004/02/skos/core#}Concept")
                skosElt.set(
                    "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}about",
                    conceptUri(concept))
                elt = etree.SubElement(
                    skosElt,
                    "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}type")
                elt.set(
                    "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource",
                    "http://www.w3.org/2002/07/owl#Thing")
                elt = etree.SubElement(
                    skosElt, "{http://www.w3.org/2004/02/skos/core#}notation")
                elt.text = str(concept.qname)
                definition = concept.label(
                    preferredLabel=XbrlConst.documentationLabel,
                    lang="en",
                    strip=True,
                    fallbackToQname=False)
                if definition:
                    elt = etree.SubElement(
                        skosElt,
                        "{http://www.w3.org/2004/02/skos/core#}definition")
                    elt.text = definition
                else:  # if no definition, look for any references
                    references = viewReferences(concept)
                    if references:
                        elt = etree.SubElement(
                            skosElt,
                            "{http://www.w3.org/2004/02/skos/core#}definition")
                        elt.text = references
                    linkedReferenceURI = referenceURI(concept)
                    if linkedReferenceURI:  # link to reference
                        elt = etree.SubElement(
                            skosElt,
                            "{http://www.w3.org/2004/02/skos/core#}definition")
                        elt.text = linkedReferenceURI
                labelsRelationshipSet = dts.relationshipSet(
                    XbrlConst.conceptLabel)
                if labelsRelationshipSet:
                    for modelLabelRel in labelsRelationshipSet.fromModelObject(
                            concept):
                        label = modelLabelRel.toModelObject
                        if label.role == XbrlConst.standardLabel:
                            elt = etree.SubElement(
                                skosElt,
                                "{http://www.w3.org/2004/02/skos/core#}prefLabel"
                            )
                            elt.set(
                                "{http://www.w3.org/XML/1998/namespace}lang",
                                label.xmlLang)
                            elt.text = label.text.strip()
                for rel in relationshipSet.fromModelObject(
                        concept):  # narrower
                    elt = etree.SubElement(
                        skosElt,
                        "{http://www.w3.org/2004/02/skos/core#}narrower")
                    elt.set(
                        "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource",
                        conceptUri(rel.toModelObject))
                for rel in relationshipSet.toModelObject(concept):  # broader
                    elt = etree.SubElement(
                        skosElt,
                        "{http://www.w3.org/2004/02/skos/core#}broader")
                    elt.set(
                        "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource",
                        conceptUri(rel.fromModelObject))

                elt = etree.SubElement(
                    skosElt, "{http://www.w3.org/2004/02/skos/core#}inScheme")
                elt.set(
                    "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource",
                    namespaceUri(qn))

        dts.modelManager.showStatus("saving SKOS file")
        fh = open(skosFile, "w", encoding="utf-8")
        XmlUtil.writexml(fh, xmlDocument, encoding="utf-8")
        fh.close()

        dts.info(
            "info:saveSKOS",
            _("SKOS of %(entryFile)s has %(numberOfConcepts)s concepts in SKOS RDF file %(skosOutputFile)s."
              ),
            modelObject=dts,
            entryFile=dts.uri,
            numberOfConcepts=numConcepts,
            skosOutputFile=skosFile)
        dts.modelManager.showStatus("ready", 3000)
    except Exception as ex:
        dts.error("exception",
                  _("SKOS generation exception: %(error)s"),
                  error=ex,
                  modelXbrl=dts,
                  exc_info=True)
Example #10
0
def generateUpdatedTableLB(dts, updatedTableLinkbaseFile):
    import os, io
    from arelle import XmlUtil, XbrlConst
    from arelle.ViewUtil import viewReferences, referenceURI
    from arelle.ModelRenderingObject import ModelEuAxisCoord

    if dts.fileSource.isArchive:
        dts.error(
            "genTblLB:outFileIsArchive",
            _("Updated Table Linkbase file cannot be an archive: %(tableLBOutputFile)s."
              ),
            modelObject=dts,
            tableLBOutputFile=updatedTableLinkbaseFile)
        return
    tblAxisRelSet = dts.relationshipSet(XbrlConst.euTableAxis)
    axisMbrRelSet = dts.relationshipSet(XbrlConst.euAxisMember)
    if len(tblAxisRelSet.modelRelationships) == 0:
        dts.error(
            "genTblLB:noInputTables",
            _("DTS does not contain Eurofiling 2010 tables and axes: %(entryFile)s."
              ),
            modelObject=dts,
            entryFile=dts.uri)
        return

    file = io.StringIO('''
<nsmap>
<link:linkbase 
xmlns:label="http://xbrl.org/2008/label"
xmlns:gen="http://xbrl.org/2008/generic"
xmlns:df="http://xbrl.org/2008/filter/dimension"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:reference="http://xbrl.org/2008/reference"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:table="http://xbrl.org/2011/table"
xmlns:formula="http://xbrl.org/2008/formula"
xsi:schemaLocation="             
http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd
http://xbrl.org/2008/generic http://www.xbrl.org/2008/generic-link.xsd
http://xbrl.org/2008/reference http://www.xbrl.org/2008/generic-reference.xsd
http://xbrl.org/2008/label http://www.xbrl.org/2008/generic-label.xsd
http://xbrl.org/2011/table http://www.xbrl.org/2011/table.xsd
http://xbrl.org/2008/filter/dimension http://www.xbrl.org/2008/dimension-filter.xsd">
  <link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/table-filter" xlink:type="simple" 
   xlink:href="http://www.xbrl.org/2011/table.xsd#table-filter"/>
  <link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/table-axis" xlink:type="simple"
   xlink:href="http://www.xbrl.org/2011/table.xsd#table-axis"/>
  <link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/axis-subtree" xlink:type="simple"
   xlink:href="http://www.xbrl.org/2011/table.xsd#axis-subtree"/>
  <link:arcroleRef arcroleURI="http://xbrl.org/arcrole/2011/axis-filter" xlink:type="simple"
   xlink:href="http://www.xbrl.org/2011/filter-axis.xsd#axis-filter"/>
</link:linkbase>
</nsmap>
<!--  Generated by Arelle(r) http://arelle.org --> 
''')
    from arelle.ModelObjectFactory import parser
    parser, parserLookupName, parserLookupClass = parser(dts, None)
    from lxml import etree
    xmlDocument = etree.parse(file,
                              parser=parser,
                              base_url=updatedTableLinkbaseFile)
    file.close()
    nsmapElt = xmlDocument.getroot()
    #xmlDocument.getroot().init(self)  ## is this needed ??
    for lbElement in xmlDocument.iter(
            tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
        break

    class DocObj:  # fake ModelDocument for namespaces
        def __init__(self):
            self.xmlRootElement = lbElement
            self.xmlDocument = xmlDocument

    docObj = DocObj()

    numELRs = 0
    numTables = 0

    def copyAttrs(fromElt, toElt, attrTags):
        for attr in attrTags:
            if fromElt.get(attr):
                toElt.set(attr, fromElt.get(attr))

    def generateTable(newLinkElt, newTblElt, srcTblElt, tblAxisRelSet,
                      axisMbrRelSet, visited):
        if srcTblElt is not None:
            for rel in tblAxisRelSet.fromModelObject(srcTblElt):
                srcAxisElt = rel.toModelObject
                if isinstance(srcAxisElt, ModelEuAxisCoord):
                    visited.add(srcAxisElt)
                    newAxisElt = etree.SubElement(
                        newLinkElt, "{http://xbrl.org/2011/table}ruleAxis")
                    copyAttrs(srcAxisElt, newAxisElt,
                              ("id", "{http://www.w3.org/1999/xlink}type",
                               "{http://www.w3.org/1999/xlink}label"))
                    newAxisElt.set("abstract",
                                   "true")  # always true on root element
                    newArcElt = etree.SubElement(
                        newLinkElt, "{http://xbrl.org/2011/table}axisArc")
                    copyAttrs(rel, newArcElt,
                              ("id", "{http://www.w3.org/1999/xlink}type",
                               "{http://www.w3.org/1999/xlink}from",
                               "{http://www.w3.org/1999/xlink}to", "order"))
                    newArcElt.set("{http://www.w3.org/1999/xlink}arcrole",
                                  XbrlConst.tableBreakdown)
                    newArcElt.set("axisDisposition", rel.axisDisposition)
                    generateAxis(newLinkElt, newAxisElt, srcAxisElt,
                                 axisMbrRelSet, visited)
                    visited.discard(srcAxisElt)

    def generateAxis(newLinkElt, newAxisParentElt, srcAxisElt, axisMbrRelSet,
                     visited):
        for rel in axisMbrRelSet.fromModelObject(srcAxisElt):
            tgtAxisElt = rel.toModelObject
            if isinstance(tgtAxisElt,
                          ModelEuAxisCoord) and tgtAxisElt not in visited:
                visited.add(tgtAxisElt)
                newAxisElt = etree.SubElement(
                    newLinkElt, "{http://xbrl.org/2011/table}ruleAxis")
                copyAttrs(
                    tgtAxisElt, newAxisElt,
                    ("id", "abstract", "{http://www.w3.org/1999/xlink}type",
                     "{http://www.w3.org/1999/xlink}label"))
                if tgtAxisElt.primaryItemQname:
                    newRuleElt = etree.SubElement(
                        newAxisElt, "{http://xbrl.org/2008/formula}concept")
                    newQnameElt = etree.SubElement(
                        newRuleElt, "{http://xbrl.org/2008/formula}qname")
                    newQnameElt.text = XmlUtil.addQnameValue(
                        docObj, tgtAxisElt.primaryItemQname)
                for dimQname, memQname in tgtAxisElt.explicitDims:
                    newRuleElt = etree.SubElement(
                        newAxisElt,
                        "{http://xbrl.org/2008/formula}explicitDimension")
                    newRuleElt.set("dimension",
                                   XmlUtil.addQnameValue(docObj, dimQname))
                    newMbrElt = etree.SubElement(
                        newRuleElt, "{http://xbrl.org/2008/formula}member")
                    newQnameElt = etree.SubElement(
                        newMbrElt, "{http://xbrl.org/2008/formula}qname")
                    newQnameElt.text = XmlUtil.addQnameValue(docObj, memQname)
                newArcElt = etree.SubElement(
                    newLinkElt, "{http://xbrl.org/2011/table}axisArc")
                copyAttrs(rel, newArcElt,
                          ("id", "{http://www.w3.org/1999/xlink}type",
                           "{http://www.w3.org/1999/xlink}from",
                           "{http://www.w3.org/1999/xlink}to", "order"))
                newArcElt.set("{http://www.w3.org/1999/xlink}arcrole",
                              XbrlConst.tableAxisSubtree)
                generateAxis(newLinkElt, newAxisElt, tgtAxisElt, axisMbrRelSet,
                             visited)
                visited.discard(tgtAxisElt)

    # sort URIs
    linkroleUris = sorted(
        [linkroleUri for linkroleUri in tblAxisRelSet.linkRoleUris])

    firstNewLinkElt = None
    roleRefUris = set()
    for linkroleUri in linkroleUris:
        numELRs += 1
        newLinkElt = etree.SubElement(lbElement,
                                      "{http://xbrl.org/2008/generic}link")
        newLinkElt.set("{http://www.w3.org/1999/xlink}type", "extended")
        newLinkElt.set("{http://www.w3.org/1999/xlink}role", linkroleUri)
        if firstNewLinkElt is None: firstNewLinkElt = newLinkElt
        # To do: add roleRef if needed
        tblAxisRelSet = dts.relationshipSet(XbrlConst.euTableAxis, linkroleUri)
        axisMbrRelSet = dts.relationshipSet(XbrlConst.euAxisMember,
                                            linkroleUri)
        for srcTblElt in tblAxisRelSet.rootConcepts:
            if srcTblElt.tag == "{http://www.eurofiling.info/2010/rendering}table":
                numTables += 1
                newTblElt = etree.SubElement(
                    newLinkElt, "{http://xbrl.org/2011/table}table")
                newTblElt.set("aspectModel", "dimensional")
                copyAttrs(srcTblElt, newTblElt,
                          ("id", "{http://www.w3.org/1999/xlink}type",
                           "{http://www.w3.org/1999/xlink}label"))
                generateTable(newLinkElt, newTblElt, srcTblElt, tblAxisRelSet,
                              axisMbrRelSet, set())

                if linkroleUri not in roleRefUris:
                    srcRoleRefElt = XmlUtil.descendant(srcTblElt.getroottree(),
                                                       XbrlConst.link,
                                                       "roleRef", "roleURI",
                                                       linkroleUri)
                    if srcRoleRefElt is not None:
                        roleRefUris.add(linkroleUri)
                        newRoleRefElt = etree.Element(
                            "{http://www.xbrl.org/2003/linkbase}roleRef")
                        copyAttrs(
                            srcRoleRefElt, newRoleRefElt,
                            ("roleURI", "{http://www.w3.org/1999/xlink}type",
                             "{http://www.w3.org/1999/xlink}href"))
                        firstNewLinkElt.addprevious(newRoleRefElt)

    fh = open(updatedTableLinkbaseFile, "w", encoding="utf-8")
    XmlUtil.writexml(fh, xmlDocument, encoding="utf-8")
    fh.close()

    dts.info(
        "info:updateTableLinkbase",
        _("Updated Table Linkbase of %(entryFile)s has %(numberOfLinkroles)s linkroles, %(numberOfTables)s tables in file %(tableLBOutputFile)s."
          ),
        modelObject=dts,
        entryFile=dts.uri,
        numberOfLinkroles=numELRs,
        numberOfTables=numTables,
        tableLBOutputFile=updatedTableLinkbaseFile)
Example #11
0
def generateHtmlEbaTablesetFiles(dts, indexFile, lang="en"):
    try:
        import os, io
        from arelle import Version, XbrlConst, XmlUtil
        from arelle.ViewFileRenderedGrid import viewRenderedGrid
        from arelle.ModelRenderingObject import ModelEuTable, ModelTable
        
        numTableFiles = 0

        file = io.StringIO('''
<html xmlns="http://www.w3.org/1999/xhtml">
<head id="Left">
  <link type="text/css" rel="stylesheet" href="http://arelle.org/files/EBA/style20121210/eba.css" /> 
</head>
<body class="LTR IE7 ENGB">
    <ul class="CMSListMenuUL" id="Vertical2"/>
</body>
</html>
'''
         )
        from arelle.ModelObjectFactory import parser
        parser, parserLookupName, parserLookupClass = parser(dts,None)
        from lxml import etree
        indexDocument = etree.parse(file,parser=parser,base_url=indexFile)
        file.close()
        #xmlDocument.getroot().init(self)  ## is this needed ??
        for listElt in  indexDocument.iter(tag="{http://www.w3.org/1999/xhtml}ul"):
            break
    
        class nonTkBooleanVar():
            def __init__(self, value=True):
                self.value = value
            def set(self, value):
                self.value = value
            def get(self):
                return self.value
    
        class View():
            def __init__(self, tableOrELR, ignoreDimValidity, xAxisChildrenFirst, yAxisChildrenFirst):
                self.tblELR = tableOrELR
                # context menu boolean vars (non-tkinter boolean
                self.ignoreDimValidity = nonTkBooleanVar(value=ignoreDimValidity)
                self.xAxisChildrenFirst = nonTkBooleanVar(value=xAxisChildrenFirst)
                self.yAxisChildrenFirst = nonTkBooleanVar(value=yAxisChildrenFirst)
    
        indexBase = indexFile.rpartition(".")[0]
        groupTableRels = dts.modelXbrl.relationshipSet(XbrlConst.euGroupTable)
        modelTables = []
        tblCssExtras='''
body {background-image:url('http://arelle.org/files/EBA/style20121210/lhsbackground.jpg')}
table {background:#fff}
'''
        # order number is missing
        def viewTable(modelTable):
            if isinstance(modelTable, (ModelEuTable, ModelTable)):
                # status
                dts.modelManager.cntlr.addToLog("viewing: " + modelTable.id)
                # for table file name, use table ELR
                tblFile = os.path.join(os.path.dirname(indexFile), modelTable.id + ".html")
                viewRenderedGrid(dts, 
                                 tblFile, 
                                 lang=lang, 
                                 sourceView=View(modelTable, False, False, True),
                                 cssExtras=tblCssExtras)
                
                # generaate menu entry
                elt = etree.SubElement(listElt, "{http://www.w3.org/1999/xhtml}li")
                elt.set("class", "CMSListMenuLI")
                elt.set("id", modelTable.id)
                elt = etree.SubElement(elt, "{http://www.w3.org/1999/xhtml}a")
                elt.text = modelTable.genLabel(lang=lang, strip=True)
                elt.set("class", "CMSListMenuLink")
                elt.set("href", "javascript:void(0)")
                elt.set("onClick", "javascript:parent.body.location.href='{0}';".format(modelTable.id + ".html"))
                elt.text = modelTable.genLabel(lang=lang, strip=True)
                
            else:  # just a header

                # generaate menu entry
                elt = etree.SubElement(listElt, "{http://www.w3.org/1999/xhtml}li")
                elt.set("class", "CMSListMenuLink")
                elt.set("id", modelTable.id)
                elt.text = modelTable.label(lang=lang, strip=True)

            for rel in groupTableRels.fromModelObject(modelTable):
                viewTable(rel.toModelObject)

    
        for rootConcept in groupTableRels.rootConcepts:
            sourceline = 0
            for rel in dts.modelXbrl.relationshipSet(XbrlConst.euGroupTable).fromModelObject(rootConcept):
                sourceline = rel.sourceline
                break
            modelTables.append((rootConcept, sourceline))
            
        for modelTable, order in sorted(modelTables, key=lambda x: x[1]):
            viewTable(modelTable)
        
        with open(indexBase + "FormsFrame.html", "wt", encoding="utf-8") as fh:
            XmlUtil.writexml(fh, indexDocument, encoding="utf-8")
            
        with open(indexFile, "wt", encoding="utf-8") as fh:
            fh.write(
'''
<html xmlns="http://www.w3.org/1999/xhtml">
<head id="Head1">
  <title>European Banking Authority - EBA  - FINREP Taxonomy</title>
  <meta name="generator" content="Arelle(r) {0}" /> 
  <meta name="provider" content="Aguilonius(r)" />
  <meta http-equiv="content-type" content="text/html; charset=UTF-8" /> 
  <meta http-equiv="pragma" content="no-cache" /> 
  <meta http-equiv="content-style-type" content="text/css" /> 
  <meta http-equiv="content-script-type" content="text/javascript" /> 
  <link type="text/css" rel="stylesheet" href="http://arelle.org/files/EBA/style20121210/eba.css" /> 
</head>
<frameset border="0" frameborder="0" rows="90,*">
   <frame name="head" src="{1}" scrolling="no" marginwidth="0" marginheight="10"/>
   <frameset  bordercolor="#0000cc" border="10" frameborder="no" framespacing="0" cols="360, *">
      <frame src="{2}" name="menu" bordercolor="#0000cc"/>
      <frame src="{3}" name="body" bordercolor="#0000cc"/>
   </frameset>
</frameset>
'''.format(Version.version,
           os.path.basename(indexBase) + "TopFrame.html",
           os.path.basename(indexBase) + "FormsFrame.html",
           os.path.basename(indexBase) + "CenterLanding.html",
           ))
        
        with open(indexBase + "TopFrame.html", "wt", encoding="utf-8") as fh:
            fh.write(
'''
<html xmlns="http://www.w3.org/1999/xhtml">
<head id="Top">
  <link type="text/css" rel="stylesheet" href="http://arelle.org/files/EBA/style20121210/eba.css" /> 
</head>
  <body class="LTR IE7 ENGB">
   <div id="topsection">
      <div id="topsectionLeft" style="cursor:pointer;" onclick="location.href='http://www.eba.europa.eu/home.aspx';"></div>
      <div id="topsectionRight"></div>
      <div id="topnavigation">
      <ul id="menuElem">
        <li><a href="http://www.eba.europa.eu/topnav/Contacts.aspx">Contacts</a></li>
        <li><a href="http://www.eba.europa.eu/topnav/Links.aspx">Links</a></li>
        <li><a href="http://www.eba.europa.eu/topnav/Sitemap.aspx">Sitemap</a></li>
        <li><a href="http://www.eba.europa.eu/topnav/Legal-Notice.aspx">Legal Notice</a></li>
      </ul>
    </div>
  </body>
</html>
''')
        
        with open(indexBase + "CenterLanding.html", "wt", encoding="utf-8") as fh:
            fh.write(
'''
<html xmlns="http://www.w3.org/1999/xhtml">
<head id="Center">
  <link type="text/css" rel="stylesheet" href="http://http://arelle.org/files/EBA/style20121210/eba.css" /> 
</head>
<body class="LTR IE7 ENGB">
  <div id="plc_lt_zoneContent_usercontrol_userControlElem_ContentPanel">
    <div id="plc_lt_zoneContent_usercontrol_userControlElem_PanelTitle">
      <div id="pagetitle" style="float:left;width:500px;">
        <h1>Taxonomy Tables Viewer</h1>
      </div>
    </div>
  </div>
  <div style="clear:both;"></div>
  <div id="contentcenter">
    <p style="text-align: justify; margin-top: 0pt; margin-bottom: 0pt">Please select tables to view by clicking in the left column.</p>
  </div>
</body>
</html>
''')
        
        # to merge gif's and style sheets, use a zipfile sibling of the python plug-in file.
        #import zipfile
        #zf = zipfile.ZipFile(__file__.rpartition('.')[0] + "Files.zip", mode="r")
        #zf.extractall(path=os.path.dirname(indexBase))
        #zf.close()
        
        dts.info("info:saveEBAtables",
                 _("Tables index file of %(entryFile)s has %(numberTableFiles)s table files with index file %(indexFile)s."),
                 modelObject=dts,
                 entryFile=dts.uri, numberTableFiles=numTableFiles, indexFile=indexFile)
    
        dts.modelManager.showStatus(_("Saved EBA HTML Table Files"), 5000)
    except Exception as ex:
        dts.error("exception",
            _("HTML EBA Tableset files generation exception: %(error)s"), error=ex,
            modelXbrl=dts,
            exc_info=True)
Example #12
0
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath):
    # check if big instance and has header with an initial incomplete tree walk (just 2 elements
    def logSyntaxErrors(parsercontext):
        for error in parsercontext.error_log:
            modelXbrl.error("xmlSchema:syntax",
                    _("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"),
                    modelObject=modelDocument, fileName=os.path.basename(filepath), 
                    error=error.message, line=error.line, column=error.column, sourceAction="streaming")
    #### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
    #### note: iterparse wants a binary file, but file is text mode
    _file, = modelXbrl.fileSource.file(filepath, binary=True)
    startedAt = time.time()
    modelXbrl.profileActivity()
    parsercontext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
    foundInstance = False
    foundErrors = False
    streamingAspects = None
    numRootFacts1 = 0
    numElts = 0
    elt = None
    for event, elt in parsercontext:
        if event == "start":
            if elt.getparent() is not None:
                if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    if not foundInstance:
                        foundInstance = True
                        pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance")
                        if pi is None:
                            break
                        else:
                            streamingAspects = dict(pi.attrib.copy())
                    if not elt.tag.startswith("{http://www.xbrl.org/"):
                        numRootFacts1 += 1
                        if numRootFacts1 % 1000 == 0:
                            modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
                elif not foundInstance:       
                    break
            elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl" and precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None:
                modelXbrl.error("streamingExtensions:headerMisplaced",
                        _("Header is misplaced: %(error)s, must follow xbrli:xbrl element"),
                        modelObject=elt)
        elif event == "end":
            elt.clear()
            numElts += 1
            if numElts % 1000 == 0 and elt.getparent() is not None:
                while elt.getprevious() is not None and elt.getparent() is not None:
                    del elt.getparent()[0]
    if elt is not None:
        elt.clear()
    _file.seek(0,io.SEEK_SET) # allow reparsing
    if not foundInstance or streamingAspects is None:
        del elt, parsercontext
        _file.close()
        return None
    modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
    startedAt = time.time()
    try:
        version = Decimal(streamingAspects.get("version"))
        if int(version) != 1:
            modelXbrl.error("streamingExtensions:unsupportedVersion",
                    _("Streaming version %(version)s, major version number must be 1"),
                    modelObject=elt, version=version)
            foundErrors = True
    except (InvalidOperation, OverflowError):
        modelXbrl.error("streamingExtensions:versionError",
                _("Version %(version)s, number must be 1.n"),
                modelObject=elt, version=streamingAspects.get("version", "(none)"))
        foundErrors = True
    for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
        try:
            bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
            if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
                raise InvalidOperation
            elif bufAspect == "contextBuffer":
                contextBufferLimit = bufLimit
            elif bufAspect == "unitBuffer":
                unitBufferLimit = bufLimit
            elif bufAspect == "footnoteBuffer":
                footnoteBufferLimit = bufLimit
        except InvalidOperation:
            modelXbrl.error("streamingExtensions:valueError",
                    _("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"),
                    modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect))
            foundErrors = True
    if parsercontext.error_log:
        foundErrors = True
    logSyntaxErrors(parsercontext)
    
    if foundErrors:
        _file.close()
        return None
    parsercontext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
    _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl,filepath)
    eltMdlObjs = {}
    beforeInstanceStream = True
    validator = None
    contextBuffer = []
    unitBuffer = []
    footnoteBuffer = []
    factBuffer = []
    numFacts = numRootFacts2 = 1
    for event, elt in parsercontext:
        if event == "start":
            mdlObj = _parser.makeelement(elt.tag, attrib=elt.attrib, nsmap=elt.nsmap)
            mdlObj.sourceline = elt.sourceline
            eltMdlObjs[elt] = mdlObj
            if elt.getparent() is None:
                modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, etree.ElementTree(mdlObj))
                modelDocument.xmlRootElement = mdlObj
                modelXbrl.modelDocument = modelDocument # needed for incremental validation
                mdlObj.init(modelDocument)
                modelXbrl.info("streamingExtensions:streaming",
                               _("Stream processing this instance."),
                               modelObject = modelDocument)    
            else:
                eltMdlObjs[elt.getparent()].append(mdlObj)
                mdlObj._init()
                ns = mdlObj.namespaceURI
                ln = mdlObj.localName
                if (beforeInstanceStream and (
                    (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
                    (ns == XbrlConst.xbrli and ln in ("context", "unit")) or
                    (ns not in (XbrlConst.link, XbrlConst.xbrli)))):
                    beforeInstanceStream = False
                    if _streamingExtensionsValidate:
                        validator = Validate(modelXbrl)
                        validator.instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters())
                    else: # need default dimensions
                        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
            mdlObj = None # deref
                        
        elif event == "end":
            mdlObj = eltMdlObjs.pop(elt)
            if elt.text: # text available after child nodes processed
                mdlObj.text = elt.text
            ns = mdlObj.namespaceURI
            ln = mdlObj.localName
            parentMdlObj = mdlObj.getparent()
            if ns == XbrlConst.xbrli:
                if ln == "context":
                    if mdlObj.get("sticky"):
                        del mdlObj.attrib["sticky"]
                        modelDocument.contextDiscover(mdlObj)
                    else:
                        if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
                            # drop before adding as dropped may have same id as added
                            cntx = contextBuffer.pop(0)
                            dropContext(modelXbrl, cntx)
                            del parentMdlObj[parentMdlObj.index(cntx)]
                            cntx = None
                        modelDocument.contextDiscover(mdlObj)
                        if contextBufferLimit.is_finite():
                            contextBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        contextsToCheck = (mdlObj,)
                        validator.instValidator.checkContexts(contextsToCheck)
                        if modelXbrl.hasXDT:
                            validator.instValidator.checkContextsDimensions(contextsToCheck)
                        del contextsToCheck # dereference
                elif ln == "unit":
                    if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
                        # drop before additing as dropped may have same id as added
                        unit = unitBuffer.pop(0)
                        dropUnit(modelXbrl, unit)
                        del parentMdlObj[parentMdlObj.index(unit)]
                        unit = None 
                    modelDocument.unitDiscover(mdlObj)
                    if unitBufferLimit.is_finite():
                        unitBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        validator.instValidator.checkUnits( (mdlObj,) )
                elif ln == "xbrl": # end of document
                    # check remaining footnote refs
                    for footnoteLink in footnoteBuffer:
                        checkFootnoteHrefs(modelXbrl, footnoteLink)
                elt.clear()
            elif ns == XbrlConst.link:
                if ln in ("schemaRef", "linkbaseRef"):
                    modelDocument.discoverHref(mdlObj)
                elif ln in ("roleRef", "arcroleRef"):
                    modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
                elif ln == "footnoteLink":
                    footnoteLinks = (mdlObj,)
                    modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
                    if footnoteBufferLimit.is_finite():
                        footnoteBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        validator.instValidator.checkLinks(footnoteLinks)
                        if len(footnoteBuffer) > footnoteBufferLimit:
                            # check that hrefObjects for locators were all satisfied
                                # drop before addition as dropped may have same id as added
                            footnoteLink = footnoteBuffer.pop(0)
                            checkFootnoteHrefs(modelXbrl, footnoteLink)
                            dropFootnoteLink(modelXbrl, footnoteLink)
                            del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            footnoteLink = None
                    footnoteLinks = None
                elt.clear()
            elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
                numRootFacts2 += 1
                modelDocument.factDiscover(mdlObj, modelXbrl.facts)
                XmlValidate.validate(modelXbrl, mdlObj)
                if _streamingExtensionsValidate:
                    factsToCheck = (mdlObj,)
                    validator.instValidator.checkFacts(factsToCheck)
                    if modelXbrl.hasXDT:
                        validator.instValidator.checkFactsDimensions(factsToCheck)
                    del factsToCheck
                    dropFact(modelXbrl, mdlObj, modelXbrl.facts)
                    del parentMdlObj[parentMdlObj.index(mdlObj)]
                if numRootFacts2 % 1000 == 0:
                    modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(numRootFacts2, numRootFacts1, 100.0 * numRootFacts2 / numRootFacts1), 
                                              minTimeToShow=20.0)
                # get rid of root element from iterparse's tree
                elt.clear()
                while elt.getprevious() is not None:  # cleans up any prior siblings
                    del elt.getparent()[0]
            mdlObj = None # deref
    logSyntaxErrors(parsercontext)
    del parsercontext
    if validator is not None:
        validator.close()
    _file.close()
    modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
    return modelDocument
Example #13
0
def create(modelXbrl, type, uri, schemaRefs=None, isEntry=False):
    normalizedUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
        uri, None)
    if isEntry:
        modelXbrl.uri = normalizedUri
        modelXbrl.entryLoadingUrl = normalizedUri
        modelXbrl.uriDir = os.path.dirname(normalizedUri)
        for i in range(modelXbrl.modelManager.disclosureSystem.
                       maxSubmissionSubdirectoryEntryNesting):
            modelXbrl.uriDir = os.path.dirname(modelXbrl.uriDir)
    filepath = modelXbrl.modelManager.cntlr.webCache.getfilename(normalizedUri)
    # XML document has nsmap root element to replace nsmap as new xmlns entries are required
    if type == Type.INSTANCE:
        # modelXbrl.uriDir = os.path.dirname(normalizedUri)
        Xml = ('<nsmap>'
               '<xbrl xmlns="http://www.xbrl.org/2003/instance"'
               ' xmlns:link="http://www.xbrl.org/2003/linkbase"'
               ' xmlns:xlink="http://www.w3.org/1999/xlink">')
        if schemaRefs:
            for schemaRef in schemaRefs:
                Xml += '<link:schemaRef xlink:type="simple" xlink:href="{0}"/>'.format(
                    schemaRef.replace("\\", "/"))
        Xml += '</xbrl></nsmap>'
    elif type == Type.SCHEMA:
        Xml = (
            '<nsmap><schema xmlns="http://www.w3.org/2001/XMLSchema" /></nsmap>'
        )
    elif type == Type.RSSFEED:
        Xml = '<nsmap><rss version="2.0" /></nsmap>'
    elif type == Type.DTSENTRIES:
        Xml = None
    else:
        type = Type.UnknownXML
        Xml = '<nsmap/>'
    if Xml:
        import io
        file = io.StringIO(Xml)
        _parser, _parserLookupName, _parserLookupClass = parser(
            modelXbrl, filepath)
        xmlDocument = etree.parse(file, parser=_parser, base_url=filepath)
        file.close()
    else:
        xmlDocument = None
    if type == Type.RSSFEED:
        from arelle.ModelRssObject import ModelRssObject
        modelDocument = ModelRssObject(modelXbrl, type, uri, filepath,
                                       xmlDocument)
    else:
        modelDocument = ModelDocument(modelXbrl, type, normalizedUri, filepath,
                                      xmlDocument)
    if Xml:
        modelDocument.parser = _parser  # needed for XmlUtil addChild's makeelement
        modelDocument.parserLookupName = _parserLookupName
        modelDocument.parserLookupClass = _parserLookupClass
        modelDocument.documentEncoding = "utf-8"
        rootNode = xmlDocument.getroot()
        rootNode.init(modelDocument)
        if xmlDocument:
            for semanticRoot in rootNode.iterchildren():
                if isinstance(semanticRoot, ModelObject):
                    modelDocument.xmlRootElement = semanticRoot
                    break
    if type == Type.INSTANCE:
        modelDocument.instanceDiscover(modelDocument.xmlRootElement)
    elif type == Type.RSSFEED:
        modelDocument.rssFeedDiscover(modelDocument.xmlRootElement)
    elif type == Type.SCHEMA:
        modelDocument.targetNamespace = None
        modelDocument.isQualifiedElementFormDefault = False
        modelDocument.isQualifiedAttributeFormDefault = False
    modelDocument.definesUTR = False
    return modelDocument
Example #14
0
def load(modelXbrl,
         uri,
         base=None,
         referringElement=None,
         isEntry=False,
         isDiscovered=False,
         isIncluded=None,
         namespace=None,
         reloadCache=False):
    if referringElement is None:  # used for error messages
        referringElement = modelXbrl
    normalizedUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
        uri, base)
    if isEntry:
        modelXbrl.entryLoadingUrl = normalizedUri  # for error loggiong during loading
        modelXbrl.uri = normalizedUri
        modelXbrl.uriDir = os.path.dirname(normalizedUri)
        for i in range(modelXbrl.modelManager.disclosureSystem.
                       maxSubmissionSubdirectoryEntryNesting):
            modelXbrl.uriDir = os.path.dirname(modelXbrl.uriDir)
    if modelXbrl.modelManager.validateDisclosureSystem and \
       not normalizedUri.startswith(modelXbrl.uriDir) and \
       not modelXbrl.modelManager.disclosureSystem.hrefValid(normalizedUri):
        blocked = modelXbrl.modelManager.disclosureSystem.blockDisallowedReferences
        modelXbrl.error(
            ("EFM.6.22.02", "GFM.1.1.3", "SBR.NL.2.1.0.06"
             if normalizedUri.startswith("http") else "SBR.NL.2.2.0.17"),
            _("Prohibited file for filings %(blockedIndicator)s: %(url)s"),
            modelObject=referringElement,
            url=normalizedUri,
            blockedIndicator=_(" blocked") if blocked else "")
        if blocked:
            return None
    if normalizedUri in modelXbrl.modelManager.disclosureSystem.mappedFiles:
        mappedUri = modelXbrl.modelManager.disclosureSystem.mappedFiles[
            normalizedUri]
    else:  # handle mapped paths
        mappedUri = normalizedUri
        for mapFrom, mapTo in modelXbrl.modelManager.disclosureSystem.mappedPaths:
            if normalizedUri.startswith(mapFrom):
                mappedUri = mapTo + normalizedUri[len(mapFrom):]
                break
    if isEntry:
        modelXbrl.entryLoadingUrl = mappedUri  # for error loggiong during loading
    if modelXbrl.fileSource.isInArchive(mappedUri):
        filepath = mappedUri
    else:
        filepath = modelXbrl.modelManager.cntlr.webCache.getfilename(
            mappedUri, reload=reloadCache)
        if filepath:
            uri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(filepath)
    if filepath is None:  # error such as HTTPerror is already logged
        modelXbrl.error("FileNotLoadable",
                        _("File can not be loaded: %(fileName)s"),
                        modelObject=referringElement,
                        fileName=mappedUri)
        return None

    modelDocument = modelXbrl.urlDocs.get(mappedUri)
    if modelDocument:
        return modelDocument

    # load XML and determine type of model document
    modelXbrl.modelManager.showStatus(_("parsing {0}").format(uri))
    file = None
    try:
        if (modelXbrl.modelManager.validateDisclosureSystem
                and modelXbrl.modelManager.disclosureSystem.validateFileText):
            file, _encoding = ValidateFilingText.checkfile(modelXbrl, filepath)
        else:
            file, _encoding = modelXbrl.fileSource.file(filepath)
        _parser, _parserLookupName, _parserLookupClass = parser(
            modelXbrl, filepath)
        xmlDocument = etree.parse(file, parser=_parser, base_url=filepath)
        file.close()
    except (EnvironmentError,
            KeyError) as err:  # missing zip file raises KeyError
        if file:
            file.close()
        # retry in case of well known schema locations
        if not isIncluded and namespace and namespace in XbrlConst.standardNamespaceSchemaLocations and uri != XbrlConst.standardNamespaceSchemaLocations[
                namespace]:
            return load(modelXbrl,
                        XbrlConst.standardNamespaceSchemaLocations[namespace],
                        base, referringElement, isEntry, isDiscovered,
                        isIncluded, namespace, reloadCache)
        modelXbrl.error("IOerror",
                        _("%(fileName)s: file error: %(error)s"),
                        modelObject=referringElement,
                        fileName=os.path.basename(uri),
                        error=str(err))
        return None
    except (
            etree.LxmlError, ValueError
    ) as err:  # ValueError raised on bad format of qnames, xmlns'es, or parameters
        if file:
            file.close()
        if not isEntry and str(
                err) == "Start tag expected, '<' not found, line 1, column 1":
            return ModelDocument(modelXbrl, Type.UnknownNonXML, mappedUri,
                                 filepath, None)
        else:
            modelXbrl.error(
                "xmlSchema:syntax",
                _("%(error)s, %(fileName)s, %(sourceAction)s source element"),
                modelObject=referringElement,
                fileName=os.path.basename(uri),
                error=str(err),
                sourceAction=("including" if isIncluded else "importing"))
            return None

    # identify document
    #modelXbrl.modelManager.addToLog("discovery: {0}".format(
    #            os.path.basename(uri)))
    modelXbrl.modelManager.showStatus(_("loading {0}").format(uri))
    modelDocument = None

    rootNode = xmlDocument.getroot()
    if rootNode is not None:
        ln = rootNode.localName
        ns = rootNode.namespaceURI

        # type classification
        if ns == XbrlConst.xsd and ln == "schema":
            type = Type.SCHEMA
        elif ns == XbrlConst.link:
            if ln == "linkbase":
                type = Type.LINKBASE
            elif ln == "xbrl":
                type = Type.INSTANCE
        elif ns == XbrlConst.xbrli:
            if ln == "xbrl":
                type = Type.INSTANCE
        elif ns == XbrlConst.xhtml and \
             (ln == "html" or ln == "xhtml"):
            type = Type.UnknownXML
            if XbrlConst.ixbrl in rootNode.nsmap.values():
                type = Type.INLINEXBRL
        elif ln == "report" and ns == XbrlConst.ver:
            type = Type.VERSIONINGREPORT
        elif ln == "testcases" or ln == "documentation":
            type = Type.TESTCASESINDEX
        elif ln == "testcase":
            type = Type.TESTCASE
        elif ln == "registry" and ns == XbrlConst.registry:
            type = Type.REGISTRY
        elif ln == "rss":
            type = Type.RSSFEED
        elif ln == "ptvl":
            type = Type.ARCSINFOSET
        elif ln == "facts":
            type = Type.FACTDIMSINFOSET
        else:
            type = Type.UnknownXML
            nestedInline = None
            for htmlElt in rootNode.iter(
                    tag="{http://www.w3.org/1999/xhtml}html"):
                nestedInline = htmlElt
                break
            if nestedInline is None:
                for htmlElt in rootNode.iter(
                        tag="{http://www.w3.org/1999/xhtml}xhtml"):
                    nestedInline = htmlElt
                    break
            if nestedInline is not None:
                if XbrlConst.ixbrl in nestedInline.nsmap.values():
                    type = Type.INLINEXBRL
                    rootNode = nestedInline

        #create modelDocument object or subtype as identified
        if type == Type.VERSIONINGREPORT:
            from arelle.ModelVersReport import ModelVersReport
            modelDocument = ModelVersReport(modelXbrl, type, mappedUri,
                                            filepath, xmlDocument)
        elif type == Type.RSSFEED:
            from arelle.ModelRssObject import ModelRssObject
            modelDocument = ModelRssObject(modelXbrl, type, mappedUri,
                                           filepath, xmlDocument)
        else:
            modelDocument = ModelDocument(modelXbrl, type, mappedUri, filepath,
                                          xmlDocument)
        rootNode.init(modelDocument)
        modelDocument.parser = _parser  # needed for XmlUtil addChild's makeelement
        modelDocument.parserLookupName = _parserLookupName
        modelDocument.parserLookupClass = _parserLookupClass
        modelDocument.xmlRootElement = rootNode
        modelDocument.schemaLocationElements.add(rootNode)
        modelDocument.documentEncoding = _encoding

        if isEntry or isDiscovered:
            modelDocument.inDTS = True

        # discovery (parsing)
        if type == Type.SCHEMA:
            modelDocument.schemaDiscover(rootNode, isIncluded, namespace)
        elif type == Type.LINKBASE:
            modelDocument.linkbaseDiscover(rootNode)
        elif type == Type.INSTANCE:
            modelDocument.instanceDiscover(rootNode)
        elif type == Type.INLINEXBRL:
            modelDocument.inlineXbrlDiscover(rootNode)
        elif type == Type.VERSIONINGREPORT:
            modelDocument.versioningReportDiscover(rootNode)
        elif type == Type.TESTCASESINDEX:
            modelDocument.testcasesIndexDiscover(xmlDocument)
        elif type == Type.TESTCASE:
            modelDocument.testcaseDiscover(rootNode)
        elif type == Type.REGISTRY:
            modelDocument.registryDiscover(rootNode)
        elif type == Type.VERSIONINGREPORT:
            modelDocument.versioningReportDiscover(rootNode)
        elif type == Type.RSSFEED:
            modelDocument.rssFeedDiscover(rootNode)
    return modelDocument
Example #15
0
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, **kwargs):
    # check if big instance and has header with an initial incomplete tree walk (just 2 elements
    if not _streamingExtensionsCheck:
        return None
    
    # track whether modelXbrl has been validated by this streaming extension
    modelXbrl._streamingExtensionValidated = False
        
    def logSyntaxErrors(parsercontext):
        for error in parsercontext.error_log:
            modelXbrl.error("xmlSchema:syntax",
                    _("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"),
                    modelObject=modelXbrl, fileName=os.path.basename(filepath), 
                    error=error.message, line=error.line, column=error.column, sourceAction="streaming")
    #### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
    #### note: iterparse wants a binary file, but file is text mode
    _file, = modelXbrl.fileSource.file(filepath, binary=True)
    startedAt = time.time()
    modelXbrl.profileActivity()
    ''' this seems twice as slow as iterparse
    class instInfoTarget():
        def __init__(self, element_factory=None, parser=None):
            self.newTree = True
            self.streamingAspects = None
            self.foundInstance = False
            self.creationSoftwareComment = ''
            self.currentEltTag = "(before xbrli:xbrl)"
            self.numRootFacts = 0
        def start(self, tag, attrib, nsmap=None):
            if self.newTree:
                if tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    self.foundInstance = True
                    self.newTree = False
                else: # break 
                    raise NotInstanceDocumentException()
            elif not tag.startswith("{http://www.xbrl.org/"):
                self.numRootFacts += 1
                if self.numRootFacts % 1000 == 0:
                    modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
            self.currentEltTag = tag
        def end(self, tag):
            pass
        def data(self, data):
            pass
        def comment(self, text):
            if not self.foundInstance: # accumulate comments before xbrli:xbrl
                self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text
            elif not self.creationSoftwareComment:
                self.creationSoftwareComment = text # or first comment after xbrli:xbrl
        def pi(self, target, data):
            if target == "xbrl-streamable-instance":
                if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl":
                    self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results
                else:
                    modelXbrl.error("streamingExtensions:headerMisplaced",
                            _("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"),
                            modelObject=modelXbrl, target=target, element=self.currentEltTag)
        def close(self):
            if not self.creationSoftwareComment:
                self.creationSoftwareComment = None
            return True
    instInfo = instInfoTarget()
    infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo)
    try:
        etree.parse(_file, parser=infoParser, base_url=filepath)
    except NotInstanceDocumentException:
        pass
    '''
    foundErrors = False
    foundInstance = False
    streamingAspects = None
    creationSoftwareComment = None
    instInfoNumRootFacts = 0
    numElts = 0
    elt = None
    instInfoContext = etree.iterparse(_file, events=("start","end"), huge_tree=True)
    for event, elt in instInfoContext:
        if event == "start":
            if elt.getparent() is not None:
                if elt.getparent().tag == "{http://www.xbrl.org/2003/instance}xbrl":
                    if not foundInstance:
                        foundInstance = True
                        pi = precedingProcessingInstruction(elt, "xbrl-streamable-instance")
                        if pi is None:
                            break
                        else:
                            streamingAspects = dict(pi.attrib.copy())
                            if creationSoftwareComment is None:
                                creationSoftwareComment = precedingComment(elt)
                    if not elt.tag.startswith("{http://www.xbrl.org/"):
                        instInfoNumRootFacts += 1
                        if instInfoNumRootFacts % 1000 == 0:
                            modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
                elif not foundInstance:       
                    break
            elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl":
                creationSoftwareComment = precedingComment(elt)
                if precedingProcessingInstruction(elt, "xbrl-streamable-instance") is not None:
                    modelXbrl.error("streamingExtensions:headerMisplaced",
                            _("Header is misplaced: %(error)s, must follow xbrli:xbrl element"),
                            modelObject=elt)
        elif event == "end":
            elt.clear()
            numElts += 1
            if numElts % 1000 == 0 and elt.getparent() is not None:
                while elt.getprevious() is not None and elt.getparent() is not None:
                    del elt.getparent()[0]
    if elt is not None:
        elt.clear()
    
    _file.seek(0,io.SEEK_SET) # allow reparsing
    if not foundInstance or streamingAspects is None:
        del elt
        _file.close()
        return None
    modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
    startedAt = time.time()
    try:
        version = Decimal(streamingAspects.get("version"))
        if int(version) != 1:
            modelXbrl.error("streamingExtensions:unsupportedVersion",
                    _("Streaming version %(version)s, major version number must be 1"),
                    modelObject=elt, version=version)
            foundErrors = True
    except (InvalidOperation, OverflowError):
        modelXbrl.error("streamingExtensions:versionError",
                _("Version %(version)s, number must be 1.n"),
                modelObject=elt, version=streamingAspects.get("version", "(none)"))
        foundErrors = True
    for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
        try:
            bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
            if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
                raise InvalidOperation
            elif bufAspect == "contextBuffer":
                contextBufferLimit = bufLimit
            elif bufAspect == "unitBuffer":
                unitBufferLimit = bufLimit
            elif bufAspect == "footnoteBuffer":
                footnoteBufferLimit = bufLimit
        except InvalidOperation:
            modelXbrl.error("streamingExtensions:valueError",
                    _("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"),
                    modelObject=elt, attrib=bufAspect, value=streamingAspects.get(bufAspect))
            foundErrors = True
    if _streamingExtensionsValidate:
        incompatibleValidations = []
        _validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
        _disclosureSystem = modelXbrl.modelManager.disclosureSystem
        if _validateDisclosureSystem and _disclosureSystem.EFM:
            incompatibleValidations.append("EFM")
        if _validateDisclosureSystem and _disclosureSystem.GFM:
            incompatibleValidations.append("GFM")
        if _validateDisclosureSystem and _disclosureSystem.EBA:
            incompatibleValidations.append("EBA")
        if _validateDisclosureSystem and _disclosureSystem.HMRC:
            incompatibleValidations.append("EBA")
        if modelXbrl.modelManager.validateCalcLB:
            incompatibleValidations.append("calculation LB")
        if incompatibleValidations:            
            modelXbrl.error("streamingExtensions:incompatibleValidation",
                    _("Streaming instance validation does not support %(incompatibleValidations)s validation"),
                    modelObject=modelXbrl, incompatibleValidations=', '.join(incompatibleValidations))
            foundErrors = True
    if instInfoContext.error_log:
        foundErrors = True
    logSyntaxErrors(instInfoContext)
    del instInfoContext # dereference
    
    if foundErrors:
        _file.close()
        return None

    _encoding = XmlUtil.encoding(_file.read(512))
    _file.seek(0,io.SEEK_SET) # allow reparsing

    if _streamingExtensionsValidate:
        validator = Validate(modelXbrl)
        instValidator = validator.instValidator

    eltMdlObjs = {}
    contextBuffer = []
    unitBuffer = []
    footnoteBuffer = []
    factBuffer = []
    numFacts = 1
    
    class modelLoaderTarget():
        def __init__(self, element_factory=None, parser=None):
            self.newTree = True
            self.currentMdlObj = None
            self.beforeInstanceStream = True
            self.numRootFacts = 1
        def start(self, tag, attrib, nsmap=None):
            mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
            mdlObj.sourceline = 1
            if self.newTree:
                self.newTree = False
                self.currentMdlObj = mdlObj
                modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
                modelXbrl.modelDocument = modelDocument # needed for incremental validation
                mdlObj.init(modelDocument)
                modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement 
                modelDocument.parserLookupName = _parserLookupName
                modelDocument.parserLookupClass = _parserLookupClass
                modelDocument.xmlRootElement = mdlObj
                modelDocument.schemaLocationElements.add(mdlObj)
                modelDocument.documentEncoding = _encoding
                modelDocument._creationSoftwareComment = creationSoftwareComment
                modelXbrl.info("streamingExtensions:streaming",
                               _("Stream processing this instance."),
                               modelObject = modelDocument)    
            else:
                self.currentMdlObj.append(mdlObj)
                self.currentMdlObj = mdlObj
                mdlObj._init()
                ns = mdlObj.namespaceURI
                ln = mdlObj.localName
                if (self.beforeInstanceStream and (
                    (ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
                    (ns == XbrlConst.xbrli and ln in ("context", "unit")) or
                    (ns not in (XbrlConst.link, XbrlConst.xbrli)))):
                    self.beforeInstanceStream = False
                    if _streamingExtensionsValidate:
                        instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters())
                    else: # need default dimensions
                        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
            return mdlObj
        def end(self, tag):
            modelDocument = modelXbrl.modelDocument
            mdlObj = self.currentMdlObj
            parentMdlObj = mdlObj.getparent()
            self.currentMdlObj = parentMdlObj
            ns = mdlObj.namespaceURI
            ln = mdlObj.localName
            if ns == XbrlConst.xbrli:
                if ln == "context":
                    if mdlObj.get("sticky"):
                        del mdlObj.attrib["sticky"]
                        XmlValidate.validate(modelXbrl, mdlObj)
                        modelDocument.contextDiscover(mdlObj)
                    else:
                        if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
                            # drop before adding as dropped may have same id as added
                            cntx = contextBuffer.pop(0)
                            dropContext(modelXbrl, cntx)
                            del parentMdlObj[parentMdlObj.index(cntx)]
                            cntx = None
                        XmlValidate.validate(modelXbrl, mdlObj)
                        modelDocument.contextDiscover(mdlObj)
                        if contextBufferLimit.is_finite():
                            contextBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        contextsToCheck = (mdlObj,)
                        instValidator.checkContexts(contextsToCheck)
                        if modelXbrl.hasXDT:
                            instValidator.checkContextsDimensions(contextsToCheck)
                        del contextsToCheck # dereference
                elif ln == "unit":
                    if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
                        # drop before additing as dropped may have same id as added
                        unit = unitBuffer.pop(0)
                        dropUnit(modelXbrl, unit)
                        del parentMdlObj[parentMdlObj.index(unit)]
                        unit = None 
                    XmlValidate.validate(modelXbrl, mdlObj)
                    modelDocument.unitDiscover(mdlObj)
                    if unitBufferLimit.is_finite():
                        unitBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkUnits( (mdlObj,) )
                elif ln == "xbrl": # end of document
                    # check remaining footnote refs
                    for footnoteLink in footnoteBuffer:
                        checkFootnoteHrefs(modelXbrl, footnoteLink)
            elif ns == XbrlConst.link:
                if ln == "footnoteLink":
                    XmlValidate.validate(modelXbrl, mdlObj)
                    footnoteLinks = (mdlObj,)
                    modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
                    if footnoteBufferLimit.is_finite():
                        footnoteBuffer.append(mdlObj)
                    if _streamingExtensionsValidate:
                        instValidator.checkLinks(footnoteLinks)
                        if len(footnoteBuffer) > footnoteBufferLimit:
                            # check that hrefObjects for locators were all satisfied
                                # drop before addition as dropped may have same id as added
                            footnoteLink = footnoteBuffer.pop(0)
                            checkFootnoteHrefs(modelXbrl, footnoteLink)
                            dropFootnoteLink(modelXbrl, footnoteLink)
                            del parentMdlObj[parentMdlObj.index(footnoteLink)]
                            footnoteLink = None
                    footnoteLinks = None
                elif ln in ("schemaRef", "linkbaseRef"):
                    modelDocument.discoverHref(mdlObj)
                elif not modelXbrl.skipDTS:
                    if ln in ("roleRef", "arcroleRef"):
                        modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
            elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
                self.numRootFacts += 1
                XmlValidate.validate(modelXbrl, mdlObj)
                modelDocument.factDiscover(mdlObj, modelXbrl.facts)
                if _streamingExtensionsValidate:
                    factsToCheck = (mdlObj,)
                    instValidator.checkFacts(factsToCheck)
                    if modelXbrl.hasXDT:
                        instValidator.checkFactsDimensions(factsToCheck)
                    del factsToCheck
                    dropFact(modelXbrl, mdlObj, modelXbrl.facts)
                    del parentMdlObj[parentMdlObj.index(mdlObj)]
                if self.numRootFacts % 1000 == 0:
                    modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts, 
                                                                                              100.0 * self.numRootFacts / instInfoNumRootFacts), 
                                              minTimeToShow=20.0)
            return mdlObj
        def data(self, data):
            self.currentMdlObj.text = data
        def comment(self, text):
            pass
        def pi(self, target, data):
            pass
        def close(self):
            return None
        
    _parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget())
    etree.parse(_file, parser=_parser, base_url=filepath)
    logSyntaxErrors(_parser)
    _file.close()
    if _streamingExtensionsValidate and validator is not None:
        del instValidator
        validator.close()
        # track that modelXbrl has been validated by this streaming extension
        modelXbrl._streamingExtensionValidated = True
        
    modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
    return modelXbrl.modelDocument