예제 #1
0
 def factAspects(fact):
     aspects = {qnOimConceptAspect: oimValue(fact.qname)}
     if hasId and fact.id:
         aspects[qnOimIdAspect] = fact.id
     if hasLocation:
         aspects[qnOimLocationAspect] = elementChildSequence(fact)
     concept = fact.concept
     if concept is not None:
         if concept.baseXbrliType in ("string", "normalizedString",
                                      "token") and fact.xmlLang:
             aspects[qnOimLangAspect] = fact.xmlLang
     aspects[qnOimTypeAspect] = concept.baseXbrliType
     if fact.isItem:
         aspects[qnOimValueAspect] = (NILVALUE if fact.isNil else oimValue(
             fact.xValue, inferredDecimals(fact)))
     cntx = fact.context
     if cntx is not None:
         if cntx.entityIdentifierElement is not None:
             aspects[qnOimEntityAspect] = oimValue(
                 qname(*cntx.entityIdentifier))
         if cntx.period is not None:
             aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
         for dim in cntx.qnameDims.values():
             aspects[dim.dimensionQname] = (oimValue(dim.memberQname)
                                            if dim.isExplicit else
                                            dim.typedMember.stringValue)
     unit = fact.unit
     if unit is not None:
         _mMul, _mDiv = unit.measures
         if isJSON:
             aspects[
                 qnOimUnitAspect] = (  # use tuple instead of list for hashability
                     tuple(
                         oimValue(m)
                         for m in sorted(_mMul, key=lambda m: str(m))),
                     tuple(
                         oimValue(m)
                         for m in sorted(_mDiv, key=lambda m: str(m))))
         else:  # CSV
             if _mMul:
                 aspects[qnOimUnitMulAspect] = ",".join(
                     oimValue(m)
                     for m in sorted(_mMul, key=lambda m: str(m)))
             if _mDiv:
                 aspects[qnOimUnitDivAspect] = ",".join(
                     oimValue(m)
                     for m in sorted(_mDiv, key=lambda m: str(m)))
     return aspects
예제 #2
0
 def factAspects(fact):
     aspects = {qnOimConceptAspect: oimValue(fact.qname)}
     if hasId and fact.id:
         aspects[qnOimIdAspect] = fact.id
     if hasLocation:
         aspects[qnOimLocationAspect] = elementChildSequence(fact)
     concept = fact.concept
     if concept is not None:
         if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
             aspects[qnOimLangAspect] = fact.xmlLang
     aspects[qnOimTypeAspect] = concept.baseXbrliType
     if fact.isItem:
         aspects[qnOimValueAspect] = (NILVALUE if fact.isNil else
                                      oimValue(fact.xValue, inferredDecimals(fact)))
     cntx = fact.context
     if cntx is not None:
         if cntx.entityIdentifierElement is not None:
             aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier))
         if cntx.period is not None:
             aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
         for dim in cntx.qnameDims.values():
             aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit
                                            else dim.typedMember.stringValue)
     unit = fact.unit
     if unit is not None:
         _mMul, _mDiv = unit.measures
         if isJSON:
             aspects[qnOimUnitAspect] = ( # use tuple instead of list for hashability
                 tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))),
                 tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m))))
         else: # CSV
             if _mMul:
                 aspects[qnOimUnitMulAspect] = ",".join(oimValue(m)
                                                     for m in sorted(_mMul, key=lambda m: str(m)))
             if _mDiv:
                 aspects[qnOimUnitDivAspect] = ",".join(oimValue(m)
                                                     for m in sorted(_mDiv, key=lambda m: str(m)))
     return aspects
예제 #3
0
def saveTargetDocument(modelXbrl,
                       targetDocumentFilename,
                       targetDocumentSchemaRefs,
                       outputZip=None,
                       filingFiles=None,
                       *args,
                       **kwargs):
    targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
        targetDocumentFilename, modelXbrl.modelDocument.filepath)

    def addLocallyReferencedFile(elt, filingFiles):
        if elt.tag in ("a", "img"):
            for attrTag, attrValue in elt.items():
                if attrTag in ("href", "src") and not isHttpUrl(
                        attrValue) and not os.path.isabs(attrValue):
                    attrValue = attrValue.partition('#')[0]  # remove anchor
                    if attrValue:  # ignore anchor references to base document
                        attrValue = os.path.normpath(
                            attrValue
                        )  # change url path separators to host separators
                        file = os.path.join(sourceDir, attrValue)
                        if modelXbrl.fileSource.isInArchive(
                                file,
                                checkExistence=True) or os.path.exists(file):
                            filingFiles.add(file)

    targetUrlParts = targetUrl.rpartition(".")
    targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
    modelXbrl.modelManager.showStatus(
        _("Extracting instance ") + os.path.basename(targetUrl))
    rootElt = modelXbrl.modelDocument.xmlRootElement
    # take baseXmlLang from <html> or <base>
    baseXmlLang = rootElt.get(
        "{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang")
    for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants(
            tag="{http://www.w3.org/1999/xhtml}body"):
        baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang"
                                ) or rootElt.get("lang") or baseXmlLang
    targetInstance = ModelXbrl.create(
        modelXbrl.modelManager,
        newDocumentType=Type.INSTANCE,
        url=targetUrl,
        schemaRefs=targetDocumentSchemaRefs,
        isEntry=True,
        discover=False)  # don't attempt to load DTS
    if baseXmlLang:
        targetInstance.modelDocument.xmlRootElement.set(
            "{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
    ValidateXbrlDimensions.loadDimensionDefaults(
        targetInstance)  # need dimension defaults
    # roleRef and arcroleRef (of each inline document)
    for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
        for roleRefElt in sourceRefs.values():
            addChild(targetInstance.modelDocument.xmlRootElement,
                     roleRefElt.qname,
                     attributes=roleRefElt.items())

    # contexts
    for context in sorted(modelXbrl.contexts.values(),
                          key=lambda c: elementChildSequence(c)):
        ignore = targetInstance.createContext(
            context.entityIdentifier[0],
            context.entityIdentifier[1],
            'instant' if context.isInstantPeriod else
            'duration' if context.isStartEndPeriod else 'forever',
            context.startDatetime,
            context.endDatetime,
            None,
            context.qnameDims, [], [],
            id=context.id)
    for unit in modelXbrl.units.values():
        measures = unit.measures
        ignore = targetInstance.createUnit(measures[0],
                                           measures[1],
                                           id=unit.id)

    modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
    newFactForOldObjId = {}

    def createFacts(facts, parent):
        for fact in facts:
            if fact.isItem:  # HF does not de-duplicate, which is currently-desired behavior
                attrs = {"contextRef": fact.contextID}
                if fact.id:
                    attrs["id"] = fact.id
                if fact.isNumeric:
                    attrs["unitRef"] = fact.unitID
                    if fact.get("decimals"):
                        attrs["decimals"] = fact.get("decimals")
                    if fact.get("precision"):
                        attrs["precision"] = fact.get("precision")
                if fact.isNil:
                    attrs[XbrlConst.qnXsiNil] = "true"
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.textValue
                    if fact.concept is not None and fact.concept.baseXsdType in (
                            "string", "normalizedString"):  # default
                        xmlLang = fact.xmlLang
                        if xmlLang is not None and xmlLang != baseXmlLang:
                            attrs[
                                "{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang
                newFact = targetInstance.createFact(fact.qname,
                                                    attributes=attrs,
                                                    text=text,
                                                    parent=parent)
                # if fact.isFraction, create numerator and denominator
                newFactForOldObjId[fact.objectIndex] = newFact
                if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
                    # check for img and other filing references so that referenced files are included in the zip.
                    for xmltext in [text] + CDATApattern.findall(text):
                        try:
                            for elt in XML("<body>\n{0}\n</body>\n".format(
                                    xmltext)).iter():
                                addLocallyReferencedFile(elt, filingFiles)
                        except (XMLSyntaxError, UnicodeDecodeError):
                            pass  # TODO: Why ignore UnicodeDecodeError?
            elif fact.isTuple:
                newTuple = targetInstance.createFact(fact.qname, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newTuple
                createFacts(fact.modelTupleFacts, newTuple)

    createFacts(modelXbrl.facts, None)
    modelXbrl.modelManager.showStatus(
        _("Creating and validating footnotes and relationships"))
    HREF = "{http://www.w3.org/1999/xlink}href"
    footnoteLinks = defaultdict(list)
    footnoteIdCount = {}
    for linkKey, linkPrototypes in modelXbrl.baseSets.items():
        arcrole, linkrole, linkqname, arcqname = linkKey
        if (linkrole and linkqname and arcqname and  # fully specified roles
                arcrole != "XBRL-footnotes" and any(
                    lP.modelDocument.type == Type.INLINEXBRL
                    for lP in linkPrototypes)):
            for linkPrototype in linkPrototypes:
                if linkPrototype not in footnoteLinks[linkrole]:
                    footnoteLinks[linkrole].append(linkPrototype)
    for linkrole in sorted(footnoteLinks.keys()):
        for linkPrototype in footnoteLinks[linkrole]:
            newLink = addChild(targetInstance.modelDocument.xmlRootElement,
                               linkPrototype.qname,
                               attributes=linkPrototype.attributes)
            for linkChild in linkPrototype:
                attributes = linkChild.attributes
                if isinstance(linkChild, LocPrototype):
                    if HREF not in linkChild.attributes:
                        linkChild.attributes[HREF] = \
                        "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
                    addChild(newLink, linkChild.qname, attributes=attributes)
                elif isinstance(linkChild, ArcPrototype):
                    addChild(newLink, linkChild.qname, attributes=attributes)
                elif isinstance(linkChild, ModelInlineFootnote):
                    idUseCount = footnoteIdCount.get(linkChild.footnoteID,
                                                     0) + 1
                    if idUseCount > 1:  # if footnote with id in other links bump the id number
                        attributes = linkChild.attributes.copy()
                        attributes["id"] = "{}_{}".format(
                            attributes["id"], idUseCount)
                    footnoteIdCount[linkChild.footnoteID] = idUseCount
                    newChild = addChild(newLink,
                                        linkChild.qname,
                                        attributes=attributes)
                    xmlLang = linkChild.xmlLang
                    if xmlLang is not None and xmlLang != baseXmlLang:  # default
                        newChild.set(
                            "{http://www.w3.org/XML/1998/namespace}lang",
                            xmlLang)
                    copyIxFootnoteHtml(
                        linkChild,
                        newChild,
                        targetModelDocument=targetInstance.modelDocument,
                        withText=True)

                    if filingFiles and linkChild.textValue:
                        footnoteHtml = XML("<body/>")
                        copyIxFootnoteHtml(linkChild, footnoteHtml)
                        for elt in footnoteHtml.iter():
                            addLocallyReferencedFile(elt, filingFiles)
    targetInstance.saveInstance(overrideFilepath=targetUrl,
                                outputZip=outputZip)
    if getattr(modelXbrl, "isTestcaseVariation", False):
        modelXbrl.extractedInlineInstance = True  # for validation comparison
    modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
예제 #4
0
def saveLoadableOIM(modelXbrl, oimFile, oimStyle, oimQNameSeparator):
    
    isJSON = oimFile.endswith(".json")
    isCSV = oimFile.endswith(".csv")

    namespacePrefixes = {}
    def compileQname(qname):
        if qname.namespaceURI not in namespacePrefixes:
            namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
            
    aspectsDefined = {
        qnOimConceptAspect,
        qnOimLocationAspect,
        qnOimValueAspect,
        qnOimPeriodAspect,
        qnOimEntityAspect}

            
    def oimValue(object, decimals=None):
        if isinstance(object, QName):
            if oimQNameSeparator == "clark":
                return object.clarkNotation;
            if object.namespaceURI not in namespacePrefixes:
                if object.prefix:
                    namespacePrefixes[object.namespaceURI] = object.prefix
                else:
                    _prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_")))
                    namespacePrefixes[object.namespaceURI] = _prefix
            return "{}{}{}".format(namespacePrefixes[object.namespaceURI], 
                                   oimQNameSeparator, 
                                   object.localName)
        if isinstance(object, Decimal):
            try:
                if decimals is not None and not isnan(decimals) and not isinf(decimals):
                    if decimals != 0:
                        object = object / (TEN ** -decimals)
                    return "{}e{}".format(object, -decimals)
                else:
                    return "{}".format(object) # force to string to prevent json floating error
            except:
                return str(object)
        if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time,
                               gYearMonth, gMonthDay, gYear, gMonth, gDay)):
            return str(object)
        return object
    
    def oimPeriodValue(cntx):
        if cntx.isForeverPeriod:
            return "forever"
        elif cntx.isStartEndPeriod:
            return "{}/{}".format(dateunionValue(cntx.startDatetime, dateOnlyHour=0), 
                                  dateunionValue(cntx.endDatetime, subtractOneDay=True, dateOnlyHour=24))
        else: # instant
            return "PT0S/{}".format(dateunionValue(cntx.endDatetime, subtractOneDay=True, dateOnlyHour=24))
              
    hasId = False
    hasLocation = False # may be optional based on style?
    hasType = True
    hasLang = False
    hasUnits = False      
    hasUnitMulMeasures = False
    hasUnitDivMeasures = False
    hasTuple = False
    
    #compile QNames in instance for OIM
    for fact in modelXbrl.factsInInstance:
        if fact.id:
            hasId = True
        concept = fact.concept
        if concept is not None:
            if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
                hasLang = True
        compileQname(fact.qname)
        if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
            compileQname(fact.xValue)
        unit = fact.unit
        if unit is not None:
            hasUnits = True
            if unit.measures[0]:
                hasUnitMulMeasures = True
            if unit.measures[1]:
                hasUnitDivMeasures = True
        if fact.modelTupleFacts:
            hasTuple = True
            
    entitySchemePrefixes = {}
    for cntx in modelXbrl.contexts.values():
        if cntx.entityIdentifierElement is not None:
            scheme = cntx.entityIdentifier[0]
            if scheme not in entitySchemePrefixes:
                if not entitySchemePrefixes: # first one is just scheme
                    if scheme == "http://www.sec.gov/CIK":
                        _schemePrefix = "cik"
                    elif scheme == "http://standard.iso.org/iso/17442":
                        _schemePrefix = "lei"
                    else:
                        _schemePrefix = "scheme"
                else:
                    _schemePrefix = "scheme-{}".format(len(entitySchemePrefixes) + 1)
                entitySchemePrefixes[scheme] = _schemePrefix
                namespacePrefixes[scheme] = _schemePrefix
        for dim in cntx.qnameDims.values():
            compileQname(dim.dimensionQname)
            aspectsDefined.add(dim.dimensionQname)
            if dim.isExplicit:
                compileQname(dim.memberQname)
                
    for unit in modelXbrl.units.values():
        if unit is not None:
            for measures in unit.measures:
                for measure in measures:
                    compileQname(measure)
                    
    if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli":
        namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix

    if hasId: aspectsDefined.add(qnOimIdAspect)
    if hasLang: aspectsDefined.add(qnOimLangAspect)
    if hasTuple: aspectsDefined.add(qnOimTupleAspect)
    if hasUnits: aspectsDefined.add(qnOimUnitAspect)
    if hasUnitMulMeasures: aspectsDefined.add(qnOimUnitMulAspect)
    if hasUnitDivMeasures: aspectsDefined.add(qnOimUnitDivAspect)
                    
    # compile footnotes and relationships
    factRelationships = []
    factFootnotes = []
    for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
        oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
        factRelationships.append(oimRel)
        oimRel["fromIds"] = [obj.id if obj.id 
                             else elementChildSequence(obj)
                             for obj in rel.fromModelObjects]
        oimRel["toIds"] = [obj.id if obj.id
                           else elementChildSequence(obj)
                           for obj in rel.toModelObjects]
        _order = rel.arcElement.get("order")
        if _order is not None:
            oimRel["order"] = _order
        for obj in rel.toModelObjects:
            if isinstance(obj, ModelResource): # footnote
                oimFootnote = {"role": obj.role,
                               "id": obj.id if obj.id
                                     else elementChildSequence(obj),
                                # value needs work for html elements and for inline footnotes
                               "value": xmlstring(obj, stripXmlns=True)}
                if obj.xmlLang:
                    oimFootnote["lang"] = obj.xmlLang
                factFootnotes.append(oimFootnote)
                oimFootnote
            
    dtsReferences = [
        {"type": "schema" if doc.type == ModelDocument.Type.SCHEMA
                 else "linkbase" if doc.type == ModelDocument.Type.LINKBASE
                 else "other",
         "href": doc.basename}
        for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
        if ref.referringModelObject.qname in SCHEMA_LB_REFS]
        
    roleTypes = [
        {"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef",
         "href": ref.referringModelObject["href"]}
        for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
        if ref.referringModelObject.qname in ROLE_REFS]
    

    def factAspects(fact):
        aspects = {qnOimConceptAspect: oimValue(fact.qname)}
        if hasId and fact.id:
            aspects[qnOimIdAspect] = fact.id
        if hasLocation:
            aspects[qnOimLocationAspect] = elementChildSequence(fact)
        concept = fact.concept
        if concept is not None:
            if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
                aspects[qnOimLangAspect] = fact.xmlLang
        aspects[qnOimTypeAspect] = concept.baseXbrliType
        if fact.isItem:
            aspects[qnOimValueAspect] = (NILVALUE if fact.isNil else
                                         oimValue(fact.xValue, inferredDecimals(fact)))
        cntx = fact.context
        if cntx is not None:
            if cntx.entityIdentifierElement is not None:
                aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier))
            if cntx.period is not None:
                aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
            for dim in cntx.qnameDims.values():
                aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit
                                               else dim.typedMember.stringValue)
        unit = fact.unit
        if unit is not None:
            _mMul, _mDiv = unit.measures
            if isJSON:
                aspects[qnOimUnitAspect] = ( # use tuple instead of list for hashability
                    tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))),
                    tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m))))
            else: # CSV
                if _mMul:
                    aspects[qnOimUnitMulAspect] = ",".join(oimValue(m)
                                                        for m in sorted(_mMul, key=lambda m: str(m)))
                if _mDiv:
                    aspects[qnOimUnitDivAspect] = ",".join(oimValue(m)
                                                        for m in sorted(_mDiv, key=lambda m: str(m)))
        return aspects
    
    if isJSON:
        # save JSON
        
        oim = {} # top level of oim json output
            
        oimFacts = []
        oimReport = []
        oimReport.append({"url": modelXbrl.modelDocument.uri})
        if oimQNameSeparator != "clark":
            oimReport.append({"prefixMap": dict((p,ns) for ns,p in namespacePrefixes.items())})
        oimReport.append({"DTSreferences": dtsReferences})
        oimReport.append({"roleTypes": roleTypes})
        oimReport.append({"facts": oimFacts})
        oimReport.append({"footnotes": factFootnotes})
        oimReport.append({"relationships": factRelationships})

        if oimStyle == "flat":
            
            def saveFlatJsonFacts(facts, oimFacts):
                for fact in facts:
                    oimFact = factAspects(fact)
                    if fact.modelTupleFacts:
                        tupleFacts = []
                        oimFact[qnOimTupleAspect] = tupleFacts
                        saveFlatJsonFacts(fact.modelTupleFacts, tupleFacts)
                    oimFacts.append(dict((oimValue(k),v) for k,v in oimFact.items()))
                    
            saveFlatJsonFacts(modelXbrl.facts, oimFacts)
        
        elif oimStyle == "clustered":
            
            # build aspect-value usage per fact for every fact
            categoricalAspectValueSets = {} # for each aspect, value facts-set
            aspectIndex = {}
            indexAspect = {}
            def addCategoricalAspect(aspectQn):
                i = len(aspectIndex)
                aspectIndex[aspectQn] = i 
                indexAspect[i] = oimValue(aspectQn)
                categoricalAspectValueSets[i] = defaultdict(set)

            addCategoricalAspect(qnOimConceptAspect)
            addCategoricalAspect(qnOimEntityAspect)
            addCategoricalAspect(qnOimPeriodAspect)
            for aspectQn in aspectsDefined:
                if aspectQn.namespaceURI != nsOim or aspectQn in (
                    qnOimIdAspect, qnOimLangAspect, qnOimUnitAspect):
                    addCategoricalAspect(aspectQn) 
            
            
            for fact in modelXbrl.facts:
                fact._factAspectValues = {}
                fact._factAspectSet = set()
                for aspectQn, value in factAspects(fact).items():
                    if aspectQn in aspectIndex:
                        i = aspectIndex[aspectQn]
                        v = oimValue(value)
                        categoricalAspectValueSets[i][v].add(fact)
                        fact._factAspectValues[i] = v
                        fact._factAspectSet.add(i)
                        
            # order aspectValues by largest population
            maxAspectValuePopulation = [(aspectIndex, max(len(factSet) for factSet in oimValueFacts.values()))
                                        for aspectIndex, oimValueFacts in categoricalAspectValueSets.items()]
                        
            maxAspectValuePopulation.sort(key=lambda ai_max: -ai_max[1])
            
            factsClustered = set()
            _aspectValue = {}
                        
            def clusterAspect(_avpi, _data):
                if _avpi >= len(maxAspectValuePopulation): 
                    return # end of aspects
                _ai = maxAspectValuePopulation[_avpi][0]
                for _v, _vFactsSet in categoricalAspectValueSets[_ai].items():
                    _aspectValue[_ai] = _v
                    _nestedData = []
                    _nestedAspect = {indexAspect[_ai]: _v, "data": _nestedData}
                    for _fact in _vFactsSet - factsClustered:
                        if (_fact._factAspectSet == _aspectValue.keys() and
                            all([_fact._factAspectValues[__ai] == _aspectValue[__ai]
                                for __ai in _aspectValue])):
                            _factAspects = factAspects(_fact)
                            _oimFactItem = {oimValue(qnOimValueAspect): _factAspects[qnOimValueAspect]}
                            if hasLocation:
                                _oimFactItem[oimValue(qnOimLocationAspect)] = _factAspects[qnOimLocationAspect]
                            if hasType:
                                _oimFactItem[oimValue(qnOimTypeAspect)] = _factAspects[qnOimTypeAspect]
                            _nestedData.append(_oimFactItem)
                            factsClustered.add(_fact)
                    clusterAspect(_avpi+1, _nestedData)
                    if _nestedData:
                        _data.append(_nestedAspect)
                    del _aspectValue[_ai]
            clusterAspect(0, oimFacts)
            
        with open(oimFile, "w", encoding="utf-8") as fh:
            fh.write(json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=True))

            
        
    elif isCSV:
        # save CSV
        
        # levels of tuple nesting
        def tupleDepth(facts, parentDepth):
            _levelDepth = parentDepth
            for fact in facts:
                _factDepth = tupleDepth(fact.modelTupleFacts, parentDepth + 1)
                if _factDepth > _levelDepth:
                    _levelDepth = _factDepth
            return _levelDepth
        maxDepth = tupleDepth(modelXbrl.facts, 0)
        
        aspectQnCol = {oimValue(qnOimConceptAspect): maxDepth - 1}
        aspectsHeader = [oimValue(qnOimConceptAspect)]
        
        for i in range(maxDepth - 1):
            aspectsHeader.append(None)
        
        def addAspectQnCol(aspectQn):
            aspectQnCol[aspectQn] = len(aspectsHeader)
            aspectsHeader.append(oimValue(aspectQn))
            
        # pre-ordered aspect columns
        if hasId:
            addAspectQnCol(qnOimIdAspect)
        if hasLocation:
            addAspectQnCol(qnOimLocationAspect)
        if hasType:
            addAspectQnCol(qnOimTypeAspect)
        addAspectQnCol(qnOimValueAspect)
        if qnOimEntityAspect in aspectsDefined:
            addAspectQnCol(qnOimEntityAspect)
        if qnOimPeriodAspect in aspectsDefined:
            addAspectQnCol(qnOimPeriodAspect)
        if qnOimUnitMulAspect in aspectsDefined:
            addAspectQnCol(qnOimUnitMulAspect)
        if qnOimUnitDivAspect in aspectsDefined:
            addAspectQnCol(qnOimUnitDivAspect)
        for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
            if aspectQn.namespaceURI != nsOim:
                addAspectQnCol(aspectQn) 
        
        def aspectCols(fact, depth):
            cols = [None for i in range(len(aspectsHeader))]
            for aspectQn, aspectValue in factAspects(fact).items():
                if aspectQn == qnOimConceptAspect:
                    cols[depth - 1] = aspectValue
                elif aspectQn in aspectQnCol:
                    cols[aspectQnCol[aspectQn]] = aspectValue
            return cols
        
        # save facts
        csvFile = open(oimFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
        csvWriter = csv.writer(csvFile, dialect="excel")
        csvWriter.writerow(aspectsHeader)
        
        def saveCSVfacts(facts, thisDepth):
            for fact in facts:
                csvWriter.writerow(aspectCols(fact, thisDepth))
                saveCSVfacts(fact.modelTupleFacts, thisDepth + 1)
                
        saveCSVfacts(modelXbrl.facts, 1)
        csvFile.close()
        
        # save namespaces
        if oimQNameSeparator == "clark":
            csvFile = open(oimFile.replace(".csv", "-prefixMap.csv"), csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
            csvWriter = csv.writer(csvFile, dialect="excel")
            csvWriter.writerow(("prefix", "mappedURI"))
            for namespaceURI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]):
                csvWriter.writerow((prefix, namespaceURI))
            csvFile.close()
        
        # save dts references
        csvFile = open(oimFile.replace(".csv", "-dts.csv"), csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
        csvWriter = csv.writer(csvFile, dialect="excel")
        csvWriter.writerow(("type", "href"))
        for oimRef in dtsReferences:
            csvWriter.writerow((oimRef["type"], oimRef["href"]))
        csvFile.close()
        
        # save role and arc type references
        if roleTypes:
            csvFile = open(oimFile.replace(".csv", "-roleTypes.csv"), csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
            csvWriter = csv.writer(csvFile, dialect="excel")
            csvWriter.writerow(("type", "href"))
            for oimRef in roleTypes:
                csvWriter.writerow((oimRef["type"], oimRef["href"]))
            csvFile.close()
        
        # save relationships
        csvFile = open(oimFile.replace(".csv", "-relationships.csv"), csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
        csvWriter = csv.writer(csvFile, dialect="excel")
        hasOrder = any(hasattribute(imRel,"order") for oimRel in factRelationships)
        csvWriter.writerow(("fromIds", "toIds", "linkrole", "arcrole") + 
                           (("order",) if hasOrder else ()))
        for oimRel in factRelationships:
            csvWriter.writerow((",".join(oimRel["fromIds"]),
                                ",".join(oimRel["toIds"]),
                                oimRel["linkrole"],
                                oimRel["arcrole"]) +
                               ((oimRel.get("order",None),) if hasOrder else ()))
        csvFile.close()
        
        # save footnotes
        csvFile = open(oimFile.replace(".csv", "-footnotes.csv"), csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
        csvWriter = csv.writer(csvFile, dialect="excel")
        hasLang = any(hasattribute(oimFnt,"lang") for oimFnt in factFootnotes)
        csvWriter.writerow(("id", "role") + (("lang",) if hasLang else ()) + ("value",))
        for oimFnt in factFootnotes:
            csvWriter.writerow((oimFtn["id"], oimFtn["role"]) +
                               ((oimFtn.get("lang",None),) if hasLang else ()) +
                               (oimFtn["value"],))
        csvFile.close()
예제 #5
0
def saveLoadableOIM(modelXbrl, oimFile, oimStyle, oimQNameSeparator):

    isJSON = oimFile.endswith(".json")
    isCSV = oimFile.endswith(".csv")

    namespacePrefixes = {}

    def compileQname(qname):
        if qname.namespaceURI not in namespacePrefixes:
            namespacePrefixes[qname.namespaceURI] = qname.prefix or ""

    aspectsDefined = {
        qnOimConceptAspect, qnOimLocationAspect, qnOimValueAspect,
        qnOimPeriodAspect, qnOimEntityAspect
    }

    def oimValue(object, decimals=None):
        if isinstance(object, QName):
            if oimQNameSeparator == "clark":
                return object.clarkNotation
            if object.namespaceURI not in namespacePrefixes:
                if object.prefix:
                    namespacePrefixes[object.namespaceURI] = object.prefix
                else:
                    _prefix = "_{}".format(
                        sum(1 for p in namespacePrefixes if p.startswith("_")))
                    namespacePrefixes[object.namespaceURI] = _prefix
            return "{}{}{}".format(namespacePrefixes[object.namespaceURI],
                                   oimQNameSeparator, object.localName)
        if isinstance(object, Decimal):
            try:
                if decimals is not None and not isnan(decimals) and not isinf(
                        decimals):
                    if decimals != 0:
                        object = object / (TEN**-decimals)
                    return "{}e{}".format(object, -decimals)
                else:
                    return "{}".format(
                        object
                    )  # force to string to prevent json floating error
            except:
                return str(object)
        if isinstance(object,
                      (DateTime, YearMonthDuration, DayTimeDuration, Time,
                       gYearMonth, gMonthDay, gYear, gMonth, gDay)):
            return str(object)
        return object

    def oimPeriodValue(cntx):
        if cntx.isForeverPeriod:
            return "forever"
        elif cntx.isStartEndPeriod:
            return "{}/{}".format(
                dateunionValue(cntx.startDatetime, dateOnlyHour=0),
                dateunionValue(cntx.endDatetime,
                               subtractOneDay=True,
                               dateOnlyHour=24))
        else:  # instant
            return "PT0S/{}".format(
                dateunionValue(cntx.endDatetime,
                               subtractOneDay=True,
                               dateOnlyHour=24))

    hasId = False
    hasLocation = False  # may be optional based on style?
    hasType = True
    hasLang = False
    hasUnits = False
    hasUnitMulMeasures = False
    hasUnitDivMeasures = False
    hasTuple = False

    #compile QNames in instance for OIM
    for fact in modelXbrl.factsInInstance:
        if fact.id:
            hasId = True
        concept = fact.concept
        if concept is not None:
            if concept.baseXbrliType in ("string", "normalizedString",
                                         "token") and fact.xmlLang:
                hasLang = True
        compileQname(fact.qname)
        if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
            compileQname(fact.xValue)
        unit = fact.unit
        if unit is not None:
            hasUnits = True
            if unit.measures[0]:
                hasUnitMulMeasures = True
            if unit.measures[1]:
                hasUnitDivMeasures = True
        if fact.modelTupleFacts:
            hasTuple = True

    entitySchemePrefixes = {}
    for cntx in modelXbrl.contexts.values():
        if cntx.entityIdentifierElement is not None:
            scheme = cntx.entityIdentifier[0]
            if scheme not in entitySchemePrefixes:
                if not entitySchemePrefixes:  # first one is just scheme
                    if scheme == "http://www.sec.gov/CIK":
                        _schemePrefix = "cik"
                    elif scheme == "http://standard.iso.org/iso/17442":
                        _schemePrefix = "lei"
                    else:
                        _schemePrefix = "scheme"
                else:
                    _schemePrefix = "scheme-{}".format(
                        len(entitySchemePrefixes) + 1)
                entitySchemePrefixes[scheme] = _schemePrefix
                namespacePrefixes[scheme] = _schemePrefix
        for dim in cntx.qnameDims.values():
            compileQname(dim.dimensionQname)
            aspectsDefined.add(dim.dimensionQname)
            if dim.isExplicit:
                compileQname(dim.memberQname)

    for unit in modelXbrl.units.values():
        if unit is not None:
            for measures in unit.measures:
                for measure in measures:
                    compileQname(measure)

    if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[
            XbrlConst.xbrli] != "xbrli":
        namespacePrefixes[XbrlConst.xbrli] = "xbrli"  # normalize xbrli prefix

    if hasId: aspectsDefined.add(qnOimIdAspect)
    if hasLang: aspectsDefined.add(qnOimLangAspect)
    if hasTuple: aspectsDefined.add(qnOimTupleAspect)
    if hasUnits: aspectsDefined.add(qnOimUnitAspect)
    if hasUnitMulMeasures: aspectsDefined.add(qnOimUnitMulAspect)
    if hasUnitDivMeasures: aspectsDefined.add(qnOimUnitDivAspect)

    # compile footnotes and relationships
    factRelationships = []
    factFootnotes = []
    for rel in modelXbrl.relationshipSet(modelXbrl,
                                         "XBRL-footnotes").modelRelationships:
        oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
        factRelationships.append(oimRel)
        oimRel["fromIds"] = [
            obj.id if obj.id else elementChildSequence(obj)
            for obj in rel.fromModelObjects
        ]
        oimRel["toIds"] = [
            obj.id if obj.id else elementChildSequence(obj)
            for obj in rel.toModelObjects
        ]
        _order = rel.arcElement.get("order")
        if _order is not None:
            oimRel["order"] = _order
        for obj in rel.toModelObjects:
            if isinstance(obj, ModelResource):  # footnote
                oimFootnote = {
                    "role": obj.role,
                    "id": obj.id if obj.id else elementChildSequence(obj),
                    # value needs work for html elements and for inline footnotes
                    "value": xmlstring(obj, stripXmlns=True)
                }
                if obj.xmlLang:
                    oimFootnote["lang"] = obj.xmlLang
                factFootnotes.append(oimFootnote)
                oimFootnote

    dtsReferences = [{
        "type":
        "schema" if doc.type == ModelDocument.Type.SCHEMA else
        "linkbase" if doc.type == ModelDocument.Type.LINKBASE else "other",
        "href":
        doc.basename
    } for doc, ref in modelXbrl.modelDocument.referencesDocument.items()
                     if ref.referringModelObject.qname in SCHEMA_LB_REFS]

    roleTypes = [{
        "type":
        "role"
        if ref.referringModelObject.localName == "roleRef" else "arcroleRef",
        "href":
        ref.referringModelObject["href"]
    } for doc, ref in modelXbrl.modelDocument.referencesDocument.items()
                 if ref.referringModelObject.qname in ROLE_REFS]

    def factAspects(fact):
        aspects = {qnOimConceptAspect: oimValue(fact.qname)}
        if hasId and fact.id:
            aspects[qnOimIdAspect] = fact.id
        if hasLocation:
            aspects[qnOimLocationAspect] = elementChildSequence(fact)
        concept = fact.concept
        if concept is not None:
            if concept.baseXbrliType in ("string", "normalizedString",
                                         "token") and fact.xmlLang:
                aspects[qnOimLangAspect] = fact.xmlLang
        aspects[qnOimTypeAspect] = concept.baseXbrliType
        if fact.isItem:
            aspects[qnOimValueAspect] = (NILVALUE if fact.isNil else oimValue(
                fact.xValue, inferredDecimals(fact)))
        cntx = fact.context
        if cntx is not None:
            if cntx.entityIdentifierElement is not None:
                aspects[qnOimEntityAspect] = oimValue(
                    qname(*cntx.entityIdentifier))
            if cntx.period is not None:
                aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
            for dim in cntx.qnameDims.values():
                aspects[dim.dimensionQname] = (oimValue(dim.memberQname)
                                               if dim.isExplicit else
                                               dim.typedMember.stringValue)
        unit = fact.unit
        if unit is not None:
            _mMul, _mDiv = unit.measures
            if isJSON:
                aspects[
                    qnOimUnitAspect] = (  # use tuple instead of list for hashability
                        tuple(
                            oimValue(m)
                            for m in sorted(_mMul, key=lambda m: str(m))),
                        tuple(
                            oimValue(m)
                            for m in sorted(_mDiv, key=lambda m: str(m))))
            else:  # CSV
                if _mMul:
                    aspects[qnOimUnitMulAspect] = ",".join(
                        oimValue(m)
                        for m in sorted(_mMul, key=lambda m: str(m)))
                if _mDiv:
                    aspects[qnOimUnitDivAspect] = ",".join(
                        oimValue(m)
                        for m in sorted(_mDiv, key=lambda m: str(m)))
        return aspects

    if isJSON:
        # save JSON

        oim = {}  # top level of oim json output

        oimFacts = []
        oimReport = []
        oimReport.append({"url": modelXbrl.modelDocument.uri})
        if oimQNameSeparator != "clark":
            oimReport.append({
                "prefixMap":
                dict((p, ns) for ns, p in namespacePrefixes.items())
            })
        oimReport.append({"DTSreferences": dtsReferences})
        oimReport.append({"roleTypes": roleTypes})
        oimReport.append({"facts": oimFacts})
        oimReport.append({"footnotes": factFootnotes})
        oimReport.append({"relationships": factRelationships})

        if oimStyle == "flat":

            def saveFlatJsonFacts(facts, oimFacts):
                for fact in facts:
                    oimFact = factAspects(fact)
                    if fact.modelTupleFacts:
                        tupleFacts = []
                        oimFact[qnOimTupleAspect] = tupleFacts
                        saveFlatJsonFacts(fact.modelTupleFacts, tupleFacts)
                    oimFacts.append(
                        dict((oimValue(k), v) for k, v in oimFact.items()))

            saveFlatJsonFacts(modelXbrl.facts, oimFacts)

        elif oimStyle == "clustered":

            # build aspect-value usage per fact for every fact
            categoricalAspectValueSets = {}  # for each aspect, value facts-set
            aspectIndex = {}
            indexAspect = {}

            def addCategoricalAspect(aspectQn):
                i = len(aspectIndex)
                aspectIndex[aspectQn] = i
                indexAspect[i] = oimValue(aspectQn)
                categoricalAspectValueSets[i] = defaultdict(set)

            addCategoricalAspect(qnOimConceptAspect)
            addCategoricalAspect(qnOimEntityAspect)
            addCategoricalAspect(qnOimPeriodAspect)
            for aspectQn in aspectsDefined:
                if aspectQn.namespaceURI != nsOim or aspectQn in (
                        qnOimIdAspect, qnOimLangAspect, qnOimUnitAspect):
                    addCategoricalAspect(aspectQn)

            for fact in modelXbrl.facts:
                fact._factAspectValues = {}
                fact._factAspectSet = set()
                for aspectQn, value in factAspects(fact).items():
                    if aspectQn in aspectIndex:
                        i = aspectIndex[aspectQn]
                        v = oimValue(value)
                        categoricalAspectValueSets[i][v].add(fact)
                        fact._factAspectValues[i] = v
                        fact._factAspectSet.add(i)

            # order aspectValues by largest population
            maxAspectValuePopulation = [
                (aspectIndex,
                 max(len(factSet) for factSet in oimValueFacts.values()))
                for aspectIndex, oimValueFacts in
                categoricalAspectValueSets.items()
            ]

            maxAspectValuePopulation.sort(key=lambda ai_max: -ai_max[1])

            factsClustered = set()
            _aspectValue = {}

            def clusterAspect(_avpi, _data):
                if _avpi >= len(maxAspectValuePopulation):
                    return  # end of aspects
                _ai = maxAspectValuePopulation[_avpi][0]
                for _v, _vFactsSet in categoricalAspectValueSets[_ai].items():
                    _aspectValue[_ai] = _v
                    _nestedData = []
                    _nestedAspect = {indexAspect[_ai]: _v, "data": _nestedData}
                    for _fact in _vFactsSet - factsClustered:
                        if (_fact._factAspectSet == _aspectValue.keys()
                                and all([
                                    _fact._factAspectValues[__ai]
                                    == _aspectValue[__ai]
                                    for __ai in _aspectValue
                                ])):
                            _factAspects = factAspects(_fact)
                            _oimFactItem = {
                                oimValue(qnOimValueAspect):
                                _factAspects[qnOimValueAspect]
                            }
                            if hasLocation:
                                _oimFactItem[oimValue(
                                    qnOimLocationAspect
                                )] = _factAspects[qnOimLocationAspect]
                            if hasType:
                                _oimFactItem[
                                    oimValue(qnOimTypeAspect
                                             )] = _factAspects[qnOimTypeAspect]
                            _nestedData.append(_oimFactItem)
                            factsClustered.add(_fact)
                    clusterAspect(_avpi + 1, _nestedData)
                    if _nestedData:
                        _data.append(_nestedAspect)
                    del _aspectValue[_ai]

            clusterAspect(0, oimFacts)

        with open(oimFile, "w", encoding="utf-8") as fh:
            fh.write(
                json.dumps(oimReport,
                           ensure_ascii=False,
                           indent=1,
                           sort_keys=True))

    elif isCSV:
        # save CSV

        # levels of tuple nesting
        def tupleDepth(facts, parentDepth):
            _levelDepth = parentDepth
            for fact in facts:
                _factDepth = tupleDepth(fact.modelTupleFacts, parentDepth + 1)
                if _factDepth > _levelDepth:
                    _levelDepth = _factDepth
            return _levelDepth

        maxDepth = tupleDepth(modelXbrl.facts, 0)

        aspectQnCol = {oimValue(qnOimConceptAspect): maxDepth - 1}
        aspectsHeader = [oimValue(qnOimConceptAspect)]

        for i in range(maxDepth - 1):
            aspectsHeader.append(None)

        def addAspectQnCol(aspectQn):
            aspectQnCol[aspectQn] = len(aspectsHeader)
            aspectsHeader.append(oimValue(aspectQn))

        # pre-ordered aspect columns
        if hasId:
            addAspectQnCol(qnOimIdAspect)
        if hasLocation:
            addAspectQnCol(qnOimLocationAspect)
        if hasType:
            addAspectQnCol(qnOimTypeAspect)
        addAspectQnCol(qnOimValueAspect)
        if qnOimEntityAspect in aspectsDefined:
            addAspectQnCol(qnOimEntityAspect)
        if qnOimPeriodAspect in aspectsDefined:
            addAspectQnCol(qnOimPeriodAspect)
        if qnOimUnitMulAspect in aspectsDefined:
            addAspectQnCol(qnOimUnitMulAspect)
        if qnOimUnitDivAspect in aspectsDefined:
            addAspectQnCol(qnOimUnitDivAspect)
        for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
            if aspectQn.namespaceURI != nsOim:
                addAspectQnCol(aspectQn)

        def aspectCols(fact, depth):
            cols = [None for i in range(len(aspectsHeader))]
            for aspectQn, aspectValue in factAspects(fact).items():
                if aspectQn == qnOimConceptAspect:
                    cols[depth - 1] = aspectValue
                elif aspectQn in aspectQnCol:
                    cols[aspectQnCol[aspectQn]] = aspectValue
            return cols

        # save facts
        csvFile = open(oimFile,
                       csvOpenMode,
                       newline=csvOpenNewline,
                       encoding='utf-8-sig')
        csvWriter = csv.writer(csvFile, dialect="excel")
        csvWriter.writerow(aspectsHeader)

        def saveCSVfacts(facts, thisDepth):
            for fact in facts:
                csvWriter.writerow(aspectCols(fact, thisDepth))
                saveCSVfacts(fact.modelTupleFacts, thisDepth + 1)

        saveCSVfacts(modelXbrl.facts, 1)
        csvFile.close()

        # save namespaces
        if oimQNameSeparator == "clark":
            csvFile = open(oimFile.replace(".csv", "-prefixMap.csv"),
                           csvOpenMode,
                           newline=csvOpenNewline,
                           encoding='utf-8-sig')
            csvWriter = csv.writer(csvFile, dialect="excel")
            csvWriter.writerow(("prefix", "mappedURI"))
            for namespaceURI, prefix in sorted(namespacePrefixes.items(),
                                               key=lambda item: item[1]):
                csvWriter.writerow((prefix, namespaceURI))
            csvFile.close()

        # save dts references
        csvFile = open(oimFile.replace(".csv", "-dts.csv"),
                       csvOpenMode,
                       newline=csvOpenNewline,
                       encoding='utf-8-sig')
        csvWriter = csv.writer(csvFile, dialect="excel")
        csvWriter.writerow(("type", "href"))
        for oimRef in dtsReferences:
            csvWriter.writerow((oimRef["type"], oimRef["href"]))
        csvFile.close()

        # save role and arc type references
        if roleTypes:
            csvFile = open(oimFile.replace(".csv", "-roleTypes.csv"),
                           csvOpenMode,
                           newline=csvOpenNewline,
                           encoding='utf-8-sig')
            csvWriter = csv.writer(csvFile, dialect="excel")
            csvWriter.writerow(("type", "href"))
            for oimRef in roleTypes:
                csvWriter.writerow((oimRef["type"], oimRef["href"]))
            csvFile.close()

        # save relationships
        csvFile = open(oimFile.replace(".csv", "-relationships.csv"),
                       csvOpenMode,
                       newline=csvOpenNewline,
                       encoding='utf-8-sig')
        csvWriter = csv.writer(csvFile, dialect="excel")
        hasOrder = any(
            hasattribute(imRel, "order") for oimRel in factRelationships)
        csvWriter.writerow(("fromIds", "toIds", "linkrole", "arcrole") +
                           (("order", ) if hasOrder else ()))
        for oimRel in factRelationships:
            csvWriter.writerow(
                (",".join(oimRel["fromIds"]), ",".join(oimRel["toIds"]),
                 oimRel["linkrole"], oimRel["arcrole"]) +
                ((oimRel.get("order", None), ) if hasOrder else ()))
        csvFile.close()

        # save footnotes
        csvFile = open(oimFile.replace(".csv", "-footnotes.csv"),
                       csvOpenMode,
                       newline=csvOpenNewline,
                       encoding='utf-8-sig')
        csvWriter = csv.writer(csvFile, dialect="excel")
        hasLang = any(hasattribute(oimFnt, "lang") for oimFnt in factFootnotes)
        csvWriter.writerow(("id", "role") + (("lang", ) if hasLang else ()) +
                           ("value", ))
        for oimFnt in factFootnotes:
            csvWriter.writerow((oimFtn["id"], oimFtn["role"]) + (
                (oimFtn.get("lang", None), ) if hasLang else ()) +
                               (oimFtn["value"], ))
        csvFile.close()
예제 #6
0
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None, *args, **kwargs):
    targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath)
    def addLocallyReferencedFile(elt,filingFiles):
        if elt.tag in ("a", "img"):
            for attrTag, attrValue in elt.items():
                if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrValue):
                    attrValue = attrValue.partition('#')[0] # remove anchor
                    if attrValue: # ignore anchor references to base document
                        attrValue = os.path.normpath(attrValue) # change url path separators to host separators
                        file = os.path.join(sourceDir,attrValue)
                        if modelXbrl.fileSource.isInArchive(file, checkExistence=True) or os.path.exists(file):
                            filingFiles.add(file)
    targetUrlParts = targetUrl.rpartition(".")
    targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
    modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
    rootElt = modelXbrl.modelDocument.xmlRootElement
    # take baseXmlLang from <html> or <base>
    baseXmlLang = rootElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang")
    for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/1999/xhtml}body"):
        baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang") or baseXmlLang
    targetInstance = ModelXbrl.create(modelXbrl.modelManager, 
                                      newDocumentType=Type.INSTANCE,
                                      url=targetUrl,
                                      schemaRefs=targetDocumentSchemaRefs,
                                      isEntry=True,
                                      discover=False) # don't attempt to load DTS
    if baseXmlLang:
        targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang)
    ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults 
    # roleRef and arcroleRef (of each inline document)
    for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
        for roleRefElt in sourceRefs.values():
            addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, 
                     attributes=roleRefElt.items())
    
    # contexts
    for context in sorted(modelXbrl.contexts.values(), key=lambda c: elementChildSequence(c)):
        ignore = targetInstance.createContext(context.entityIdentifier[0],
                                               context.entityIdentifier[1],
                                               'instant' if context.isInstantPeriod else
                                               'duration' if context.isStartEndPeriod
                                               else 'forever',
                                               context.startDatetime,
                                               context.endDatetime,
                                               None, 
                                               context.qnameDims, [], [],
                                               id=context.id)
    for unit in modelXbrl.units.values():
        measures = unit.measures
        ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id)

    modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
    newFactForOldObjId = {}
    def createFacts(facts, parent):
        for fact in facts:
            if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior
                attrs = {"contextRef": fact.contextID}
                if fact.id:
                    attrs["id"] = fact.id
                if fact.isNumeric:
                    attrs["unitRef"] = fact.unitID
                    if fact.get("decimals"):
                        attrs["decimals"] = fact.get("decimals")
                    if fact.get("precision"):
                        attrs["precision"] = fact.get("precision")
                if fact.isNil:
                    attrs[XbrlConst.qnXsiNil] = "true"
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.textValue
                    if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default
                        xmlLang = fact.xmlLang
                        if xmlLang is not None and xmlLang != baseXmlLang:
                            attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang
                newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
                # if fact.isFraction, create numerator and denominator
                newFactForOldObjId[fact.objectIndex] = newFact
                if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock:
                    # check for img and other filing references so that referenced files are included in the zip.
                    for xmltext in [text] + CDATApattern.findall(text):
                        try:
                            for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter():
                                addLocallyReferencedFile(elt, filingFiles)
                        except (XMLSyntaxError, UnicodeDecodeError):
                            pass  # TODO: Why ignore UnicodeDecodeError?
            elif fact.isTuple:
                newTuple = targetInstance.createFact(fact.qname, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newTuple
                createFacts(fact.modelTupleFacts, newTuple)
                
    createFacts(modelXbrl.facts, None)
    modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships"))
    HREF = "{http://www.w3.org/1999/xlink}href"
    footnoteLinks = defaultdict(list)
    footnoteIdCount = {}
    for linkKey, linkPrototypes in modelXbrl.baseSets.items():
        arcrole, linkrole, linkqname, arcqname = linkKey
        if (linkrole and linkqname and arcqname and  # fully specified roles
            arcrole != "XBRL-footnotes" and
            any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)):
            for linkPrototype in linkPrototypes:
                if linkPrototype not in footnoteLinks[linkrole]:
                    footnoteLinks[linkrole].append(linkPrototype)
    for linkrole in sorted(footnoteLinks.keys()):
        for linkPrototype in footnoteLinks[linkrole]:
            newLink = addChild(targetInstance.modelDocument.xmlRootElement, 
                               linkPrototype.qname, 
                               attributes=linkPrototype.attributes)
            for linkChild in linkPrototype:
                attributes = linkChild.attributes
                if isinstance(linkChild, LocPrototype):
                    if HREF not in linkChild.attributes:
                        linkChild.attributes[HREF] = \
                        "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
                    addChild(newLink, linkChild.qname, 
                             attributes=attributes)
                elif isinstance(linkChild, ArcPrototype):
                    addChild(newLink, linkChild.qname, attributes=attributes)
                elif isinstance(linkChild, ModelInlineFootnote):
                    idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1
                    if idUseCount > 1: # if footnote with id in other links bump the id number
                        attributes = linkChild.attributes.copy()
                        attributes["id"] = "{}_{}".format(attributes["id"], idUseCount)
                    footnoteIdCount[linkChild.footnoteID] = idUseCount
                    newChild = addChild(newLink, linkChild.qname, 
                                        attributes=attributes)
                    xmlLang = linkChild.xmlLang
                    if xmlLang is not None and xmlLang != baseXmlLang: # default
                        newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang)
                    copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True)

                    if filingFiles and linkChild.textValue:
                        footnoteHtml = XML("<body/>")
                        copyIxFootnoteHtml(linkChild, footnoteHtml)
                        for elt in footnoteHtml.iter():
                            addLocallyReferencedFile(elt,filingFiles)
    targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip)
    if getattr(modelXbrl, "isTestcaseVariation", False):
        modelXbrl.extractedInlineInstance = True # for validation comparison
    modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)