예제 #1
0
    def loadXbrlFromDB(self, loadDBsaveToFile):
        # load from database
        modelXbrl = self.modelXbrl
        
        # find instance in DB
        instanceURI = os.path.basename(loadDBsaveToFile)
        results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant"
                               " FROM dInstance WHERE FileName = '{}'"
                               .format(instanceURI))
        instanceId = moduleId = None
        for instanceId, moduleId, entScheme, entId, datePerEnd in results:
            break

        # find module in DB        
        results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId))
        xbrlSchemaRef = None
        for result in results:
            xbrlSchemaRef = result[0]
            break
        
        if not instanceId or not xbrlSchemaRef:
            raise XPDBException("sqlDB:MissingDTS",
                    _("The instance and module were not found for %(instanceURI)"),
                    instanceURI = instanceURI) 
            
        if modelXbrl.skipDTS:
            # find prefixes and namespaces in DB
            results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]")            
            dpmPrefixedNamespaces = dict((prefix, namespace)
                                         for owner, prefix, namespace in results)
            
        # create the instance document and resulting filing
        modelXbrl.blockDpmDBrecursion = True
        modelXbrl.modelDocument = createModelDocument(modelXbrl, 
                                                      Type.INSTANCE,
                                                      loadDBsaveToFile,
                                                      schemaRefs=[xbrlSchemaRef],
                                                      isEntry=True)
        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults 
        
        addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement, 
                                 'xbrl-streamable-instance', 
                                 'version="1.0" contextBuffer="1"')

        # add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue)
        
        # filing indicator code IDs
        # get filing indicators
        results = self.execute("SELECT mToT.TemplateOrTableCode "
                               "  FROM dFilingIndicator dFI, mTemplateOrTable mToT "
                               "  WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID"
                               .format(instanceId))
        filingIndicatorCodes = [code[0] for code in results]
        
        if filingIndicatorCodes:
            modelXbrl.createContext(entScheme,
                        entId,
                        'instant',
                        None,
                        datePerEnd,
                        None, # no dimensional validity checking (like formula does)
                        {}, [], [],
                        id='c')
            filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators)
            for filingIndicatorCode in filingIndicatorCodes:
                modelXbrl.createFact(qnFindFilingIndicator, 
                                     parent=filingIndicatorsTuple,
                                     attributes={"contextRef": "c"}, 
                                     text=filingIndicatorCode)

        
        # facts in this instance
        factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards,"
                                " Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue "
                                "FROM dFact WHERE InstanceID = {} "
                                "ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL "
                                "                          THEN DataPointSignature"
                                "                          ELSE DataPointSignatureWithValuesForWildcards"
                                "                END, instr(DataPointSignature,'|') + 1)"
                                .format(instanceId))
        
        # results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal

        # get typed dimension values
        prefixedNamespaces = modelXbrl.prefixedNamespaces
        prefixedNamespaces["iso4217"] = XbrlConst.iso4217
        if modelXbrl.skipDTS:
            prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed
        
        cntxTbl = {} # index by d
        unitTbl = {}
        
        def typedDimElt(s):
            # add xmlns into s for known qnames
            tag, angleBrkt, rest = s[1:].partition('>')
            text, angleBrkt, rest = rest.partition("<")
            qn = qname(tag, prefixedNamespaces)
            # a modelObject xml element is needed for all of the instance functions to manage the typed dim
            return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
        
        # contexts and facts
        for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl:
            metric, sep, dims = (dpSigTypedDims or dpSig).partition('|')
            conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces)
            concept = modelXbrl.qnameConcepts.get(conceptQn)
            isNumeric = isBool = isDateTime = isQName = isText = False
            if concept is not None:
                if concept.isNumeric:
                    isNumeric = True
                else:
                    baseXbrliType = concept.baseXbrliType
                    if baseXbrliType == "booleanItemType":
                        isBool = True
                    elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
                        isDateTime = True
                    elif baseXbrliType == "QNameItemType":
                        isQName = True
            else:
                c = conceptQn.localName[0]
                if c == 'm':
                    isNumeric = True
                elif c == 'd':
                    isDateTime = True
                elif c == 'b':
                    isBool = True
                elif c == 'e':
                    isQName = True
            isText = not (isNumeric or isBool or isDateTime or isQName)
            if isinstance(datePerEnd, _STR_BASE):
                datePerEnd = datetimeValue(datePerEnd, addOneDay=True)
            cntxKey = (dims, entId, datePerEnd)
            if cntxKey in cntxTbl:
                cntxId = cntxTbl[cntxKey]
            else:
                cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                cntxTbl[cntxKey] = cntxId
                qnameDims = {}
                if dims:
                    for dim in dims.split('|'):
                        dQn, sep, dVal = dim[:-1].partition('(')
                        dimQname = qname(dQn, prefixedNamespaces)
                        if dVal.startswith('<'):
                            mem = typedDimElt(dVal)  # typed dim
                        else:
                            mem = qname(dVal, prefixedNamespaces) # explicit dim (even if treat-as-typed)
                        qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario")
                    
                modelXbrl.createContext(entScheme,
                                        entId,
                                        'instant',
                                        None,
                                        datePerEnd,
                                        None, # no dimensional validity checking (like formula does)
                                        qnameDims, [], [],
                                        id=cntxId)
            if unit:
                if unit in unitTbl:
                    unitId = unitTbl[unit]
                else:
                    unitQn = qname(unit, prefixedNamespaces)
                    unitId = 'u{}'.format(unitQn.localName)
                    unitTbl[unit] = unitId
                    modelXbrl.createUnit([unitQn], [], id=unitId)
            else:
                unitId = None
            attrs = {"contextRef": cntxId}
            if unitId:
                attrs["unitRef"] = unitId
            if dec is not None:
                if isinstance(dec, float): # must be an integer
                    dec = int(dec)
                elif isinstance(dec, _STR_BASE) and '.' in dec:
                    dec = dec.partition('.')[0] # drop .0 from any SQLite string
                attrs["decimals"] = str(dec)  # somehow it is float from the database
            if False: # fact.isNil:
                attrs[XbrlConst.qnXsiNil] = "true"
                text = None
            elif numVal is not None:
                num = roundValue(numVal, None, dec) # round using reported decimals
                if dec is None or dec == "INF":  # show using decimals or reported format
                    dec = len(numVal.partition(".")[2])
                else: # max decimals at 28
                    dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
                text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num))
            elif dateVal is not None:
                text = dateVal
            elif boolVal is not None:
                text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false'
            else:
                if isQName: # declare namespace
                    addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces))
                text = textVal
            modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
            
        # add footnotes if any
        
        # save to file
        modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile)
        modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
        return modelXbrl.modelDocument
예제 #2
0
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
    from openpyxl import load_workbook
    from arelle import ModelDocument, ModelXbrl, XmlUtil
    from arelle.ModelDocument import ModelDocumentReference
    from arelle.ModelValue import qname
    
    try:
        currentAction = "initializing"
        startedAt = time.time()
        
        if os.path.isabs(oimFile):
            # allow relative filenames to loading directory
            priorCWD = os.getcwd()
            os.chdir(os.path.dirname(oimFile))
        else:
            priorCWD = None
            
        currentAction = "determining file type"
        isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json")
        isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
        isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
        isCSVorXL = isCSV or isXL
        instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"
        
        if isJSON:
            currentAction = "loading and parsing JSON OIM file"
            with io.open(oimFile, 'rt', encoding='utf-8') as f:
                oimObject = json.load(f)
            missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject]
            if missing:
                raise OIMException("oime:missingJSONelements", 
                                   _("The required elements %(missing)s {} in JSON input"),
                                   missing = ", ".join(missing))
            currentAction = "identifying JSON objects"
            dtsReferences = oimObject["dtsReferences"]
            prefixes = oimObject["prefixes"]
            facts = oimObject["facts"]
            footnotes = oimOject["facts"] # shares this object
        elif isCSV:
            currentAction = "identifying CSV input tables"
            if sys.version[0] >= '3':
                csvOpenMode = 'w'
                csvOpenNewline = ''
            else:
                csvOpenMode = 'wb' # for 2.7
                csvOpenNewline = None
                
            oimFileBase = None
            if "-facts" in oimFile:
                oimFileBase = oimFile.partition("-facts")[0]
            else:
                for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"):
                    if oimFile.endswith(suffix):
                        oimFileBase = oimFile[:-length(suffix)]
                        break
            if oimFileBase is None:
                raise OIMException("oime:missingCSVtables", 
                                   _("Unable to identify CSV tables file name pattern"))
            if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or
                not os.path.exists(oimFileBase + "-prefixes.csv")):
                raise OIMException("oime:missingCSVtables", 
                                   _("Unable to identify CSV tables for dtsReferences or prefixes"))
            instanceFileName = oimFileBase + ".xbrl"
            currentAction = "loading CSV dtsReferences table"
            dtsReferences = []
            with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = row
                    else:
                        dtsReferences.append(dict((header[j], col) for j, col in enumerate(row)))
            currentAction = "loading CSV prefixes table"
            prefixes = {}
            with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = dict((col,i) for i,col in enumerate(row))
                    else:
                        prefixes[row[header["prefix"]]] = row[header["URI"]]
            currentAction = "loading CSV defaults table"
            defaults = {}
            if os.path.exists(oimFileBase + "-defaults.csv"):
                with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                            fileCol = row.index("file")
                        else:
                            defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol)
            currentAction = "loading CSV facts tables"
            facts = []
            _dir = os.path.dirname(oimFileBase)
            for filename in os.listdir(_dir):
                filepath = os.path.join(_dir, filename)
                if "-facts" in filename and filepath.startswith(oimFileBase):
                    tableDefaults = defaults.get(filename, {})
                    with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
                        csvReader = csv.reader(f)
                        for i, row in enumerate(csvReader):
                            if i == 0:
                                header = row
                            else:
                                fact = {}
                                fact.update(tableDefaults)
                                for j, col in enumerate(row):
                                    if col is not None:
                                        if header[j].endswith("Value"):
                                            if col: # ignore empty columns (= null CSV value)
                                                fact["value"] = col
                                        else:
                                            fact[header[j]] = col
                                facts.append(fact)
            footnotes = []
            if os.path.exists(oimFileBase + "-footnotes.csv"):
                with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                        else:
                            footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col))
        elif isXL:
            oimWb = load_workbook(oimFile, read_only=True, data_only=True)
            sheetNames = oimWb.get_sheet_names()
            if (not any(sheetName == "prefixes" for sheetName in sheetNames) or
                not any(sheetName == "dtsReferences" for sheetName in sheetNames) or
                not any("facts" in sheetName for sheetName in sheetNames)):
                if priorCWD: os.chdir(priorCWD)
                return None
            try:
                dtsReferences = []
                for i, row in oimWb["dtsReferences"]:
                    if i == 0:
                        header = [col.value for col in row]
                    else:
                        dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row)))
                prefixes = {}
                for i, row in oimWb["dtsReferences"]:
                    if i == 0:
                        header = dict((col.value,i) for i,col in enumerate(row))
                    else:
                        prefixes[row[header["prefix"]].value] = row[header["URI"].value]
                defaults = {}
                for i, row in oimW.get("defaults", ()):
                    if i == 0:
                        header = dict((col.value,i) for i,col in enumerate(row))
                        fileCol = header["file"]
                    else:
                        defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol)
                facts = []
                _dir = os.path.dirpart(oimFileBase)
                for sheetName in sheetNames:
                    if sheetName == "facts" or "-facts" in sheetName:
                        for i, row in oimWb[sheetName]:
                            if i == 0:
                                header = [col.value for col in row]
                            else:
                                fact = {}
                                fact.update(tableDefaults)
                                for j, col in enumerate(row):
                                    if col.value is not None:
                                        if header[j].endswith("Value"):
                                            fact["value"] = str(col.value)
                                        else:
                                            fact[header[j]] = str(col.value)
                                facts.append(fact)
                footnotes = []
                for i, row in oimWb.get("footnotes", ()):
                    if i == 0:
                        header = dict((col.value,i) for i,col in enumerate(row) if col.value)
                    else:
                        footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value))
            except Exception as ex:
                if priorCWD: os.chdir(priorCWD)
                return None
    
        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults 
            
        # create the instance document
        modelXbrl.blockDpmDBrecursion = True
        modelXbrl.modelDocument = createModelDocument(
              modelXbrl, 
              Type.INSTANCE,
              instanceFileName,
              schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"],
              isEntry=True,
              initialComment="extracted from OIM {}".format(mappedUri),
              documentEncoding="utf-8")
        cntxTbl = {}
        unitTbl = {}
        for fact in facts:
            conceptQn = qname(fact["oim:concept"], prefixes)
            concept = modelXbrl.qnameConcepts.get(conceptQn)
            entityAsQn = qname(fact["oim:entity"], prefixes)
            if "oim:period" in fact:
                periodStart = fact["oim:period"]["start"]
                periodEnd = fact["oim:period"]["end"]
            else:
                periodStart = fact["oim:periodStart"]
                periodEnd = fact["oim:periodEnd"]
            cntxKey = ( # hashable context key
                ("periodType", concept.periodType),
                ("entity", entityAsQn),
                ("periodStart", periodStart),
                ("periodEnd", periodEnd)) + tuple(sorted(
                    (dimName, dimVal) 
                    for dimName, dimVal in fact.items()
                    if ":" in dimName and not dimName.startswith("oim:")))
            if cntxKey in cntxTbl:
                _cntx = cntxTbl[cntxKey]
            else:
                cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                qnameDims = {}
                for dimName, dimVal in fact.items():
                    if ":" in dimName and not dimName.startswith("oim:"):
                        dimQname = qname(dimName, prefixes)
                        dimConcept = modelXbrl.qnameConcepts.get(dimQname)
                        if ":" in dimVal and dimVal.partition[':'][0] in prefixes:
                            mem = qname(dimVal, prefixes) # explicit dim
                        elif dimConcept.isTypedDimension:
                            # a modelObject xml element is needed for all of the instance functions to manage the typed dim
                            mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
                        qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
                _cntx = modelXbrl.createContext(
                                        entityAsQn.namespaceURI,
                                        entityAsQn.localName,
                                        concept.periodType,
                                        None if concept.periodType == "instnat" else dateTime(periodStart, type=DATETIME),
                                        dateTime(periodEnd, type=DATETIME),
                                        None, # no dimensional validity checking (like formula does)
                                        qnameDims, [], [],
                                        id=cntxId)
                cntxTbl[cntxKey] = _cntx
            if "oim:unit" in fact:
                unitKey = fact["oim:unit"]
                if unitKey in unitTbl:
                    _unit = unitTbl[unitKey]
                else:
                    _mul, _sep, _div = unitKey.partition('/')
                    if _mul.startswith('('):
                        _mul = _mul[1:-1]
                    mulQns = [qname(u, prefixes) for u in _mul.split('*') if u]
                    if _div.startswith('('):
                        _div = _div[1:-1]
                    divQns = [qname(u, prefixes) for u in _div.split('*') if u]
                    unitId = 'u-{:02}'.format(len(unitTbl) + 1)
                    for _measures in mulQns, divQns:
                        for _measure in _measures:
                            addQnameValue(modelXbrl.modelDocument, _measure)
                    _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId)
                    unitTbl[unitKey] = _unit
            else:
                _unit = None
            
            attrs = {"contextRef": _cntx.id}
    
            if fact.get("value") is None:
                attrs[XbrlConst.qnXsiNil] = "true"
                text = None
            else:
                text = fact["value"]
                
            if fact.get("id"):
                attrs["id"] = fact["id"]
                
            if concept.isNumeric:
                if _unit is not None:
                    attrs["unitRef"] = _unit.id
                if "accuracy" in fact:
                    attrs["decimals"] = fact["accuracy"]
                    
            # is value a QName?
            if concept.baseXbrliType == "QName":
                addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
    
            f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
            
        footnoteLinks = {} # ELR elements
        factLocs = {} # index by (linkrole, factId)
        footnoteNbr = 0
        locNbr = 0
        for factOrFootnote in footnotes:
            if "factId" in factOrFootnote:
                factId = factOrFootnote["factId"]
                factFootnotes = (factOrFootnote,) # CSV or XL
            elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
                factId = factOrFootnote["id"]
                factFootnotes = factOrFootnote["footnotes"]
            else:
                factFootnotes = ()
            for footnote in factFootnotes:
                linkrole = footnote.get("group")
                arcrole = footnote.get("footnoteType")
                if not factId or not linkrole or not arcrole or not (
                    footnote.get("factRef") or footnote.get("footnote")):
                    # invalid footnote
                    continue
                if linkrole not in footnoteLinks:
                    footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, 
                                                       XbrlConst.qnLinkFootnoteLink, 
                                                       attributes={"{http://www.w3.org/1999/xlink}type": "extended",
                                                                   "{http://www.w3.org/1999/xlink}role": linkrole})
                footnoteLink = footnoteLinks[linkrole]
                if (linkrole, factId) not in factLocs:
                    locNbr += 1
                    locLabel = "l_{:02}".format(locNbr)
                    factLocs[(linkrole, factId)] = locLabel
                    addChild(footnoteLink, XbrlConst.qnLinkLoc, 
                             attributes={XLINKTYPE: "locator",
                                         XLINKHREF: "#" + factId,
                                         XLINKLABEL: locLabel})
                locLabel = factLocs[(linkrole, factId)]
                if footnote.get("footnote"):
                    footnoteNbr += 1
                    footnoteLabel = "f_{:02}".format(footnoteNbr)
                    attrs = {XLINKTYPE: "resource",
                             XLINKLABEL: footnoteLabel}
                    if footnote.get("language"):
                        attrs[XMLLANG] = footnote["language"]
                    # note, for HTML will need to build an element structure
                    addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"])
                elif footnote.get("factRef"):
                    factRef = footnote.get("factRef")
                    if (linkrole, factRef) not in factLocs:
                        locNbr += 1
                        locLabel = "f_{:02}".format(footnoteNbr)
                        factLoc[(linkrole, factRef)] = locLabel
                        addChild(footnoteLink, XbrlConst.qnLinkLoc, 
                                 attributes={XLINKTYPE: "locator",
                                             XLINKHREF: "#" + factRef,
                                             XLINKLABEL: locLabel})
                    footnoteLabel = factLoc[(linkrole, factId)]
                footnoteArc = addChild(footnoteLink, 
                                       XbrlConst.qnLinkFootnoteArc, 
                                       attributes={XLINKTYPE: "arc",
                                                   XLINKARCROLE: arcrole,
                                                   XLINKFROM: locLabel,
                                                   XLINKTO: footnoteLabel})
                    
        
        #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
        #               messageCode="loadFromExcel:info")
    except Exception as ex:
        if ex is OIMException:
            modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
        else:
            modelXbrl.error("Error while %(action)s, error %(error)s",
                            modelObject=modelXbrl, action=currentAction, error=ex)
    
    if priorCWD:
        os.chdir(priorCWD) # restore prior current working directory
    return modelXbrl.modelDocument
예제 #3
0
     def createModelFact(fact, parentModelFact, topTupleFact):
         aspects = fact.get("aspects", EMPTYDICT)
         if oimConcept not in aspects:
             modelXbrl.error("{}:conceptQName".format(errPrefix),
                             _("The concept QName could not be determined"),
                             modelObject=modelXbrl)
             return
         conceptQn = qname(aspects[oimConcept], prefixes)
         concept = modelXbrl.qnameConcepts.get(conceptQn)
         attrs = {}
         if concept.isItem:
             missingAspects = []
             if oimEntity not in aspects: 
                 missingAspects.append(oimEntity)
             if oimPeriod not in aspects and (oimPeriodStart not in aspects or oimPeriodEnd not in aspects):
                 missingAspects.append(oimPeriod)
             if missingAspects:
                 modelXbrl.error("{}:missingAspects".format(errPrefix),
                                 _("The concept %(element)s is missing aspects %(missingAspects)s"),
                                 modelObject=modelXbrl, element=conceptQn, missingAspects=", ".join(missingAspects))
                 return
             entityAsQn = qname(aspects[oimEntity], prefixes) or qname("error",fact[oimEntity])
             if oimPeriod in aspects:
                 periodStart = periodEnd = aspects[oimPeriod]
             if oimPeriodStart in aspects and oimPeriodEnd in aspects:
                 periodStart = aspects[oimPeriodStart]
                 periodEnd = aspects[oimPeriodEnd]
             cntxKey = ( # hashable context key
                 ("periodType", concept.periodType),
                 ("entity", entityAsQn),
                 ("periodStart", periodStart),
                 ("periodEnd", periodEnd)) + tuple(sorted(
                     (dimName, dimVal["value"] if isinstance(dimVal,dict) else dimVal) 
                     for dimName, dimVal in aspects.items()
                     if ":" in dimName and not dimName.startswith(oimPrefix)))
             if cntxKey in cntxTbl:
                 _cntx = cntxTbl[cntxKey]
             else:
                 cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                 qnameDims = {}
                 for dimName, dimVal in aspects.items():
                     if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal:
                         dimQname = qname(dimName, prefixes)
                         dimConcept = modelXbrl.qnameConcepts.get(dimQname)
                         if dimConcept is None:
                             modelXbrl.error("{}:taxonomyDefinedAspectQName".format(errPrefix),
                                             _("The taxonomy defined aspect concept QName %(qname)s could not be determined"),
                                             modelObject=modelXbrl, qname=dimQname)
                             continue
                         if isinstance(dimVal, dict):
                             dimVal = dimVal["value"]
                         else:
                             dimVal = str(dimVal) # may be int or boolean
                         if isinstance(dimVal,str) and ":" in dimVal and dimVal.partition(':')[0] in prefixes:
                             mem = qname(dimVal, prefixes) # explicit dim
                         elif dimConcept.isTypedDimension:
                             # a modelObject xml element is needed for all of the instance functions to manage the typed dim
                             mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
                         qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
                 _cntx = modelXbrl.createContext(
                                         entityAsQn.namespaceURI,
                                         entityAsQn.localName,
                                         concept.periodType,
                                         None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME),
                                         dateTime(periodEnd, type=DATETIME),
                                         None, # no dimensional validity checking (like formula does)
                                         qnameDims, [], [],
                                         id=cntxId,
                                         beforeSibling=topTupleFact)
                 cntxTbl[cntxKey] = _cntx
             if oimUnit in aspects and concept.isNumeric:
                 unitKey = aspects[oimUnit]
                 if unitKey in unitTbl:
                     _unit = unitTbl[unitKey]
                 else:
                     _unit = None
                     # validate unit
                     unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey)
                     if not UnitPattern.match(unitKeySub):
                         modelXbrl.error("oime:unitStringRepresentation",
                                         _("Unit string representation is lexically invalid, %(unit)s"),
                                         modelObject=modelXbrl, unit=unitKey)
                     else:
                         _mul, _sep, _div = unitKey.partition('/')
                         if _mul.startswith('('):
                             _mul = _mul[1:-1]
                         _muls = [u for u in _mul.split('*') if u]
                         if _div.startswith('('):
                             _div = _div[1:-1]
                         _divs = [u for u in _div.split('*') if u]
                         if _muls != sorted(_muls) or _divs != sorted(_divs):
                             modelXbrl.error("oime:unitStringRepresentation",
                                             _("Unit string representation measures are not in alphabetical order, %(unit)s"),
                                             modelObject=modelXbrl, unit=unitKey)
                         try:
                             mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
                                                                                       _("Unit prefix is not declared: %(unit)s"),
                                                                                       unit=u)) 
                                       for u in _muls]
                             divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
                                                                                       _("Unit prefix is not declared: %(unit)s"),
                                                                                       unit=u))
                                       for u in _divs]
                             unitId = 'u-{:02}'.format(len(unitTbl) + 1)
                             for _measures in mulQns, divQns:
                                 for _measure in _measures:
                                     addQnameValue(modelXbrl.modelDocument, _measure)
                             _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId, beforeSibling=topTupleFact)
                         except OIMException as ex:
                             modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
                     unitTbl[unitKey] = _unit
             else:
                 _unit = None
         
             attrs["contextRef"] = _cntx.id
     
             if fact.get("value") is None:
                 attrs[XbrlConst.qnXsiNil] = "true"
                 text = None
             else:
                 text = fact["value"]
                 
             if concept.isNumeric:
                 if _unit is None:
                     return # skip creating fact because unit was invalid
                 attrs["unitRef"] = _unit.id
                 if "accuracy" in attrs or attrs.get(XbrlConst.qnXsiNil, "false") != "true":
                     attrs["decimals"] = fact.get("accuracy", "INF")
         else:
             text = None #tuple
                 
         id = fact.get("id")
         if id is not None:
             attrs["id"] = fact["id"]
                 
         # is value a QName?
         if concept.baseXbrliType == "QName":
             addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
 
         f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text, parent=parentModelFact, validate=False)
         
         if id is not None and id in parentedFacts:
             # create child facts
             for i in sorted(parentedFacts[id],
                             key=lambda j: facts[j].get("aspects", EMPTYDICT).get(oimTupleOrder,0)):
                 createModelFact(facts[i], f, f if topTupleFact is None else topTupleFact)
                 
         # validate after creating tuple contents
         xmlValidate(modelXbrl, f)
예제 #4
0
    def loadXbrlFromDB(self, loadDBsaveToFile):
        # load from database
        modelXbrl = self.modelXbrl
        
        # find instance in DB
        instanceURI = os.path.basename(loadDBsaveToFile)
        results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant"
                               " FROM dInstance WHERE FileName = '{}'"
                               .format(instanceURI))
        instanceId = moduleId = None
        for instanceId, moduleId, entScheme, entId, datePerEnd in results:
            break

        # find module in DB        
        results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId))
        xbrlSchemaRef = None
        for result in results:
            xbrlSchemaRef = result[0]
            break
        
        if not instanceId or not xbrlSchemaRef:
            raise XPDBException("sqlDB:MissingDTS",
                    _("The instance and module were not found for %(instanceURI)"),
                    instanceURI = instanceURI) 
            
        if modelXbrl.skipDTS:
            # find prefixes and namespaces in DB
            results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]")            
            dpmPrefixedNamespaces = dict((prefix, namespace)
                                         for owner, prefix, namespace in results)
            
        # create the instance document and resulting filing
        modelXbrl.blockDpmDBrecursion = True
        modelXbrl.modelDocument = createModelDocument(modelXbrl, 
                                                      Type.INSTANCE,
                                                      loadDBsaveToFile,
                                                      schemaRefs=[xbrlSchemaRef],
                                                      isEntry=True)
        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults 
        
        addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement, 
                                 'xbrl-streamable-instance', 
                                 'version="1.0" contextBuffer="1"')

        # add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue)
        
        # filing indicator code IDs
        # get filing indicators
        results = self.execute("SELECT mToT.TemplateOrTableCode "
                               "  FROM dFilingIndicator dFI, mTemplateOrTable mToT "
                               "  WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID"
                               .format(instanceId))
        filingIndicatorCodes = [code[0] for code in results]
        
        if filingIndicatorCodes:
            modelXbrl.createContext(entScheme,
                        entId,
                        'instant',
                        None,
                        datePerEnd,
                        None, # no dimensional validity checking (like formula does)
                        {}, [], [],
                        id='c')
            filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators)
            for filingIndicatorCode in filingIndicatorCodes:
                modelXbrl.createFact(qnFindFilingIndicator, 
                                     parent=filingIndicatorsTuple,
                                     attributes={"contextRef": "c"}, 
                                     text=filingIndicatorCode)

        
        # facts in this instance
        factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards,"
                                " Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue "
                                "FROM dFact WHERE InstanceID = {} "
                                "ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL "
                                "                          THEN DataPointSignature"
                                "                          ELSE DataPointSignatureWithValuesForWildcards"
                                "                END, instr(DataPointSignature,'|') + 1)"
                                .format(instanceId))
        
        # results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal

        # get typed dimension values
        prefixedNamespaces = modelXbrl.prefixedNamespaces
        prefixedNamespaces["iso4217"] = XbrlConst.iso4217
        if modelXbrl.skipDTS:
            prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed
        
        cntxTbl = {} # index by d
        unitTbl = {}
        
        def typedDimElt(s):
            # add xmlns into s for known qnames
            tag, angleBrkt, rest = s[1:].partition('>')
            text, angleBrkt, rest = rest.partition("<")
            qn = qname(tag, prefixedNamespaces)
            # a modelObject xml element is needed for all of the instance functions to manage the typed dim
            return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
        
        # contexts and facts
        for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl:
            metric, sep, dims = (dpSigTypedDims or dpSig).partition('|')
            conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces)
            concept = modelXbrl.qnameConcepts.get(conceptQn)
            isNumeric = isBool = isDateTime = isQName = isText = False
            if concept is not None:
                if concept.isNumeric:
                    isNumeric = True
                else:
                    baseXbrliType = concept.baseXbrliType
                    if baseXbrliType == "booleanItemType":
                        isBool = True
                    elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
                        isDateTime = True
                    elif baseXbrliType == "QNameItemType":
                        isQName = True
            else:
                c = conceptQn.localName[0]
                if c == 'm':
                    isNumeric = True
                elif c == 'd':
                    isDateTime = True
                elif c == 'b':
                    isBool = True
                elif c == 'e':
                    isQName = True
            isText = not (isNumeric or isBool or isDateTime or isQName)
            if isinstance(datePerEnd, _STR_BASE):
                datePerEnd = datetimeValue(datePerEnd, addOneDay=True)
            cntxKey = (dims, entId, datePerEnd)
            if cntxKey in cntxTbl:
                cntxId = cntxTbl[cntxKey]
            else:
                cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                cntxTbl[cntxKey] = cntxId
                qnameDims = {}
                if dims:
                    for dim in dims.split('|'):
                        dQn, sep, dVal = dim[:-1].partition('(')
                        dimQname = qname(dQn, prefixedNamespaces)
                        if dVal.startswith('<'): # typed dim
                            mem = typedDimElt(dVal)
                        else:
                            mem = qname(dVal, prefixedNamespaces)
                        qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario")
                    
                modelXbrl.createContext(entScheme,
                                        entId,
                                        'instant',
                                        None,
                                        datePerEnd,
                                        None, # no dimensional validity checking (like formula does)
                                        qnameDims, [], [],
                                        id=cntxId)
            if unit:
                if unit in unitTbl:
                    unitId = unitTbl[unit]
                else:
                    unitQn = qname(unit, prefixedNamespaces)
                    unitId = 'u{}'.format(unitQn.localName)
                    unitTbl[unit] = unitId
                    modelXbrl.createUnit([unitQn], [], id=unitId)
            else:
                unitId = None
            attrs = {"contextRef": cntxId}
            if unitId:
                attrs["unitRef"] = unitId
            if dec is not None:
                if isinstance(dec, float): # must be an integer
                    dec = int(dec)
                elif isinstance(dec, _STR_BASE) and '.' in dec:
                    dec = dec.partition('.')[0] # drop .0 from any SQLite string
                attrs["decimals"] = str(dec)  # somehow it is float from the database
            if False: # fact.isNil:
                attrs[XbrlConst.qnXsiNil] = "true"
                text = None
            elif numVal is not None:
                num = roundValue(numVal, None, dec) # round using reported decimals
                if dec is None or dec == "INF":  # show using decimals or reported format
                    dec = len(numVal.partition(".")[2])
                else: # max decimals at 28
                    dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
                text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num))
            elif dateVal is not None:
                text = dateVal
            elif boolVal is not None:
                text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false'
            else:
                if isQName: # declare namespace
                    addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces))
                text = textVal
            modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
            
        # add footnotes if any
        
        # save to file
        modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile)
        modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
        return modelXbrl.modelDocument
예제 #5
0
 def yAxisByCol(self, leftCol, row, yParentStructuralNode, childrenFirst, renderNow, atTop):
     if yParentStructuralNode is not None:
         nestedBottomRow = row
         for yStructuralNode in yParentStructuralNode.childStructuralNodes:
             nestRow, nextRow = self.yAxisByCol(leftCol + 1, row, yStructuralNode,  # nested items before totals
                                                childrenFirst, childrenFirst, False)
             isAbstract = (yStructuralNode.isAbstract or 
                           (yStructuralNode.childStructuralNodes and
                            not isinstance(yStructuralNode.definitionNode, (ModelClosedDefinitionNode, ModelEuAxisCoord))))
             isNonAbstract = not isAbstract
             label = yStructuralNode.header(lang=self.lang,
                                            returnGenLabel=isinstance(yStructuralNode.definitionNode, (ModelClosedDefinitionNode, ModelEuAxisCoord)))
             topRow = row
             if childrenFirst and isNonAbstract:
                 row = nextRow
             #print ( "thisCol {0} leftCol {1} rightCol {2} topRow{3} renderNow {4} label {5}".format(thisCol, leftCol, rightCol, topRow, renderNow, label))
             if renderNow:
                 rowspan= nestRow - row + 1
                 cellElt = etree.Element(tableModelQName("cell"),
                                         attrib={"span": str(rowspan)} if rowspan > 1 else None)
                 elt = etree.SubElement(cellElt, tableModelQName("label"))
                 elt.text = label
                 self.rowHdrElts[leftCol - 1].append(cellElt)
                 self.structuralNodeModelElements.append((yStructuralNode, cellElt))
                 for aspect in sorted(yStructuralNode.aspectsCovered(), key=lambda a: aspectStr(a)):
                     if yStructuralNode.hasAspect(aspect) and aspect != Aspect.DIMENSIONS:
                         aspectValue = yStructuralNode.aspectValue(aspect)
                         if aspectValue is None: aspectValue = "(bound dynamically)"
                         elt = etree.SubElement(cellElt, tableModelQName("constraint"))
                         etree.SubElement(elt, tableModelQName("aspect")
                                          ).text = aspectStr(aspect)
                         etree.SubElement(elt, tableModelQName("value")
                                          ).text = addQnameValue(self.xmlDoc, aspectValue)
                 for rollUpCol in range(leftCol, self.rowHdrCols - 1):
                     rollUpElt = etree.Element(tableModelQName("cell"),
                                               attrib={"rollup":"true"})
                     self.rowHdrElts[rollUpCol].append(rollUpElt)
                 if isNonAbstract:
                     cellElt = etree.Element(tableModelQName("cell"),
                                             attrib={"span": str(rowspan)} if rowspan > 1 else None)
                     for i, role in enumerate(self.rowHdrNonStdRoles):
                         labelElt = etree.SubElement(cellElt, tableModelQName("label"),
                                                     attrib={"role":role,
                                                             "lang":self.lang})
                         labelElt.text = yStructuralNode.header(role=role, lang=self.lang)
                     self.rowHdrElts[self.rowHdrCols - 1 + i].append(cellElt)
                     for aspect in sorted(yStructuralNode.aspectsCovered(), key=lambda a: aspectStr(a)):
                         if yStructuralNode.hasAspect(aspect) and aspect != Aspect.DIMENSIONS:
                             aspectValue = yStructuralNode.aspectValue(aspect)
                             if aspectValue is None: aspectValue = "(bound dynamically)"
                             elt = etree.SubElement(cellElt, tableModelQName("constraint"))
                             etree.SubElement(elt, tableModelQName("aspect")
                                              ).text = aspectStr(aspect)
                             etree.SubElement(elt, tableModelQName("value")
                                              ).text = addQnameValue(self.xmlDoc, aspectValue)
                     '''
                     if self.rowHdrDocCol:
                         labelElt = etree.SubElement(cellElt, tableModelQName("label"),
                                                     attrib={"span": str(rowspan)} if rowspan > 1 else None)
                         elt.text = yStructuralNode.header(role="http://www.xbrl.org/2008/role/documentation",
                                                    lang=self.lang)
                         self.rowHdrElts[self.rowHdrCols - 1].append(elt)
                     if self.rowHdrCodeCol:
                         elt = etree.Element(tableModelQName("label"),
                                             attrib={"span": str(rowspan)} if rowspan > 1 else None)
                         elt.text = yStructuralNode.header(role="http://www.eurofiling.info/role/2010/coordinate-code",
                                                    lang=self.lang)
                         self.rowHdrElts[self.rowHdrCols - 1 + self.rowHdrDocCol].append(elt)
                     '''
             if isNonAbstract:
                 row += 1
             elif childrenFirst:
                 row = nextRow
             if nestRow > nestedBottomRow:
                 nestedBottomRow = nestRow + (isNonAbstract and not childrenFirst)
             if row > nestedBottomRow:
                 nestedBottomRow = row
             #if renderNow and not childrenFirst:
             #    dummy, row = self.yAxis(leftCol + 1, row, yStructuralNode, childrenFirst, True, False) # render on this pass
             if not childrenFirst:
                 dummy, row = self.yAxisByCol(leftCol + 1, row, yStructuralNode, childrenFirst, renderNow, False) # render on this pass
         return (nestedBottomRow, row)
예제 #6
0
 def xAxis(self, leftCol, topRow, rowBelow, xParentStructuralNode, xStructuralNodes, childrenFirst, renderNow, atTop):
     if xParentStructuralNode is not None:
         parentRow = rowBelow
         noDescendants = True
         rightCol = leftCol
         widthToSpanParent = 0
         sideBorder = not xStructuralNodes
         for xStructuralNode in xParentStructuralNode.childStructuralNodes:
             noDescendants = False
             rightCol, row, width, leafNode = self.xAxis(leftCol, topRow + 1, rowBelow, xStructuralNode, xStructuralNodes, # nested items before totals
                                                         childrenFirst, childrenFirst, False)
             if row - 1 < parentRow:
                 parentRow = row - 1
             #if not leafNode: 
             #    rightCol -= 1
             nonAbstract = not xStructuralNode.isAbstract
             if nonAbstract:
                 width += 100 # width for this label
             widthToSpanParent += width
             label = xStructuralNode.header(lang=self.lang,
                                            returnGenLabel=isinstance(xStructuralNode.definitionNode, (ModelClosedDefinitionNode, ModelEuAxisCoord)))
             if childrenFirst:
                 thisCol = rightCol
             else:
                 thisCol = leftCol
             #print ( "thisCol {0} leftCol {1} rightCol {2} topRow{3} renderNow {4} label {5}".format(thisCol, leftCol, rightCol, topRow, renderNow, label))
             if renderNow:
                 columnspan = rightCol - leftCol + (1 if nonAbstract else 0)
                 if self.type == HTML:
                     if rightCol == self.dataFirstCol + self.dataCols - 1:
                         edgeBorder = "border-right:.5pt solid windowtext;"
                     else:
                         edgeBorder = ""
                     attrib = {"class":"xAxisHdr",
                               "style":"text-align:center;max-width:{0}pt;{1}".format(width,edgeBorder)}
                     if columnspan > 1:
                         attrib["colspan"] = str(columnspan)
                     if leafNode and row > topRow:
                         attrib["rowspan"] = str(row - topRow + 1)
                     elt = etree.Element("{http://www.w3.org/1999/xhtml}th",
                                         attrib=attrib)
                     self.rowElts[topRow-1].insert(leftCol,elt)
                 elif self.type == XML:
                     cellElt = etree.Element(tableModelQName("cell"),
                                         attrib={"span": str(columnspan)} if columnspan > 1 else None)
                     self.colHdrElts[topRow - self.colHdrTopRow].insert(leftCol,cellElt)
                     self.structuralNodeModelElements.append((xStructuralNode, cellElt))
                     elt = etree.SubElement(cellElt, tableModelQName("label"))
                     if nonAbstract or (leafNode and row > topRow):
                         for rollUpCol in range(topRow - self.colHdrTopRow + 1, self.colHdrRows - 1):
                             rollUpElt = etree.Element(tableModelQName("label"),
                                                       attrib={"rollup":"true"})
                             if childrenFirst:
                                 self.colHdrElts[rollUpCol].append(rollUpElt)
                             else:
                                 self.colHdrElts[rollUpCol].insert(leftCol,rollUpElt)
                     for i, role in enumerate(self.colHdrNonStdRoles):
                         etree.SubElement(cellElt, tableModelQName("label"), 
                                          attrib={"role": role,
                                                  "lang": self.lang}
                                          ).text = xStructuralNode.header(
                                                role=role, lang=self.lang)
                     for aspect in sorted(xStructuralNode.aspectsCovered(), key=lambda a: aspectStr(a)):
                         if xStructuralNode.hasAspect(aspect) and aspect != Aspect.DIMENSIONS:
                             aspectValue = xStructuralNode.aspectValue(aspect)
                             if aspectValue is None: aspectValue = "(bound dynamically)"
                             aspElt = etree.SubElement(cellElt, tableModelQName("constraint"))
                             etree.SubElement(aspElt, tableModelQName("aspect")
                                              ).text = aspectStr(aspect)
                             etree.SubElement(aspElt, tableModelQName("value")
                                              ).text = addQnameValue(self.xmlDoc, aspectValue)
                 elt.text = label or "\u00A0" #produces &nbsp;
                 if nonAbstract:
                     if columnspan > 1 and rowBelow > topRow:   # add spanned left leg portion one row down
                         if self.type == HTML:
                             attrib= {"class":"xAxisSpanLeg",
                                      "rowspan": str(rowBelow - row)}
                             if edgeBorder:
                                 attrib["style"] = edgeBorder
                             elt = etree.Element("{http://www.w3.org/1999/xhtml}th",
                                                 attrib=attrib)
                             elt.text = "\u00A0"
                             if childrenFirst:
                                 self.rowElts[topRow].append(elt)
                             else:
                                 self.rowElts[topRow].insert(leftCol,elt)
                     if self.type == HTML:
                         for i, role in enumerate(self.colHdrNonStdRoles):
                             elt = etree.Element("{http://www.w3.org/1999/xhtml}th",
                                                 attrib={"class":"xAxisHdr",
                                                         "style":"text-align:center;max-width:100pt;{0}".format(edgeBorder)})
                             self.rowElts[self.dataFirstRow - 1 - len(self.colHdrNonStdRoles) + i].insert(thisCol,elt)
                             elt.text = xStructuralNode.header(role=role, lang=self.lang) or "\u00A0"
                     '''
                     if self.colHdrDocRow:
                         doc = xStructuralNode.header(role="http://www.xbrl.org/2008/role/documentation", lang=self.lang)
                         if self.type == HTML:
                             elt = etree.Element("{http://www.w3.org/1999/xhtml}th",
                                                 attrib={"class":"xAxisHdr",
                                                         "style":"text-align:center;max-width:100pt;{0}".format(edgeBorder)})
                             self.rowElts[self.dataFirstRow - 2 - self.rowHdrCodeCol].insert(thisCol,elt)
                         elif self.type == XML:
                             elt = etree.Element(tableModelQName("label"))
                             self.colHdrElts[self.colHdrRows - 1].insert(thisCol,elt)
                         elt.text = doc or "\u00A0"
                     if self.colHdrCodeRow:
                         code = xStructuralNode.header(role="http://www.eurofiling.info/role/2010/coordinate-code")
                         if self.type == HTML:
                             elt = etree.Element("{http://www.w3.org/1999/xhtml}th",
                                                 attrib={"class":"xAxisHdr",
                                                         "style":"text-align:center;max-width:100pt;{0}".format(edgeBorder)})
                             self.rowElts[self.dataFirstRow - 2].insert(thisCol,elt)
                         elif self.type == XML:
                             elt = etree.Element(tableModelQName("label"))
                             self.colHdrElts[self.colHdrRows - 1 + self.colHdrDocRow].insert(thisCol,elt)
                         elt.text = code or "\u00A0"
                     '''
                     xStructuralNodes.append(xStructuralNode)
             if nonAbstract:
                 rightCol += 1
             if renderNow and not childrenFirst:
                 self.xAxis(leftCol + (1 if nonAbstract else 0), topRow + 1, rowBelow, xStructuralNode, xStructuralNodes, childrenFirst, True, False) # render on this pass
             leftCol = rightCol
         return (rightCol, parentRow, widthToSpanParent, noDescendants)
예제 #7
0
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
    from openpyxl import load_workbook
    from arelle import ModelDocument, ModelXbrl, XmlUtil
    from arelle.ModelDocument import ModelDocumentReference
    from arelle.ModelValue import qname
    
    try:
        currentAction = "initializing"
        startingErrorCount = len(modelXbrl.errors)
        startedAt = time.time()
        
        if os.path.isabs(oimFile):
            # allow relative filenames to loading directory
            priorCWD = os.getcwd()
            os.chdir(os.path.dirname(oimFile))
        else:
            priorCWD = None
            
        currentAction = "determining file type"
        isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json")
        isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
        isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
        isCSVorXL = isCSV or isXL
        instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"
        
        if isJSON:
            currentAction = "loading and parsing JSON OIM file"
            def loadDict(keyValuePairs):
                _dict = OrderedDict() # preserve fact order in resulting instance
                for key, value in keyValuePairs:
                    if isinstance(value, dict):
                        if DUPJSONKEY in value:
                            for _errKey, _errValue, _otherValue in value[DUPJSONKEY]:
                                if key == "prefixes":
                                    modelXbrl.error("oime:duplicatedPrefix",
                                                    _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"),
                                                    modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue)
                            del value[DUPJSONKEY]
                    if key in _dict:
                        if DUPJSONKEY not in _dict:
                            _dict[DUPJSONKEY] = []
                        _dict[DUPJSONKEY].append((key, value, _dict[key]))
                    else:
                        _dict[key] = value
                return _dict
            with io.open(oimFile, 'rt', encoding='utf-8') as f:
                oimObject = json.load(f, object_pairs_hook=loadDict)
            missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject]
            if missing:
                raise OIMException("oime:missingJSONElements", 
                                   _("Required element(s) are missing from JSON input: %(missing)s"),
                                   missing = ", ".join(missing))
            currentAction = "identifying JSON objects"
            dtsReferences = oimObject["dtsReferences"]
            prefixesList = oimObject["prefixes"].items()
            facts = oimObject["facts"]
            footnotes = oimObject["facts"] # shares this object
        elif isCSV:
            currentAction = "identifying CSV input tables"
            if sys.version[0] >= '3':
                csvOpenMode = 'w'
                csvOpenNewline = ''
            else:
                csvOpenMode = 'wb' # for 2.7
                csvOpenNewline = None
                
            oimFileBase = None
            if "-facts" in oimFile:
                oimFileBase = oimFile.partition("-facts")[0]
            else:
                for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"):
                    if oimFile.endswith(suffix):
                        oimFileBase = oimFile[:-len(suffix)]
                        break
            if oimFileBase is None:
                raise OIMException("oime:missingCSVTables", 
                                   _("Unable to identify CSV tables file name pattern"))
            if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or
                not os.path.exists(oimFileBase + "-prefixes.csv")):
                raise OIMException("oime:missingCSVTables", 
                                   _("Unable to identify CSV tables for dtsReferences or prefixes"))
            instanceFileName = oimFileBase + ".xbrl"
            currentAction = "loading CSV dtsReferences table"
            dtsReferences = []
            with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = row
                    else:
                        dtsReferences.append(dict((header[j], col) for j, col in enumerate(row)))
            currentAction = "loading CSV prefixes table"
            prefixesList = []
            with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = dict((col,i) for i,col in enumerate(row))
                    else:
                        prefixesList.append((row[header["prefix"]], row[header["URI"]]))
            defaults = {}
            if os.path.exists(oimFileBase + "-defaults.csv"):
                currentAction = "loading CSV defaults table"
                with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                            fileCol = row.index("file")
                        else:
                            defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol)
            currentAction = "loading CSV facts tables"
            facts = []
            _dir = os.path.dirname(oimFileBase)
            factsFileBasename = os.path.basename(oimFileBase) + "-facts"
            for filename in os.listdir(_dir):
                filepath = os.path.join(_dir, filename)
                if filename.startswith(factsFileBasename):
                    currentAction = "loading CSV facts table {}".format(filename)
                    tableDefaults = defaults.get(filename, {})
                    with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
                        csvReader = csv.reader(f)
                        for i, row in enumerate(csvReader):
                            if i == 0:
                                header = row
                            else:
                                fact = {}
                                fact.update(tableDefaults)
                                for j, col in enumerate(row):
                                    if col is not None:
                                        if header[j]: # skip cols with no header
                                            if header[j].endswith("Value"):
                                                if col: # ignore empty columns (= null CSV value)
                                                    fact["value"] = col
                                            else:
                                                fact[header[j]] = col
                                facts.append(fact)
            footnotes = []
            if os.path.exists(oimFileBase + "-footnotes.csv"):
                currentAction = "loading CSV footnotes table"
                with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                        else:
                            footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col))
        elif isXL:
            currentAction = "identifying workbook input worksheets"
            oimWb = load_workbook(oimFile, read_only=True, data_only=True)
            sheetNames = oimWb.get_sheet_names()
            if (not any(sheetName == "prefixes" for sheetName in sheetNames) or
                not any(sheetName == "dtsReferences" for sheetName in sheetNames) or
                not any("facts" in sheetName for sheetName in sheetNames)):
                raise OIMException("oime:missingWorkbookWorksheets", 
                                   _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts"))
            currentAction = "loading worksheet: dtsReferences"
            dtsReferences = []
            for i, row in enumerate(oimWb["dtsReferences"]):
                if i == 0:
                    header = [col.value for col in row]
                else:
                    dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row)))
            currentAction = "loading worksheet: prefixes"
            prefixesList = []
            for i, row in enumerate(oimWb["prefixes"]):
                if i == 0:
                    header = dict((col.value,i) for i,col in enumerate(row))
                else:
                    prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value))
            defaults = {}
            if "defaults" in sheetNames:
                currentAction = "loading worksheet: defaults"
                for i, row in enumerate(oimWb["defaults"]):
                    if i == 0:
                        header = dict((col.value,i) for i,col in enumerate(row))
                        fileCol = header["file"]
                    else:
                        defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol)
            facts = []
            for sheetName in sheetNames:
                if sheetName == "facts" or "-facts" in sheetName:
                    currentAction = "loading worksheet: {}".format(sheetName)
                    tableDefaults = defaults.get(sheetName, {})
                    for i, row in enumerate(oimWb[sheetName]):
                        if i == 0:
                            header = [col.value for col in row]
                        else:
                            fact = {}
                            fact.update(tableDefaults)
                            for j, col in enumerate(row):
                                if col.value is not None:
                                    if header[j]: # skip cols with no header
                                        if header[j].endswith("Value"):
                                            fact["value"] = str(col.value)
                                        else:
                                            fact[header[j]] = str(col.value)
                            facts.append(fact)
            footnotes = []
            if "footnotes" in sheetNames:
                currentAction = "loading worksheet: footnotes"
                for i, row in enumerate(oimWb["footnotes"]):
                    if i == 0:
                        header = dict((col.value,i) for i,col in enumerate(row) if col.value)
                    else:
                        footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value))
    
        currentAction = "identifying default dimensions"
        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults 
        
        currentAction = "validating OIM"
        prefixes = {}
        prefixedUris = {}
        for _prefix, _uri in prefixesList:
            if not _prefix:
                modelXbrl.error("oime:emptyPrefix",
                                _("The empty string must not be used as a prefix, uri %(uri)s"),
                                modelObject=modelXbrl, uri=_uri)
            elif not NCNamePattern.match(_prefix):
                modelXbrl.error("oime:prefixPattern",
                                _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"),
                                modelObject=modelXbrl, prefix=_prefix, uri=_uri)
            elif _prefix in prefixes:
                modelXbrl.error("oime:duplicatedPrefix",
                                _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"),
                                modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri)
            elif _uri in prefixedUris:
                modelXbrl.error("oime:duplicatedUri",
                                _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"),
                                modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix)
            else:
                prefixes[_prefix] = _uri
                prefixedUris[_uri] = _prefix
                
        oimPrefix = None
        for _nsOim in nsOim:
            if _nsOim in prefixedUris:
                oimPrefix = prefixedUris[_nsOim]
        if not oimPrefix:
            raise OIMException("oime:noOimPrefix",
                               _("The oim namespace must have a declared prefix"))
        oimConcept = "{}:concept".format(oimPrefix)
        oimEntity = "{}:entity".format(oimPrefix)
        oimPeriod = "{}:period".format(oimPrefix)
        oimPeriodStart = "{}:periodStart".format(oimPrefix)
        oimPeriodEnd = "{}:periodEnd".format(oimPrefix)
        oimUnit = "{}:unit".format(oimPrefix)
        oimPrefix = "{}:".format(oimPrefix)
            
        # create the instance document
        currentAction = "creating instance document"
        modelXbrl.blockDpmDBrecursion = True
        modelXbrl.modelDocument = createModelDocument(
              modelXbrl, 
              Type.INSTANCE,
              instanceFileName,
              schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"],
              isEntry=True,
              initialComment="extracted from OIM {}".format(mappedUri),
              documentEncoding="utf-8")
        cntxTbl = {}
        unitTbl = {}
        currentAction = "creating facts"
        for fact in facts:
            conceptQn = qname(fact[oimConcept], prefixes)
            concept = modelXbrl.qnameConcepts.get(conceptQn)
            entityAsQn = qname(fact[oimEntity], prefixes)
            if oimPeriod in fact:
                periodStart = fact[oimPeriod]["start"]
                periodEnd = fact[oimPeriod]["end"]
            else:
                periodStart = fact[oimPeriodStart]
                periodEnd = fact[oimPeriodEnd]
            cntxKey = ( # hashable context key
                ("periodType", concept.periodType),
                ("entity", entityAsQn),
                ("periodStart", periodStart),
                ("periodEnd", periodEnd)) + tuple(sorted(
                    (dimName, dimVal) 
                    for dimName, dimVal in fact.items()
                    if ":" in dimName and not dimName.startswith(oimPrefix)))
            if cntxKey in cntxTbl:
                _cntx = cntxTbl[cntxKey]
            else:
                cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                qnameDims = {}
                for dimName, dimVal in fact.items():
                    if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal:
                        dimQname = qname(dimName, prefixes)
                        dimConcept = modelXbrl.qnameConcepts.get(dimQname)
                        if ":" in dimVal and dimVal.partition(':')[0] in prefixes:
                            mem = qname(dimVal, prefixes) # explicit dim
                        elif dimConcept.isTypedDimension:
                            # a modelObject xml element is needed for all of the instance functions to manage the typed dim
                            mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
                        qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
                _cntx = modelXbrl.createContext(
                                        entityAsQn.namespaceURI,
                                        entityAsQn.localName,
                                        concept.periodType,
                                        None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME),
                                        dateTime(periodEnd, type=DATETIME),
                                        None, # no dimensional validity checking (like formula does)
                                        qnameDims, [], [],
                                        id=cntxId)
                cntxTbl[cntxKey] = _cntx
            if oimUnit in fact:
                unitKey = fact[oimUnit]
                if unitKey in unitTbl:
                    _unit = unitTbl[unitKey]
                else:
                    _unit = None
                    # validate unit
                    unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey)
                    if not UnitPattern.match(unitKeySub):
                        modelXbrl.error("oime:unitStringRepresentation",
                                        _("Unit string representation is lexically invalid, %(unit)s"),
                                        modelObject=modelXbrl, unit=unitKey)
                    else:
                        _mul, _sep, _div = unitKey.partition('/')
                        if _mul.startswith('('):
                            _mul = _mul[1:-1]
                        _muls = [u for u in _mul.split('*') if u]
                        if _div.startswith('('):
                            _div = _div[1:-1]
                        _divs = [u for u in _div.split('*') if u]
                        if _muls != sorted(_muls) or _divs != sorted(_divs):
                            modelXbrl.error("oime:unitStringRepresentation",
                                            _("Unit string representation measures are not in alphabetical order, %(unit)s"),
                                            modelObject=modelXbrl, unit=unitKey)
                        try:
                            mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
                                                                                      _("Unit prefix is not declared: %(unit)s"),
                                                                                      unit=u)) 
                                      for u in _muls]
                            divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
                                                                                      _("Unit prefix is not declared: %(unit)s"),
                                                                                      unit=u))
                                      for u in _divs]
                            unitId = 'u-{:02}'.format(len(unitTbl) + 1)
                            for _measures in mulQns, divQns:
                                for _measure in _measures:
                                    addQnameValue(modelXbrl.modelDocument, _measure)
                            _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId)
                        except OIMException as ex:
                            modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
                    unitTbl[unitKey] = _unit
            else:
                _unit = None
            
            attrs = {"contextRef": _cntx.id}
    
            if fact.get("value") is None:
                attrs[XbrlConst.qnXsiNil] = "true"
                text = None
            else:
                text = fact["value"]
                
            if fact.get("id"):
                attrs["id"] = fact["id"]
                
            if concept.isNumeric:
                if _unit is None:
                    continue # skip creating fact because unit was invalid
                attrs["unitRef"] = _unit.id
                if "accuracy" in fact:
                    attrs["decimals"] = fact["accuracy"]
                    
            # is value a QName?
            if concept.baseXbrliType == "QName":
                addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
    
            f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
            
        currentAction = "creating footnotes"
        footnoteLinks = {} # ELR elements
        factLocs = {} # index by (linkrole, factId)
        footnoteNbr = 0
        locNbr = 0
        for factOrFootnote in footnotes:
            if "factId" in factOrFootnote:
                factId = factOrFootnote["factId"]
                factFootnotes = (factOrFootnote,) # CSV or XL
            elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
                factId = factOrFootnote["id"]
                factFootnotes = factOrFootnote["footnotes"]
            else:
                factFootnotes = ()
            for footnote in factFootnotes:
                linkrole = footnote.get("group")
                arcrole = footnote.get("footnoteType")
                if not factId or not linkrole or not arcrole or not (
                    footnote.get("factRef") or footnote.get("footnote")):
                    # invalid footnote
                    continue
                if linkrole not in footnoteLinks:
                    footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, 
                                                       XbrlConst.qnLinkFootnoteLink, 
                                                       attributes={"{http://www.w3.org/1999/xlink}type": "extended",
                                                                   "{http://www.w3.org/1999/xlink}role": linkrole})
                footnoteLink = footnoteLinks[linkrole]
                if (linkrole, factId) not in factLocs:
                    locNbr += 1
                    locLabel = "l_{:02}".format(locNbr)
                    factLocs[(linkrole, factId)] = locLabel
                    addChild(footnoteLink, XbrlConst.qnLinkLoc, 
                             attributes={XLINKTYPE: "locator",
                                         XLINKHREF: "#" + factId,
                                         XLINKLABEL: locLabel})
                locLabel = factLocs[(linkrole, factId)]
                if footnote.get("footnote"):
                    footnoteNbr += 1
                    footnoteLabel = "f_{:02}".format(footnoteNbr)
                    attrs = {XLINKTYPE: "resource",
                             XLINKLABEL: footnoteLabel}
                    if footnote.get("language"):
                        attrs[XMLLANG] = footnote["language"]
                    # note, for HTML will need to build an element structure
                    addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"])
                elif footnote.get("factRef"):
                    factRef = footnote.get("factRef")
                    if (linkrole, factRef) not in factLocs:
                        locNbr += 1
                        locLabel = "f_{:02}".format(footnoteNbr)
                        factLoc[(linkrole, factRef)] = locLabel
                        addChild(footnoteLink, XbrlConst.qnLinkLoc, 
                                 attributes={XLINKTYPE: "locator",
                                             XLINKHREF: "#" + factRef,
                                             XLINKLABEL: locLabel})
                    footnoteLabel = factLoc[(linkrole, factId)]
                footnoteArc = addChild(footnoteLink, 
                                       XbrlConst.qnLinkFootnoteArc, 
                                       attributes={XLINKTYPE: "arc",
                                                   XLINKARCROLE: arcrole,
                                                   XLINKFROM: locLabel,
                                                   XLINKTO: footnoteLabel})
                    
        currentAction = "done loading facts and footnotes"
        
        #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
        #               messageCode="loadFromExcel:info")
    except Exception as ex:
        if isinstance(ex, OIMException):
            modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
        else:
            modelXbrl.error("arelleOIMloader:error",
                            "Error while %(action)s, error %(error)s\ntraceback %(traceback)s",
                            modelObject=modelXbrl, action=currentAction, error=ex,
                            traceback=traceback.format_tb(sys.exc_info()[2]))
    
    if priorCWD:
        os.chdir(priorCWD) # restore prior current working directory            startingErrorCount = len(modelXbrl.errors)
        
    if startingErrorCount < len(modelXbrl.errors):
        # had errors, don't allow ModelDocument.load to continue
        return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors")

    return getattr(modelXbrl, "modelDocument", None) # none if returning from exception
예제 #8
0
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
    from openpyxl import load_workbook
    from arelle import ModelDocument, ModelXbrl, XmlUtil
    from arelle.ModelDocument import ModelDocumentReference
    from arelle.ModelValue import qname

    try:
        currentAction = "initializing"
        startingErrorCount = len(modelXbrl.errors)
        startedAt = time.time()

        if os.path.isabs(oimFile):
            # allow relative filenames to loading directory
            priorCWD = os.getcwd()
            os.chdir(os.path.dirname(oimFile))
        else:
            priorCWD = None

        currentAction = "determining file type"
        isJSON = oimFile.endswith(
            ".json") and not oimFile.endswith("-metadata.json")
        isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
        isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
        isCSVorXL = isCSV or isXL
        instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"

        if isJSON:
            currentAction = "loading and parsing JSON OIM file"

            def loadDict(keyValuePairs):
                _dict = OrderedDict(
                )  # preserve fact order in resulting instance
                for key, value in keyValuePairs:
                    if isinstance(value, dict):
                        if DUPJSONKEY in value:
                            for _errKey, _errValue, _otherValue in value[
                                    DUPJSONKEY]:
                                if key == "prefixes":
                                    modelXbrl.error(
                                        "oime:duplicatedPrefix",
                                        _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"
                                          ),
                                        modelObject=modelXbrl,
                                        prefix=_errKey,
                                        uri1=_errValue,
                                        uri2=_otherValue)
                            del value[DUPJSONKEY]
                    if key in _dict:
                        if DUPJSONKEY not in _dict:
                            _dict[DUPJSONKEY] = []
                        _dict[DUPJSONKEY].append((key, value, _dict[key]))
                    else:
                        _dict[key] = value
                return _dict

            with io.open(oimFile, 'rt', encoding='utf-8') as f:
                oimObject = json.load(f, object_pairs_hook=loadDict)
            missing = [
                t for t in ("dtsReferences", "prefixes", "facts")
                if t not in oimObject
            ]
            if missing:
                raise OIMException(
                    "oime:missingJSONElements",
                    _("Required element(s) are missing from JSON input: %(missing)s"
                      ),
                    missing=", ".join(missing))
            currentAction = "identifying JSON objects"
            dtsReferences = oimObject["dtsReferences"]
            prefixesList = oimObject["prefixes"].items()
            facts = oimObject["facts"]
            footnotes = oimObject["facts"]  # shares this object
        elif isCSV:
            currentAction = "identifying CSV input tables"
            if sys.version[0] >= '3':
                csvOpenMode = 'w'
                csvOpenNewline = ''
            else:
                csvOpenMode = 'wb'  # for 2.7
                csvOpenNewline = None

            oimFileBase = None
            if "-facts" in oimFile:
                oimFileBase = oimFile.partition("-facts")[0]
            else:
                for suffix in ("-dtsReferences.csv", "-defaults.csv",
                               "-prefixes.csv", "-footnotes.csv",
                               "-metadata.json"):
                    if oimFile.endswith(suffix):
                        oimFileBase = oimFile[:-len(suffix)]
                        break
            if oimFileBase is None:
                raise OIMException(
                    "oime:missingCSVTables",
                    _("Unable to identify CSV tables file name pattern"))
            if (not os.path.exists(oimFileBase + "-dtsReferences.csv")
                    or not os.path.exists(oimFileBase + "-prefixes.csv")):
                raise OIMException(
                    "oime:missingCSVTables",
                    _("Unable to identify CSV tables for dtsReferences or prefixes"
                      ))
            instanceFileName = oimFileBase + ".xbrl"
            currentAction = "loading CSV dtsReferences table"
            dtsReferences = []
            with io.open(oimFileBase + "-dtsReferences.csv",
                         'rt',
                         encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = row
                    else:
                        dtsReferences.append(
                            dict(
                                (header[j], col) for j, col in enumerate(row)))
            currentAction = "loading CSV prefixes table"
            prefixesList = []
            with io.open(oimFileBase + "-prefixes.csv",
                         'rt',
                         encoding='utf-8-sig') as f:
                csvReader = csv.reader(f)
                for i, row in enumerate(csvReader):
                    if i == 0:
                        header = dict((col, i) for i, col in enumerate(row))
                    else:
                        prefixesList.append(
                            (row[header["prefix"]], row[header["URI"]]))
            defaults = {}
            if os.path.exists(oimFileBase + "-defaults.csv"):
                currentAction = "loading CSV defaults table"
                with io.open(oimFileBase + "-defaults.csv",
                             'rt',
                             encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                            fileCol = row.index("file")
                        else:
                            defaults[row[fileCol]] = dict(
                                (header[j], col) for j, col in enumerate(row)
                                if j != fileCol)
            currentAction = "loading CSV facts tables"
            facts = []
            _dir = os.path.dirname(oimFileBase)
            factsFileBasename = os.path.basename(oimFileBase) + "-facts"
            for filename in os.listdir(_dir):
                filepath = os.path.join(_dir, filename)
                if filename.startswith(factsFileBasename):
                    currentAction = "loading CSV facts table {}".format(
                        filename)
                    tableDefaults = defaults.get(filename, {})
                    with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
                        csvReader = csv.reader(f)
                        for i, row in enumerate(csvReader):
                            if i == 0:
                                header = row
                            else:
                                fact = {}
                                fact.update(tableDefaults)
                                for j, col in enumerate(row):
                                    if col is not None:
                                        if header[
                                                j]:  # skip cols with no header
                                            if header[j].endswith("Value"):
                                                if col:  # ignore empty columns (= null CSV value)
                                                    fact["value"] = col
                                            else:
                                                fact[header[j]] = col
                                facts.append(fact)
            footnotes = []
            if os.path.exists(oimFileBase + "-footnotes.csv"):
                currentAction = "loading CSV footnotes table"
                with io.open(oimFileBase + "-footnotes.csv",
                             'rt',
                             encoding='utf-8-sig') as f:
                    csvReader = csv.reader(f)
                    for i, row in enumerate(csvReader):
                        if i == 0:
                            header = row
                        else:
                            footnotes.append(
                                dict((header[j], col)
                                     for j, col in enumerate(row) if col))
        elif isXL:
            currentAction = "identifying workbook input worksheets"
            oimWb = load_workbook(oimFile, read_only=True, data_only=True)
            sheetNames = oimWb.get_sheet_names()
            if (not any(sheetName == "prefixes" for sheetName in sheetNames)
                    or not any(sheetName == "dtsReferences"
                               for sheetName in sheetNames)
                    or not any("facts" in sheetName
                               for sheetName in sheetNames)):
                raise OIMException(
                    "oime:missingWorkbookWorksheets",
                    _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts"
                      ))
            currentAction = "loading worksheet: dtsReferences"
            dtsReferences = []
            for i, row in enumerate(oimWb["dtsReferences"]):
                if i == 0:
                    header = [col.value for col in row]
                else:
                    dtsReferences.append(
                        dict((header[j], col.value)
                             for j, col in enumerate(row)))
            currentAction = "loading worksheet: prefixes"
            prefixesList = []
            for i, row in enumerate(oimWb["prefixes"]):
                if i == 0:
                    header = dict((col.value, i) for i, col in enumerate(row))
                else:
                    prefixesList.append((row[header["prefix"]].value,
                                         row[header["URI"]].value))
            defaults = {}
            if "defaults" in sheetNames:
                currentAction = "loading worksheet: defaults"
                for i, row in enumerate(oimWb["defaults"]):
                    if i == 0:
                        header = dict(
                            (col.value, i) for i, col in enumerate(row))
                        fileCol = header["file"]
                    else:
                        defaults[row[fileCol].value] = dict(
                            (header[j], col.value) for j, col in enumerate(row)
                            if j != fileCol)
            facts = []
            for sheetName in sheetNames:
                if sheetName == "facts" or "-facts" in sheetName:
                    currentAction = "loading worksheet: {}".format(sheetName)
                    tableDefaults = defaults.get(sheetName, {})
                    for i, row in enumerate(oimWb[sheetName]):
                        if i == 0:
                            header = [col.value for col in row]
                        else:
                            fact = {}
                            fact.update(tableDefaults)
                            for j, col in enumerate(row):
                                if col.value is not None:
                                    if header[j]:  # skip cols with no header
                                        if header[j].endswith("Value"):
                                            fact["value"] = str(col.value)
                                        else:
                                            fact[header[j]] = str(col.value)
                            facts.append(fact)
            footnotes = []
            if "footnotes" in sheetNames:
                currentAction = "loading worksheet: footnotes"
                for i, row in enumerate(oimWb["footnotes"]):
                    if i == 0:
                        header = dict((col.value, i)
                                      for i, col in enumerate(row)
                                      if col.value)
                    else:
                        footnotes.append(
                            dict((header[j], col.value)
                                 for j, col in enumerate(row) if col.value))

        currentAction = "identifying default dimensions"
        ValidateXbrlDimensions.loadDimensionDefaults(
            modelXbrl)  # needs dimension defaults

        currentAction = "validating OIM"
        prefixes = {}
        prefixedUris = {}
        for _prefix, _uri in prefixesList:
            if not _prefix:
                modelXbrl.error(
                    "oime:emptyPrefix",
                    _("The empty string must not be used as a prefix, uri %(uri)s"
                      ),
                    modelObject=modelXbrl,
                    uri=_uri)
            elif not NCNamePattern.match(_prefix):
                modelXbrl.error(
                    "oime:prefixPattern",
                    _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"
                      ),
                    modelObject=modelXbrl,
                    prefix=_prefix,
                    uri=_uri)
            elif _prefix in prefixes:
                modelXbrl.error(
                    "oime:duplicatedPrefix",
                    _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"
                      ),
                    modelObject=modelXbrl,
                    prefix=_prefix,
                    uri1=prefixes[_prefix],
                    uri2=_uri)
            elif _uri in prefixedUris:
                modelXbrl.error(
                    "oime:duplicatedUri",
                    _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"
                      ),
                    modelObject=modelXbrl,
                    uri=_uri,
                    prefix1=prefixedUris[_uri],
                    prefix2=_prefix)
            else:
                prefixes[_prefix] = _uri
                prefixedUris[_uri] = _prefix

        oimPrefix = None
        for _nsOim in nsOim:
            if _nsOim in prefixedUris:
                oimPrefix = prefixedUris[_nsOim]
        if not oimPrefix:
            raise OIMException(
                "oime:noOimPrefix",
                _("The oim namespace must have a declared prefix"))
        oimConcept = "{}:concept".format(oimPrefix)
        oimEntity = "{}:entity".format(oimPrefix)
        oimPeriod = "{}:period".format(oimPrefix)
        oimPeriodStart = "{}:periodStart".format(oimPrefix)
        oimPeriodEnd = "{}:periodEnd".format(oimPrefix)
        oimUnit = "{}:unit".format(oimPrefix)
        oimPrefix = "{}:".format(oimPrefix)

        # create the instance document
        currentAction = "creating instance document"
        modelXbrl.blockDpmDBrecursion = True
        modelXbrl.modelDocument = createModelDocument(
            modelXbrl,
            Type.INSTANCE,
            instanceFileName,
            schemaRefs=[
                dtsRef["href"] for dtsRef in dtsReferences
                if dtsRef["type"] == "schema"
            ],
            isEntry=True,
            initialComment="extracted from OIM {}".format(mappedUri),
            documentEncoding="utf-8")
        cntxTbl = {}
        unitTbl = {}
        currentAction = "creating facts"
        for fact in facts:
            conceptQn = qname(fact[oimConcept], prefixes)
            concept = modelXbrl.qnameConcepts.get(conceptQn)
            entityAsQn = qname(fact[oimEntity], prefixes)
            if oimPeriod in fact:
                periodStart = fact[oimPeriod]["start"]
                periodEnd = fact[oimPeriod]["end"]
            else:
                periodStart = fact[oimPeriodStart]
                periodEnd = fact[oimPeriodEnd]
            cntxKey = (  # hashable context key
                ("periodType", concept.periodType), ("entity", entityAsQn),
                ("periodStart", periodStart),
                ("periodEnd", periodEnd)) + tuple(
                    sorted(
                        (dimName, dimVal)
                        for dimName, dimVal in fact.items() if ":" in dimName
                        and not dimName.startswith(oimPrefix)))
            if cntxKey in cntxTbl:
                _cntx = cntxTbl[cntxKey]
            else:
                cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
                qnameDims = {}
                for dimName, dimVal in fact.items():
                    if ":" in dimName and not dimName.startswith(
                            oimPrefix) and dimVal:
                        dimQname = qname(dimName, prefixes)
                        dimConcept = modelXbrl.qnameConcepts.get(dimQname)
                        if ":" in dimVal and dimVal.partition(
                                ':')[0] in prefixes:
                            mem = qname(dimVal, prefixes)  # explicit dim
                        elif dimConcept.isTypedDimension:
                            # a modelObject xml element is needed for all of the instance functions to manage the typed dim
                            mem = addChild(modelXbrl.modelDocument,
                                           dimConcept.typedDomainElement.qname,
                                           text=dimVal,
                                           appendChild=False)
                        qnameDims[dimQname] = DimValuePrototype(
                            modelXbrl, None, dimQname, mem, "segment")
                _cntx = modelXbrl.createContext(
                    entityAsQn.namespaceURI,
                    entityAsQn.localName,
                    concept.periodType,
                    None if concept.periodType == "instant" else dateTime(
                        periodStart, type=DATETIME),
                    dateTime(periodEnd, type=DATETIME),
                    None,  # no dimensional validity checking (like formula does)
                    qnameDims,
                    [],
                    [],
                    id=cntxId)
                cntxTbl[cntxKey] = _cntx
            if oimUnit in fact:
                unitKey = fact[oimUnit]
                if unitKey in unitTbl:
                    _unit = unitTbl[unitKey]
                else:
                    _unit = None
                    # validate unit
                    unitKeySub = PrefixedQName.sub(
                        UnitPrefixedQNameSubstitutionChar, unitKey)
                    if not UnitPattern.match(unitKeySub):
                        modelXbrl.error(
                            "oime:unitStringRepresentation",
                            _("Unit string representation is lexically invalid, %(unit)s"
                              ),
                            modelObject=modelXbrl,
                            unit=unitKey)
                    else:
                        _mul, _sep, _div = unitKey.partition('/')
                        if _mul.startswith('('):
                            _mul = _mul[1:-1]
                        _muls = [u for u in _mul.split('*') if u]
                        if _div.startswith('('):
                            _div = _div[1:-1]
                        _divs = [u for u in _div.split('*') if u]
                        if _muls != sorted(_muls) or _divs != sorted(_divs):
                            modelXbrl.error(
                                "oime:unitStringRepresentation",
                                _("Unit string representation measures are not in alphabetical order, %(unit)s"
                                  ),
                                modelObject=modelXbrl,
                                unit=unitKey)
                        try:
                            mulQns = [
                                qname(
                                    u,
                                    prefixes,
                                    prefixException=OIMException(
                                        "oime:unitPrefix",
                                        _("Unit prefix is not declared: %(unit)s"
                                          ),
                                        unit=u)) for u in _muls
                            ]
                            divQns = [
                                qname(
                                    u,
                                    prefixes,
                                    prefixException=OIMException(
                                        "oime:unitPrefix",
                                        _("Unit prefix is not declared: %(unit)s"
                                          ),
                                        unit=u)) for u in _divs
                            ]
                            unitId = 'u-{:02}'.format(len(unitTbl) + 1)
                            for _measures in mulQns, divQns:
                                for _measure in _measures:
                                    addQnameValue(modelXbrl.modelDocument,
                                                  _measure)
                            _unit = modelXbrl.createUnit(mulQns,
                                                         divQns,
                                                         id=unitId)
                        except OIMException as ex:
                            modelXbrl.error(ex.code,
                                            ex.message,
                                            modelObject=modelXbrl,
                                            **ex.msgArgs)
                    unitTbl[unitKey] = _unit
            else:
                _unit = None

            attrs = {"contextRef": _cntx.id}

            if fact.get("value") is None:
                attrs[XbrlConst.qnXsiNil] = "true"
                text = None
            else:
                text = fact["value"]

            if fact.get("id"):
                attrs["id"] = fact["id"]

            if concept.isNumeric:
                if _unit is None:
                    continue  # skip creating fact because unit was invalid
                attrs["unitRef"] = _unit.id
                if "accuracy" in fact:
                    attrs["decimals"] = fact["accuracy"]

            # is value a QName?
            if concept.baseXbrliType == "QName":
                addQnameValue(modelXbrl.modelDocument,
                              qname(text.strip(), prefixes))

            f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)

        currentAction = "creating footnotes"
        footnoteLinks = {}  # ELR elements
        factLocs = {}  # index by (linkrole, factId)
        footnoteNbr = 0
        locNbr = 0
        for factOrFootnote in footnotes:
            if "factId" in factOrFootnote:
                factId = factOrFootnote["factId"]
                factFootnotes = (factOrFootnote, )  # CSV or XL
            elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
                factId = factOrFootnote["id"]
                factFootnotes = factOrFootnote["footnotes"]
            else:
                factFootnotes = ()
            for footnote in factFootnotes:
                linkrole = footnote.get("group")
                arcrole = footnote.get("footnoteType")
                if not factId or not linkrole or not arcrole or not (
                        footnote.get("factRef") or footnote.get("footnote")):
                    # invalid footnote
                    continue
                if linkrole not in footnoteLinks:
                    footnoteLinks[linkrole] = addChild(
                        modelXbrl.modelDocument.xmlRootElement,
                        XbrlConst.qnLinkFootnoteLink,
                        attributes={
                            "{http://www.w3.org/1999/xlink}type": "extended",
                            "{http://www.w3.org/1999/xlink}role": linkrole
                        })
                footnoteLink = footnoteLinks[linkrole]
                if (linkrole, factId) not in factLocs:
                    locNbr += 1
                    locLabel = "l_{:02}".format(locNbr)
                    factLocs[(linkrole, factId)] = locLabel
                    addChild(footnoteLink,
                             XbrlConst.qnLinkLoc,
                             attributes={
                                 XLINKTYPE: "locator",
                                 XLINKHREF: "#" + factId,
                                 XLINKLABEL: locLabel
                             })
                locLabel = factLocs[(linkrole, factId)]
                if footnote.get("footnote"):
                    footnoteNbr += 1
                    footnoteLabel = "f_{:02}".format(footnoteNbr)
                    attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel}
                    if footnote.get("language"):
                        attrs[XMLLANG] = footnote["language"]
                    # note, for HTML will need to build an element structure
                    addChild(footnoteLink,
                             XbrlConst.qnLinkFootnote,
                             attributes=attrs,
                             text=footnote["footnote"])
                elif footnote.get("factRef"):
                    factRef = footnote.get("factRef")
                    if (linkrole, factRef) not in factLocs:
                        locNbr += 1
                        locLabel = "f_{:02}".format(footnoteNbr)
                        factLoc[(linkrole, factRef)] = locLabel
                        addChild(footnoteLink,
                                 XbrlConst.qnLinkLoc,
                                 attributes={
                                     XLINKTYPE: "locator",
                                     XLINKHREF: "#" + factRef,
                                     XLINKLABEL: locLabel
                                 })
                    footnoteLabel = factLoc[(linkrole, factId)]
                footnoteArc = addChild(footnoteLink,
                                       XbrlConst.qnLinkFootnoteArc,
                                       attributes={
                                           XLINKTYPE: "arc",
                                           XLINKARCROLE: arcrole,
                                           XLINKFROM: locLabel,
                                           XLINKTO: footnoteLabel
                                       })

        currentAction = "done loading facts and footnotes"

        #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
        #               messageCode="loadFromExcel:info")
    except Exception as ex:
        if isinstance(ex, OIMException):
            modelXbrl.error(ex.code,
                            ex.message,
                            modelObject=modelXbrl,
                            **ex.msgArgs)
        else:
            modelXbrl.error(
                "arelleOIMloader:error",
                "Error while %(action)s, error %(error)s\ntraceback %(traceback)s",
                modelObject=modelXbrl,
                action=currentAction,
                error=ex,
                traceback=traceback.format_tb(sys.exc_info()[2]))

    if priorCWD:
        os.chdir(
            priorCWD
        )  # restore prior current working directory            startingErrorCount = len(modelXbrl.errors)

    if startingErrorCount < len(modelXbrl.errors):
        # had errors, don't allow ModelDocument.load to continue
        return OIMException("arelleOIMloader:unableToLoad",
                            "Unable to load due to reported errors")

    return getattr(modelXbrl, "modelDocument",
                   None)  # none if returning from exception