def loadXbrlFromDB(self, loadDBsaveToFile): # load from database modelXbrl = self.modelXbrl # find instance in DB instanceURI = os.path.basename(loadDBsaveToFile) results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant" " FROM dInstance WHERE FileName = '{}'" .format(instanceURI)) instanceId = moduleId = None for instanceId, moduleId, entScheme, entId, datePerEnd in results: break # find module in DB results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId)) xbrlSchemaRef = None for result in results: xbrlSchemaRef = result[0] break if not instanceId or not xbrlSchemaRef: raise XPDBException("sqlDB:MissingDTS", _("The instance and module were not found for %(instanceURI)"), instanceURI = instanceURI) if modelXbrl.skipDTS: # find prefixes and namespaces in DB results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]") dpmPrefixedNamespaces = dict((prefix, namespace) for owner, prefix, namespace in results) # create the instance document and resulting filing modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument(modelXbrl, Type.INSTANCE, loadDBsaveToFile, schemaRefs=[xbrlSchemaRef], isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement, 'xbrl-streamable-instance', 'version="1.0" contextBuffer="1"') # add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue) # filing indicator code IDs # get filing indicators results = self.execute("SELECT mToT.TemplateOrTableCode " " FROM dFilingIndicator dFI, mTemplateOrTable mToT " " WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID" .format(instanceId)) filingIndicatorCodes = [code[0] for code in results] if filingIndicatorCodes: modelXbrl.createContext(entScheme, entId, 'instant', None, datePerEnd, None, # no dimensional validity checking (like formula does) {}, [], [], id='c') filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators) for filingIndicatorCode in filingIndicatorCodes: modelXbrl.createFact(qnFindFilingIndicator, parent=filingIndicatorsTuple, attributes={"contextRef": "c"}, text=filingIndicatorCode) # facts in this instance factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards," " Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue " "FROM dFact WHERE InstanceID = {} " "ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL " " THEN DataPointSignature" " ELSE DataPointSignatureWithValuesForWildcards" " END, instr(DataPointSignature,'|') + 1)" .format(instanceId)) # results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal # get typed dimension values prefixedNamespaces = modelXbrl.prefixedNamespaces prefixedNamespaces["iso4217"] = XbrlConst.iso4217 if modelXbrl.skipDTS: prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed cntxTbl = {} # index by d unitTbl = {} def typedDimElt(s): # add xmlns into s for known qnames tag, angleBrkt, rest = s[1:].partition('>') text, angleBrkt, rest = rest.partition("<") qn = qname(tag, prefixedNamespaces) # a modelObject xml element is needed for all of the instance functions to manage the typed dim return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False) # contexts and facts for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl: metric, sep, dims = (dpSigTypedDims or dpSig).partition('|') conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces) concept = modelXbrl.qnameConcepts.get(conceptQn) isNumeric = isBool = isDateTime = isQName = isText = False if concept is not None: if concept.isNumeric: isNumeric = True else: baseXbrliType = concept.baseXbrliType if baseXbrliType == "booleanItemType": isBool = True elif baseXbrliType == "dateTimeItemType": # also is dateItemType? isDateTime = True elif baseXbrliType == "QNameItemType": isQName = True else: c = conceptQn.localName[0] if c == 'm': isNumeric = True elif c == 'd': isDateTime = True elif c == 'b': isBool = True elif c == 'e': isQName = True isText = not (isNumeric or isBool or isDateTime or isQName) if isinstance(datePerEnd, _STR_BASE): datePerEnd = datetimeValue(datePerEnd, addOneDay=True) cntxKey = (dims, entId, datePerEnd) if cntxKey in cntxTbl: cntxId = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) cntxTbl[cntxKey] = cntxId qnameDims = {} if dims: for dim in dims.split('|'): dQn, sep, dVal = dim[:-1].partition('(') dimQname = qname(dQn, prefixedNamespaces) if dVal.startswith('<'): mem = typedDimElt(dVal) # typed dim else: mem = qname(dVal, prefixedNamespaces) # explicit dim (even if treat-as-typed) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario") modelXbrl.createContext(entScheme, entId, 'instant', None, datePerEnd, None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) if unit: if unit in unitTbl: unitId = unitTbl[unit] else: unitQn = qname(unit, prefixedNamespaces) unitId = 'u{}'.format(unitQn.localName) unitTbl[unit] = unitId modelXbrl.createUnit([unitQn], [], id=unitId) else: unitId = None attrs = {"contextRef": cntxId} if unitId: attrs["unitRef"] = unitId if dec is not None: if isinstance(dec, float): # must be an integer dec = int(dec) elif isinstance(dec, _STR_BASE) and '.' in dec: dec = dec.partition('.')[0] # drop .0 from any SQLite string attrs["decimals"] = str(dec) # somehow it is float from the database if False: # fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None elif numVal is not None: num = roundValue(numVal, None, dec) # round using reported decimals if dec is None or dec == "INF": # show using decimals or reported format dec = len(numVal.partition(".")[2]) else: # max decimals at 28 dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num)) elif dateVal is not None: text = dateVal elif boolVal is not None: text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false' else: if isQName: # declare namespace addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces)) text = textVal modelXbrl.createFact(conceptQn, attributes=attrs, text=text) # add footnotes if any # save to file modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000) return modelXbrl.modelDocument
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri): from openpyxl import load_workbook from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname try: currentAction = "initializing" startedAt = time.time() if os.path.isabs(oimFile): # allow relative filenames to loading directory priorCWD = os.getcwd() os.chdir(os.path.dirname(oimFile)) else: priorCWD = None currentAction = "determining file type" isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json") isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json") isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls") isCSVorXL = isCSV or isXL instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl" if isJSON: currentAction = "loading and parsing JSON OIM file" with io.open(oimFile, 'rt', encoding='utf-8') as f: oimObject = json.load(f) missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject] if missing: raise OIMException("oime:missingJSONelements", _("The required elements %(missing)s {} in JSON input"), missing = ", ".join(missing)) currentAction = "identifying JSON objects" dtsReferences = oimObject["dtsReferences"] prefixes = oimObject["prefixes"] facts = oimObject["facts"] footnotes = oimOject["facts"] # shares this object elif isCSV: currentAction = "identifying CSV input tables" if sys.version[0] >= '3': csvOpenMode = 'w' csvOpenNewline = '' else: csvOpenMode = 'wb' # for 2.7 csvOpenNewline = None oimFileBase = None if "-facts" in oimFile: oimFileBase = oimFile.partition("-facts")[0] else: for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"): if oimFile.endswith(suffix): oimFileBase = oimFile[:-length(suffix)] break if oimFileBase is None: raise OIMException("oime:missingCSVtables", _("Unable to identify CSV tables file name pattern")) if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or not os.path.exists(oimFileBase + "-prefixes.csv")): raise OIMException("oime:missingCSVtables", _("Unable to identify CSV tables for dtsReferences or prefixes")) instanceFileName = oimFileBase + ".xbrl" currentAction = "loading CSV dtsReferences table" dtsReferences = [] with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: dtsReferences.append(dict((header[j], col) for j, col in enumerate(row))) currentAction = "loading CSV prefixes table" prefixes = {} with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = dict((col,i) for i,col in enumerate(row)) else: prefixes[row[header["prefix"]]] = row[header["URI"]] currentAction = "loading CSV defaults table" defaults = {} if os.path.exists(oimFileBase + "-defaults.csv"): with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row fileCol = row.index("file") else: defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol) currentAction = "loading CSV facts tables" facts = [] _dir = os.path.dirname(oimFileBase) for filename in os.listdir(_dir): filepath = os.path.join(_dir, filename) if "-facts" in filename and filepath.startswith(oimFileBase): tableDefaults = defaults.get(filename, {}) with io.open(filepath, 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col is not None: if header[j].endswith("Value"): if col: # ignore empty columns (= null CSV value) fact["value"] = col else: fact[header[j]] = col facts.append(fact) footnotes = [] if os.path.exists(oimFileBase + "-footnotes.csv"): with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col)) elif isXL: oimWb = load_workbook(oimFile, read_only=True, data_only=True) sheetNames = oimWb.get_sheet_names() if (not any(sheetName == "prefixes" for sheetName in sheetNames) or not any(sheetName == "dtsReferences" for sheetName in sheetNames) or not any("facts" in sheetName for sheetName in sheetNames)): if priorCWD: os.chdir(priorCWD) return None try: dtsReferences = [] for i, row in oimWb["dtsReferences"]: if i == 0: header = [col.value for col in row] else: dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row))) prefixes = {} for i, row in oimWb["dtsReferences"]: if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) else: prefixes[row[header["prefix"]].value] = row[header["URI"].value] defaults = {} for i, row in oimW.get("defaults", ()): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) fileCol = header["file"] else: defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol) facts = [] _dir = os.path.dirpart(oimFileBase) for sheetName in sheetNames: if sheetName == "facts" or "-facts" in sheetName: for i, row in oimWb[sheetName]: if i == 0: header = [col.value for col in row] else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col.value is not None: if header[j].endswith("Value"): fact["value"] = str(col.value) else: fact[header[j]] = str(col.value) facts.append(fact) footnotes = [] for i, row in oimWb.get("footnotes", ()): if i == 0: header = dict((col.value,i) for i,col in enumerate(row) if col.value) else: footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value)) except Exception as ex: if priorCWD: os.chdir(priorCWD) return None ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults # create the instance document modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument( modelXbrl, Type.INSTANCE, instanceFileName, schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"], isEntry=True, initialComment="extracted from OIM {}".format(mappedUri), documentEncoding="utf-8") cntxTbl = {} unitTbl = {} for fact in facts: conceptQn = qname(fact["oim:concept"], prefixes) concept = modelXbrl.qnameConcepts.get(conceptQn) entityAsQn = qname(fact["oim:entity"], prefixes) if "oim:period" in fact: periodStart = fact["oim:period"]["start"] periodEnd = fact["oim:period"]["end"] else: periodStart = fact["oim:periodStart"] periodEnd = fact["oim:periodEnd"] cntxKey = ( # hashable context key ("periodType", concept.periodType), ("entity", entityAsQn), ("periodStart", periodStart), ("periodEnd", periodEnd)) + tuple(sorted( (dimName, dimVal) for dimName, dimVal in fact.items() if ":" in dimName and not dimName.startswith("oim:"))) if cntxKey in cntxTbl: _cntx = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) qnameDims = {} for dimName, dimVal in fact.items(): if ":" in dimName and not dimName.startswith("oim:"): dimQname = qname(dimName, prefixes) dimConcept = modelXbrl.qnameConcepts.get(dimQname) if ":" in dimVal and dimVal.partition[':'][0] in prefixes: mem = qname(dimVal, prefixes) # explicit dim elif dimConcept.isTypedDimension: # a modelObject xml element is needed for all of the instance functions to manage the typed dim mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment") _cntx = modelXbrl.createContext( entityAsQn.namespaceURI, entityAsQn.localName, concept.periodType, None if concept.periodType == "instnat" else dateTime(periodStart, type=DATETIME), dateTime(periodEnd, type=DATETIME), None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) cntxTbl[cntxKey] = _cntx if "oim:unit" in fact: unitKey = fact["oim:unit"] if unitKey in unitTbl: _unit = unitTbl[unitKey] else: _mul, _sep, _div = unitKey.partition('/') if _mul.startswith('('): _mul = _mul[1:-1] mulQns = [qname(u, prefixes) for u in _mul.split('*') if u] if _div.startswith('('): _div = _div[1:-1] divQns = [qname(u, prefixes) for u in _div.split('*') if u] unitId = 'u-{:02}'.format(len(unitTbl) + 1) for _measures in mulQns, divQns: for _measure in _measures: addQnameValue(modelXbrl.modelDocument, _measure) _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId) unitTbl[unitKey] = _unit else: _unit = None attrs = {"contextRef": _cntx.id} if fact.get("value") is None: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact["value"] if fact.get("id"): attrs["id"] = fact["id"] if concept.isNumeric: if _unit is not None: attrs["unitRef"] = _unit.id if "accuracy" in fact: attrs["decimals"] = fact["accuracy"] # is value a QName? if concept.baseXbrliType == "QName": addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes)) f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text) footnoteLinks = {} # ELR elements factLocs = {} # index by (linkrole, factId) footnoteNbr = 0 locNbr = 0 for factOrFootnote in footnotes: if "factId" in factOrFootnote: factId = factOrFootnote["factId"] factFootnotes = (factOrFootnote,) # CSV or XL elif "id" in factOrFootnote and "footnotes" in factOrFootnote: factId = factOrFootnote["id"] factFootnotes = factOrFootnote["footnotes"] else: factFootnotes = () for footnote in factFootnotes: linkrole = footnote.get("group") arcrole = footnote.get("footnoteType") if not factId or not linkrole or not arcrole or not ( footnote.get("factRef") or footnote.get("footnote")): # invalid footnote continue if linkrole not in footnoteLinks: footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, XbrlConst.qnLinkFootnoteLink, attributes={"{http://www.w3.org/1999/xlink}type": "extended", "{http://www.w3.org/1999/xlink}role": linkrole}) footnoteLink = footnoteLinks[linkrole] if (linkrole, factId) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factId)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factId, XLINKLABEL: locLabel}) locLabel = factLocs[(linkrole, factId)] if footnote.get("footnote"): footnoteNbr += 1 footnoteLabel = "f_{:02}".format(footnoteNbr) attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel} if footnote.get("language"): attrs[XMLLANG] = footnote["language"] # note, for HTML will need to build an element structure addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"]) elif footnote.get("factRef"): factRef = footnote.get("factRef") if (linkrole, factRef) not in factLocs: locNbr += 1 locLabel = "f_{:02}".format(footnoteNbr) factLoc[(linkrole, factRef)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factRef, XLINKLABEL: locLabel}) footnoteLabel = factLoc[(linkrole, factId)] footnoteArc = addChild(footnoteLink, XbrlConst.qnLinkFootnoteArc, attributes={XLINKTYPE: "arc", XLINKARCROLE: arcrole, XLINKFROM: locLabel, XLINKTO: footnoteLabel}) #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") except Exception as ex: if ex is OIMException: modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) else: modelXbrl.error("Error while %(action)s, error %(error)s", modelObject=modelXbrl, action=currentAction, error=ex) if priorCWD: os.chdir(priorCWD) # restore prior current working directory return modelXbrl.modelDocument
def createModelFact(fact, parentModelFact, topTupleFact): aspects = fact.get("aspects", EMPTYDICT) if oimConcept not in aspects: modelXbrl.error("{}:conceptQName".format(errPrefix), _("The concept QName could not be determined"), modelObject=modelXbrl) return conceptQn = qname(aspects[oimConcept], prefixes) concept = modelXbrl.qnameConcepts.get(conceptQn) attrs = {} if concept.isItem: missingAspects = [] if oimEntity not in aspects: missingAspects.append(oimEntity) if oimPeriod not in aspects and (oimPeriodStart not in aspects or oimPeriodEnd not in aspects): missingAspects.append(oimPeriod) if missingAspects: modelXbrl.error("{}:missingAspects".format(errPrefix), _("The concept %(element)s is missing aspects %(missingAspects)s"), modelObject=modelXbrl, element=conceptQn, missingAspects=", ".join(missingAspects)) return entityAsQn = qname(aspects[oimEntity], prefixes) or qname("error",fact[oimEntity]) if oimPeriod in aspects: periodStart = periodEnd = aspects[oimPeriod] if oimPeriodStart in aspects and oimPeriodEnd in aspects: periodStart = aspects[oimPeriodStart] periodEnd = aspects[oimPeriodEnd] cntxKey = ( # hashable context key ("periodType", concept.periodType), ("entity", entityAsQn), ("periodStart", periodStart), ("periodEnd", periodEnd)) + tuple(sorted( (dimName, dimVal["value"] if isinstance(dimVal,dict) else dimVal) for dimName, dimVal in aspects.items() if ":" in dimName and not dimName.startswith(oimPrefix))) if cntxKey in cntxTbl: _cntx = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) qnameDims = {} for dimName, dimVal in aspects.items(): if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal: dimQname = qname(dimName, prefixes) dimConcept = modelXbrl.qnameConcepts.get(dimQname) if dimConcept is None: modelXbrl.error("{}:taxonomyDefinedAspectQName".format(errPrefix), _("The taxonomy defined aspect concept QName %(qname)s could not be determined"), modelObject=modelXbrl, qname=dimQname) continue if isinstance(dimVal, dict): dimVal = dimVal["value"] else: dimVal = str(dimVal) # may be int or boolean if isinstance(dimVal,str) and ":" in dimVal and dimVal.partition(':')[0] in prefixes: mem = qname(dimVal, prefixes) # explicit dim elif dimConcept.isTypedDimension: # a modelObject xml element is needed for all of the instance functions to manage the typed dim mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment") _cntx = modelXbrl.createContext( entityAsQn.namespaceURI, entityAsQn.localName, concept.periodType, None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME), dateTime(periodEnd, type=DATETIME), None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId, beforeSibling=topTupleFact) cntxTbl[cntxKey] = _cntx if oimUnit in aspects and concept.isNumeric: unitKey = aspects[oimUnit] if unitKey in unitTbl: _unit = unitTbl[unitKey] else: _unit = None # validate unit unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey) if not UnitPattern.match(unitKeySub): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation is lexically invalid, %(unit)s"), modelObject=modelXbrl, unit=unitKey) else: _mul, _sep, _div = unitKey.partition('/') if _mul.startswith('('): _mul = _mul[1:-1] _muls = [u for u in _mul.split('*') if u] if _div.startswith('('): _div = _div[1:-1] _divs = [u for u in _div.split('*') if u] if _muls != sorted(_muls) or _divs != sorted(_divs): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation measures are not in alphabetical order, %(unit)s"), modelObject=modelXbrl, unit=unitKey) try: mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _muls] divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _divs] unitId = 'u-{:02}'.format(len(unitTbl) + 1) for _measures in mulQns, divQns: for _measure in _measures: addQnameValue(modelXbrl.modelDocument, _measure) _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId, beforeSibling=topTupleFact) except OIMException as ex: modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) unitTbl[unitKey] = _unit else: _unit = None attrs["contextRef"] = _cntx.id if fact.get("value") is None: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact["value"] if concept.isNumeric: if _unit is None: return # skip creating fact because unit was invalid attrs["unitRef"] = _unit.id if "accuracy" in attrs or attrs.get(XbrlConst.qnXsiNil, "false") != "true": attrs["decimals"] = fact.get("accuracy", "INF") else: text = None #tuple id = fact.get("id") if id is not None: attrs["id"] = fact["id"] # is value a QName? if concept.baseXbrliType == "QName": addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes)) f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text, parent=parentModelFact, validate=False) if id is not None and id in parentedFacts: # create child facts for i in sorted(parentedFacts[id], key=lambda j: facts[j].get("aspects", EMPTYDICT).get(oimTupleOrder,0)): createModelFact(facts[i], f, f if topTupleFact is None else topTupleFact) # validate after creating tuple contents xmlValidate(modelXbrl, f)
def loadXbrlFromDB(self, loadDBsaveToFile): # load from database modelXbrl = self.modelXbrl # find instance in DB instanceURI = os.path.basename(loadDBsaveToFile) results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant" " FROM dInstance WHERE FileName = '{}'" .format(instanceURI)) instanceId = moduleId = None for instanceId, moduleId, entScheme, entId, datePerEnd in results: break # find module in DB results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId)) xbrlSchemaRef = None for result in results: xbrlSchemaRef = result[0] break if not instanceId or not xbrlSchemaRef: raise XPDBException("sqlDB:MissingDTS", _("The instance and module were not found for %(instanceURI)"), instanceURI = instanceURI) if modelXbrl.skipDTS: # find prefixes and namespaces in DB results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]") dpmPrefixedNamespaces = dict((prefix, namespace) for owner, prefix, namespace in results) # create the instance document and resulting filing modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument(modelXbrl, Type.INSTANCE, loadDBsaveToFile, schemaRefs=[xbrlSchemaRef], isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement, 'xbrl-streamable-instance', 'version="1.0" contextBuffer="1"') # add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue) # filing indicator code IDs # get filing indicators results = self.execute("SELECT mToT.TemplateOrTableCode " " FROM dFilingIndicator dFI, mTemplateOrTable mToT " " WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID" .format(instanceId)) filingIndicatorCodes = [code[0] for code in results] if filingIndicatorCodes: modelXbrl.createContext(entScheme, entId, 'instant', None, datePerEnd, None, # no dimensional validity checking (like formula does) {}, [], [], id='c') filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators) for filingIndicatorCode in filingIndicatorCodes: modelXbrl.createFact(qnFindFilingIndicator, parent=filingIndicatorsTuple, attributes={"contextRef": "c"}, text=filingIndicatorCode) # facts in this instance factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards," " Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue " "FROM dFact WHERE InstanceID = {} " "ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL " " THEN DataPointSignature" " ELSE DataPointSignatureWithValuesForWildcards" " END, instr(DataPointSignature,'|') + 1)" .format(instanceId)) # results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal # get typed dimension values prefixedNamespaces = modelXbrl.prefixedNamespaces prefixedNamespaces["iso4217"] = XbrlConst.iso4217 if modelXbrl.skipDTS: prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed cntxTbl = {} # index by d unitTbl = {} def typedDimElt(s): # add xmlns into s for known qnames tag, angleBrkt, rest = s[1:].partition('>') text, angleBrkt, rest = rest.partition("<") qn = qname(tag, prefixedNamespaces) # a modelObject xml element is needed for all of the instance functions to manage the typed dim return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False) # contexts and facts for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl: metric, sep, dims = (dpSigTypedDims or dpSig).partition('|') conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces) concept = modelXbrl.qnameConcepts.get(conceptQn) isNumeric = isBool = isDateTime = isQName = isText = False if concept is not None: if concept.isNumeric: isNumeric = True else: baseXbrliType = concept.baseXbrliType if baseXbrliType == "booleanItemType": isBool = True elif baseXbrliType == "dateTimeItemType": # also is dateItemType? isDateTime = True elif baseXbrliType == "QNameItemType": isQName = True else: c = conceptQn.localName[0] if c == 'm': isNumeric = True elif c == 'd': isDateTime = True elif c == 'b': isBool = True elif c == 'e': isQName = True isText = not (isNumeric or isBool or isDateTime or isQName) if isinstance(datePerEnd, _STR_BASE): datePerEnd = datetimeValue(datePerEnd, addOneDay=True) cntxKey = (dims, entId, datePerEnd) if cntxKey in cntxTbl: cntxId = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) cntxTbl[cntxKey] = cntxId qnameDims = {} if dims: for dim in dims.split('|'): dQn, sep, dVal = dim[:-1].partition('(') dimQname = qname(dQn, prefixedNamespaces) if dVal.startswith('<'): # typed dim mem = typedDimElt(dVal) else: mem = qname(dVal, prefixedNamespaces) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario") modelXbrl.createContext(entScheme, entId, 'instant', None, datePerEnd, None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) if unit: if unit in unitTbl: unitId = unitTbl[unit] else: unitQn = qname(unit, prefixedNamespaces) unitId = 'u{}'.format(unitQn.localName) unitTbl[unit] = unitId modelXbrl.createUnit([unitQn], [], id=unitId) else: unitId = None attrs = {"contextRef": cntxId} if unitId: attrs["unitRef"] = unitId if dec is not None: if isinstance(dec, float): # must be an integer dec = int(dec) elif isinstance(dec, _STR_BASE) and '.' in dec: dec = dec.partition('.')[0] # drop .0 from any SQLite string attrs["decimals"] = str(dec) # somehow it is float from the database if False: # fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None elif numVal is not None: num = roundValue(numVal, None, dec) # round using reported decimals if dec is None or dec == "INF": # show using decimals or reported format dec = len(numVal.partition(".")[2]) else: # max decimals at 28 dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num)) elif dateVal is not None: text = dateVal elif boolVal is not None: text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false' else: if isQName: # declare namespace addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces)) text = textVal modelXbrl.createFact(conceptQn, attributes=attrs, text=text) # add footnotes if any # save to file modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000) return modelXbrl.modelDocument
def yAxisByCol(self, leftCol, row, yParentStructuralNode, childrenFirst, renderNow, atTop): if yParentStructuralNode is not None: nestedBottomRow = row for yStructuralNode in yParentStructuralNode.childStructuralNodes: nestRow, nextRow = self.yAxisByCol(leftCol + 1, row, yStructuralNode, # nested items before totals childrenFirst, childrenFirst, False) isAbstract = (yStructuralNode.isAbstract or (yStructuralNode.childStructuralNodes and not isinstance(yStructuralNode.definitionNode, (ModelClosedDefinitionNode, ModelEuAxisCoord)))) isNonAbstract = not isAbstract label = yStructuralNode.header(lang=self.lang, returnGenLabel=isinstance(yStructuralNode.definitionNode, (ModelClosedDefinitionNode, ModelEuAxisCoord))) topRow = row if childrenFirst and isNonAbstract: row = nextRow #print ( "thisCol {0} leftCol {1} rightCol {2} topRow{3} renderNow {4} label {5}".format(thisCol, leftCol, rightCol, topRow, renderNow, label)) if renderNow: rowspan= nestRow - row + 1 cellElt = etree.Element(tableModelQName("cell"), attrib={"span": str(rowspan)} if rowspan > 1 else None) elt = etree.SubElement(cellElt, tableModelQName("label")) elt.text = label self.rowHdrElts[leftCol - 1].append(cellElt) self.structuralNodeModelElements.append((yStructuralNode, cellElt)) for aspect in sorted(yStructuralNode.aspectsCovered(), key=lambda a: aspectStr(a)): if yStructuralNode.hasAspect(aspect) and aspect != Aspect.DIMENSIONS: aspectValue = yStructuralNode.aspectValue(aspect) if aspectValue is None: aspectValue = "(bound dynamically)" elt = etree.SubElement(cellElt, tableModelQName("constraint")) etree.SubElement(elt, tableModelQName("aspect") ).text = aspectStr(aspect) etree.SubElement(elt, tableModelQName("value") ).text = addQnameValue(self.xmlDoc, aspectValue) for rollUpCol in range(leftCol, self.rowHdrCols - 1): rollUpElt = etree.Element(tableModelQName("cell"), attrib={"rollup":"true"}) self.rowHdrElts[rollUpCol].append(rollUpElt) if isNonAbstract: cellElt = etree.Element(tableModelQName("cell"), attrib={"span": str(rowspan)} if rowspan > 1 else None) for i, role in enumerate(self.rowHdrNonStdRoles): labelElt = etree.SubElement(cellElt, tableModelQName("label"), attrib={"role":role, "lang":self.lang}) labelElt.text = yStructuralNode.header(role=role, lang=self.lang) self.rowHdrElts[self.rowHdrCols - 1 + i].append(cellElt) for aspect in sorted(yStructuralNode.aspectsCovered(), key=lambda a: aspectStr(a)): if yStructuralNode.hasAspect(aspect) and aspect != Aspect.DIMENSIONS: aspectValue = yStructuralNode.aspectValue(aspect) if aspectValue is None: aspectValue = "(bound dynamically)" elt = etree.SubElement(cellElt, tableModelQName("constraint")) etree.SubElement(elt, tableModelQName("aspect") ).text = aspectStr(aspect) etree.SubElement(elt, tableModelQName("value") ).text = addQnameValue(self.xmlDoc, aspectValue) ''' if self.rowHdrDocCol: labelElt = etree.SubElement(cellElt, tableModelQName("label"), attrib={"span": str(rowspan)} if rowspan > 1 else None) elt.text = yStructuralNode.header(role="http://www.xbrl.org/2008/role/documentation", lang=self.lang) self.rowHdrElts[self.rowHdrCols - 1].append(elt) if self.rowHdrCodeCol: elt = etree.Element(tableModelQName("label"), attrib={"span": str(rowspan)} if rowspan > 1 else None) elt.text = yStructuralNode.header(role="http://www.eurofiling.info/role/2010/coordinate-code", lang=self.lang) self.rowHdrElts[self.rowHdrCols - 1 + self.rowHdrDocCol].append(elt) ''' if isNonAbstract: row += 1 elif childrenFirst: row = nextRow if nestRow > nestedBottomRow: nestedBottomRow = nestRow + (isNonAbstract and not childrenFirst) if row > nestedBottomRow: nestedBottomRow = row #if renderNow and not childrenFirst: # dummy, row = self.yAxis(leftCol + 1, row, yStructuralNode, childrenFirst, True, False) # render on this pass if not childrenFirst: dummy, row = self.yAxisByCol(leftCol + 1, row, yStructuralNode, childrenFirst, renderNow, False) # render on this pass return (nestedBottomRow, row)
def xAxis(self, leftCol, topRow, rowBelow, xParentStructuralNode, xStructuralNodes, childrenFirst, renderNow, atTop): if xParentStructuralNode is not None: parentRow = rowBelow noDescendants = True rightCol = leftCol widthToSpanParent = 0 sideBorder = not xStructuralNodes for xStructuralNode in xParentStructuralNode.childStructuralNodes: noDescendants = False rightCol, row, width, leafNode = self.xAxis(leftCol, topRow + 1, rowBelow, xStructuralNode, xStructuralNodes, # nested items before totals childrenFirst, childrenFirst, False) if row - 1 < parentRow: parentRow = row - 1 #if not leafNode: # rightCol -= 1 nonAbstract = not xStructuralNode.isAbstract if nonAbstract: width += 100 # width for this label widthToSpanParent += width label = xStructuralNode.header(lang=self.lang, returnGenLabel=isinstance(xStructuralNode.definitionNode, (ModelClosedDefinitionNode, ModelEuAxisCoord))) if childrenFirst: thisCol = rightCol else: thisCol = leftCol #print ( "thisCol {0} leftCol {1} rightCol {2} topRow{3} renderNow {4} label {5}".format(thisCol, leftCol, rightCol, topRow, renderNow, label)) if renderNow: columnspan = rightCol - leftCol + (1 if nonAbstract else 0) if self.type == HTML: if rightCol == self.dataFirstCol + self.dataCols - 1: edgeBorder = "border-right:.5pt solid windowtext;" else: edgeBorder = "" attrib = {"class":"xAxisHdr", "style":"text-align:center;max-width:{0}pt;{1}".format(width,edgeBorder)} if columnspan > 1: attrib["colspan"] = str(columnspan) if leafNode and row > topRow: attrib["rowspan"] = str(row - topRow + 1) elt = etree.Element("{http://www.w3.org/1999/xhtml}th", attrib=attrib) self.rowElts[topRow-1].insert(leftCol,elt) elif self.type == XML: cellElt = etree.Element(tableModelQName("cell"), attrib={"span": str(columnspan)} if columnspan > 1 else None) self.colHdrElts[topRow - self.colHdrTopRow].insert(leftCol,cellElt) self.structuralNodeModelElements.append((xStructuralNode, cellElt)) elt = etree.SubElement(cellElt, tableModelQName("label")) if nonAbstract or (leafNode and row > topRow): for rollUpCol in range(topRow - self.colHdrTopRow + 1, self.colHdrRows - 1): rollUpElt = etree.Element(tableModelQName("label"), attrib={"rollup":"true"}) if childrenFirst: self.colHdrElts[rollUpCol].append(rollUpElt) else: self.colHdrElts[rollUpCol].insert(leftCol,rollUpElt) for i, role in enumerate(self.colHdrNonStdRoles): etree.SubElement(cellElt, tableModelQName("label"), attrib={"role": role, "lang": self.lang} ).text = xStructuralNode.header( role=role, lang=self.lang) for aspect in sorted(xStructuralNode.aspectsCovered(), key=lambda a: aspectStr(a)): if xStructuralNode.hasAspect(aspect) and aspect != Aspect.DIMENSIONS: aspectValue = xStructuralNode.aspectValue(aspect) if aspectValue is None: aspectValue = "(bound dynamically)" aspElt = etree.SubElement(cellElt, tableModelQName("constraint")) etree.SubElement(aspElt, tableModelQName("aspect") ).text = aspectStr(aspect) etree.SubElement(aspElt, tableModelQName("value") ).text = addQnameValue(self.xmlDoc, aspectValue) elt.text = label or "\u00A0" #produces if nonAbstract: if columnspan > 1 and rowBelow > topRow: # add spanned left leg portion one row down if self.type == HTML: attrib= {"class":"xAxisSpanLeg", "rowspan": str(rowBelow - row)} if edgeBorder: attrib["style"] = edgeBorder elt = etree.Element("{http://www.w3.org/1999/xhtml}th", attrib=attrib) elt.text = "\u00A0" if childrenFirst: self.rowElts[topRow].append(elt) else: self.rowElts[topRow].insert(leftCol,elt) if self.type == HTML: for i, role in enumerate(self.colHdrNonStdRoles): elt = etree.Element("{http://www.w3.org/1999/xhtml}th", attrib={"class":"xAxisHdr", "style":"text-align:center;max-width:100pt;{0}".format(edgeBorder)}) self.rowElts[self.dataFirstRow - 1 - len(self.colHdrNonStdRoles) + i].insert(thisCol,elt) elt.text = xStructuralNode.header(role=role, lang=self.lang) or "\u00A0" ''' if self.colHdrDocRow: doc = xStructuralNode.header(role="http://www.xbrl.org/2008/role/documentation", lang=self.lang) if self.type == HTML: elt = etree.Element("{http://www.w3.org/1999/xhtml}th", attrib={"class":"xAxisHdr", "style":"text-align:center;max-width:100pt;{0}".format(edgeBorder)}) self.rowElts[self.dataFirstRow - 2 - self.rowHdrCodeCol].insert(thisCol,elt) elif self.type == XML: elt = etree.Element(tableModelQName("label")) self.colHdrElts[self.colHdrRows - 1].insert(thisCol,elt) elt.text = doc or "\u00A0" if self.colHdrCodeRow: code = xStructuralNode.header(role="http://www.eurofiling.info/role/2010/coordinate-code") if self.type == HTML: elt = etree.Element("{http://www.w3.org/1999/xhtml}th", attrib={"class":"xAxisHdr", "style":"text-align:center;max-width:100pt;{0}".format(edgeBorder)}) self.rowElts[self.dataFirstRow - 2].insert(thisCol,elt) elif self.type == XML: elt = etree.Element(tableModelQName("label")) self.colHdrElts[self.colHdrRows - 1 + self.colHdrDocRow].insert(thisCol,elt) elt.text = code or "\u00A0" ''' xStructuralNodes.append(xStructuralNode) if nonAbstract: rightCol += 1 if renderNow and not childrenFirst: self.xAxis(leftCol + (1 if nonAbstract else 0), topRow + 1, rowBelow, xStructuralNode, xStructuralNodes, childrenFirst, True, False) # render on this pass leftCol = rightCol return (rightCol, parentRow, widthToSpanParent, noDescendants)
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri): from openpyxl import load_workbook from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname try: currentAction = "initializing" startingErrorCount = len(modelXbrl.errors) startedAt = time.time() if os.path.isabs(oimFile): # allow relative filenames to loading directory priorCWD = os.getcwd() os.chdir(os.path.dirname(oimFile)) else: priorCWD = None currentAction = "determining file type" isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json") isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json") isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls") isCSVorXL = isCSV or isXL instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl" if isJSON: currentAction = "loading and parsing JSON OIM file" def loadDict(keyValuePairs): _dict = OrderedDict() # preserve fact order in resulting instance for key, value in keyValuePairs: if isinstance(value, dict): if DUPJSONKEY in value: for _errKey, _errValue, _otherValue in value[DUPJSONKEY]: if key == "prefixes": modelXbrl.error("oime:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"), modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue) del value[DUPJSONKEY] if key in _dict: if DUPJSONKEY not in _dict: _dict[DUPJSONKEY] = [] _dict[DUPJSONKEY].append((key, value, _dict[key])) else: _dict[key] = value return _dict with io.open(oimFile, 'rt', encoding='utf-8') as f: oimObject = json.load(f, object_pairs_hook=loadDict) missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject] if missing: raise OIMException("oime:missingJSONElements", _("Required element(s) are missing from JSON input: %(missing)s"), missing = ", ".join(missing)) currentAction = "identifying JSON objects" dtsReferences = oimObject["dtsReferences"] prefixesList = oimObject["prefixes"].items() facts = oimObject["facts"] footnotes = oimObject["facts"] # shares this object elif isCSV: currentAction = "identifying CSV input tables" if sys.version[0] >= '3': csvOpenMode = 'w' csvOpenNewline = '' else: csvOpenMode = 'wb' # for 2.7 csvOpenNewline = None oimFileBase = None if "-facts" in oimFile: oimFileBase = oimFile.partition("-facts")[0] else: for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"): if oimFile.endswith(suffix): oimFileBase = oimFile[:-len(suffix)] break if oimFileBase is None: raise OIMException("oime:missingCSVTables", _("Unable to identify CSV tables file name pattern")) if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or not os.path.exists(oimFileBase + "-prefixes.csv")): raise OIMException("oime:missingCSVTables", _("Unable to identify CSV tables for dtsReferences or prefixes")) instanceFileName = oimFileBase + ".xbrl" currentAction = "loading CSV dtsReferences table" dtsReferences = [] with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: dtsReferences.append(dict((header[j], col) for j, col in enumerate(row))) currentAction = "loading CSV prefixes table" prefixesList = [] with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = dict((col,i) for i,col in enumerate(row)) else: prefixesList.append((row[header["prefix"]], row[header["URI"]])) defaults = {} if os.path.exists(oimFileBase + "-defaults.csv"): currentAction = "loading CSV defaults table" with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row fileCol = row.index("file") else: defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol) currentAction = "loading CSV facts tables" facts = [] _dir = os.path.dirname(oimFileBase) factsFileBasename = os.path.basename(oimFileBase) + "-facts" for filename in os.listdir(_dir): filepath = os.path.join(_dir, filename) if filename.startswith(factsFileBasename): currentAction = "loading CSV facts table {}".format(filename) tableDefaults = defaults.get(filename, {}) with io.open(filepath, 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col is not None: if header[j]: # skip cols with no header if header[j].endswith("Value"): if col: # ignore empty columns (= null CSV value) fact["value"] = col else: fact[header[j]] = col facts.append(fact) footnotes = [] if os.path.exists(oimFileBase + "-footnotes.csv"): currentAction = "loading CSV footnotes table" with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col)) elif isXL: currentAction = "identifying workbook input worksheets" oimWb = load_workbook(oimFile, read_only=True, data_only=True) sheetNames = oimWb.get_sheet_names() if (not any(sheetName == "prefixes" for sheetName in sheetNames) or not any(sheetName == "dtsReferences" for sheetName in sheetNames) or not any("facts" in sheetName for sheetName in sheetNames)): raise OIMException("oime:missingWorkbookWorksheets", _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts")) currentAction = "loading worksheet: dtsReferences" dtsReferences = [] for i, row in enumerate(oimWb["dtsReferences"]): if i == 0: header = [col.value for col in row] else: dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row))) currentAction = "loading worksheet: prefixes" prefixesList = [] for i, row in enumerate(oimWb["prefixes"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) else: prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value)) defaults = {} if "defaults" in sheetNames: currentAction = "loading worksheet: defaults" for i, row in enumerate(oimWb["defaults"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) fileCol = header["file"] else: defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol) facts = [] for sheetName in sheetNames: if sheetName == "facts" or "-facts" in sheetName: currentAction = "loading worksheet: {}".format(sheetName) tableDefaults = defaults.get(sheetName, {}) for i, row in enumerate(oimWb[sheetName]): if i == 0: header = [col.value for col in row] else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col.value is not None: if header[j]: # skip cols with no header if header[j].endswith("Value"): fact["value"] = str(col.value) else: fact[header[j]] = str(col.value) facts.append(fact) footnotes = [] if "footnotes" in sheetNames: currentAction = "loading worksheet: footnotes" for i, row in enumerate(oimWb["footnotes"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row) if col.value) else: footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value)) currentAction = "identifying default dimensions" ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults currentAction = "validating OIM" prefixes = {} prefixedUris = {} for _prefix, _uri in prefixesList: if not _prefix: modelXbrl.error("oime:emptyPrefix", _("The empty string must not be used as a prefix, uri %(uri)s"), modelObject=modelXbrl, uri=_uri) elif not NCNamePattern.match(_prefix): modelXbrl.error("oime:prefixPattern", _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"), modelObject=modelXbrl, prefix=_prefix, uri=_uri) elif _prefix in prefixes: modelXbrl.error("oime:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"), modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri) elif _uri in prefixedUris: modelXbrl.error("oime:duplicatedUri", _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"), modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix) else: prefixes[_prefix] = _uri prefixedUris[_uri] = _prefix oimPrefix = None for _nsOim in nsOim: if _nsOim in prefixedUris: oimPrefix = prefixedUris[_nsOim] if not oimPrefix: raise OIMException("oime:noOimPrefix", _("The oim namespace must have a declared prefix")) oimConcept = "{}:concept".format(oimPrefix) oimEntity = "{}:entity".format(oimPrefix) oimPeriod = "{}:period".format(oimPrefix) oimPeriodStart = "{}:periodStart".format(oimPrefix) oimPeriodEnd = "{}:periodEnd".format(oimPrefix) oimUnit = "{}:unit".format(oimPrefix) oimPrefix = "{}:".format(oimPrefix) # create the instance document currentAction = "creating instance document" modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument( modelXbrl, Type.INSTANCE, instanceFileName, schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"], isEntry=True, initialComment="extracted from OIM {}".format(mappedUri), documentEncoding="utf-8") cntxTbl = {} unitTbl = {} currentAction = "creating facts" for fact in facts: conceptQn = qname(fact[oimConcept], prefixes) concept = modelXbrl.qnameConcepts.get(conceptQn) entityAsQn = qname(fact[oimEntity], prefixes) if oimPeriod in fact: periodStart = fact[oimPeriod]["start"] periodEnd = fact[oimPeriod]["end"] else: periodStart = fact[oimPeriodStart] periodEnd = fact[oimPeriodEnd] cntxKey = ( # hashable context key ("periodType", concept.periodType), ("entity", entityAsQn), ("periodStart", periodStart), ("periodEnd", periodEnd)) + tuple(sorted( (dimName, dimVal) for dimName, dimVal in fact.items() if ":" in dimName and not dimName.startswith(oimPrefix))) if cntxKey in cntxTbl: _cntx = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) qnameDims = {} for dimName, dimVal in fact.items(): if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal: dimQname = qname(dimName, prefixes) dimConcept = modelXbrl.qnameConcepts.get(dimQname) if ":" in dimVal and dimVal.partition(':')[0] in prefixes: mem = qname(dimVal, prefixes) # explicit dim elif dimConcept.isTypedDimension: # a modelObject xml element is needed for all of the instance functions to manage the typed dim mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment") _cntx = modelXbrl.createContext( entityAsQn.namespaceURI, entityAsQn.localName, concept.periodType, None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME), dateTime(periodEnd, type=DATETIME), None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) cntxTbl[cntxKey] = _cntx if oimUnit in fact: unitKey = fact[oimUnit] if unitKey in unitTbl: _unit = unitTbl[unitKey] else: _unit = None # validate unit unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey) if not UnitPattern.match(unitKeySub): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation is lexically invalid, %(unit)s"), modelObject=modelXbrl, unit=unitKey) else: _mul, _sep, _div = unitKey.partition('/') if _mul.startswith('('): _mul = _mul[1:-1] _muls = [u for u in _mul.split('*') if u] if _div.startswith('('): _div = _div[1:-1] _divs = [u for u in _div.split('*') if u] if _muls != sorted(_muls) or _divs != sorted(_divs): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation measures are not in alphabetical order, %(unit)s"), modelObject=modelXbrl, unit=unitKey) try: mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _muls] divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _divs] unitId = 'u-{:02}'.format(len(unitTbl) + 1) for _measures in mulQns, divQns: for _measure in _measures: addQnameValue(modelXbrl.modelDocument, _measure) _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId) except OIMException as ex: modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) unitTbl[unitKey] = _unit else: _unit = None attrs = {"contextRef": _cntx.id} if fact.get("value") is None: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact["value"] if fact.get("id"): attrs["id"] = fact["id"] if concept.isNumeric: if _unit is None: continue # skip creating fact because unit was invalid attrs["unitRef"] = _unit.id if "accuracy" in fact: attrs["decimals"] = fact["accuracy"] # is value a QName? if concept.baseXbrliType == "QName": addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes)) f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text) currentAction = "creating footnotes" footnoteLinks = {} # ELR elements factLocs = {} # index by (linkrole, factId) footnoteNbr = 0 locNbr = 0 for factOrFootnote in footnotes: if "factId" in factOrFootnote: factId = factOrFootnote["factId"] factFootnotes = (factOrFootnote,) # CSV or XL elif "id" in factOrFootnote and "footnotes" in factOrFootnote: factId = factOrFootnote["id"] factFootnotes = factOrFootnote["footnotes"] else: factFootnotes = () for footnote in factFootnotes: linkrole = footnote.get("group") arcrole = footnote.get("footnoteType") if not factId or not linkrole or not arcrole or not ( footnote.get("factRef") or footnote.get("footnote")): # invalid footnote continue if linkrole not in footnoteLinks: footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, XbrlConst.qnLinkFootnoteLink, attributes={"{http://www.w3.org/1999/xlink}type": "extended", "{http://www.w3.org/1999/xlink}role": linkrole}) footnoteLink = footnoteLinks[linkrole] if (linkrole, factId) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factId)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factId, XLINKLABEL: locLabel}) locLabel = factLocs[(linkrole, factId)] if footnote.get("footnote"): footnoteNbr += 1 footnoteLabel = "f_{:02}".format(footnoteNbr) attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel} if footnote.get("language"): attrs[XMLLANG] = footnote["language"] # note, for HTML will need to build an element structure addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"]) elif footnote.get("factRef"): factRef = footnote.get("factRef") if (linkrole, factRef) not in factLocs: locNbr += 1 locLabel = "f_{:02}".format(footnoteNbr) factLoc[(linkrole, factRef)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factRef, XLINKLABEL: locLabel}) footnoteLabel = factLoc[(linkrole, factId)] footnoteArc = addChild(footnoteLink, XbrlConst.qnLinkFootnoteArc, attributes={XLINKTYPE: "arc", XLINKARCROLE: arcrole, XLINKFROM: locLabel, XLINKTO: footnoteLabel}) currentAction = "done loading facts and footnotes" #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") except Exception as ex: if isinstance(ex, OIMException): modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) else: modelXbrl.error("arelleOIMloader:error", "Error while %(action)s, error %(error)s\ntraceback %(traceback)s", modelObject=modelXbrl, action=currentAction, error=ex, traceback=traceback.format_tb(sys.exc_info()[2])) if priorCWD: os.chdir(priorCWD) # restore prior current working directory startingErrorCount = len(modelXbrl.errors) if startingErrorCount < len(modelXbrl.errors): # had errors, don't allow ModelDocument.load to continue return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors") return getattr(modelXbrl, "modelDocument", None) # none if returning from exception
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri): from openpyxl import load_workbook from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname try: currentAction = "initializing" startingErrorCount = len(modelXbrl.errors) startedAt = time.time() if os.path.isabs(oimFile): # allow relative filenames to loading directory priorCWD = os.getcwd() os.chdir(os.path.dirname(oimFile)) else: priorCWD = None currentAction = "determining file type" isJSON = oimFile.endswith( ".json") and not oimFile.endswith("-metadata.json") isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json") isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls") isCSVorXL = isCSV or isXL instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl" if isJSON: currentAction = "loading and parsing JSON OIM file" def loadDict(keyValuePairs): _dict = OrderedDict( ) # preserve fact order in resulting instance for key, value in keyValuePairs: if isinstance(value, dict): if DUPJSONKEY in value: for _errKey, _errValue, _otherValue in value[ DUPJSONKEY]: if key == "prefixes": modelXbrl.error( "oime:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s" ), modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue) del value[DUPJSONKEY] if key in _dict: if DUPJSONKEY not in _dict: _dict[DUPJSONKEY] = [] _dict[DUPJSONKEY].append((key, value, _dict[key])) else: _dict[key] = value return _dict with io.open(oimFile, 'rt', encoding='utf-8') as f: oimObject = json.load(f, object_pairs_hook=loadDict) missing = [ t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject ] if missing: raise OIMException( "oime:missingJSONElements", _("Required element(s) are missing from JSON input: %(missing)s" ), missing=", ".join(missing)) currentAction = "identifying JSON objects" dtsReferences = oimObject["dtsReferences"] prefixesList = oimObject["prefixes"].items() facts = oimObject["facts"] footnotes = oimObject["facts"] # shares this object elif isCSV: currentAction = "identifying CSV input tables" if sys.version[0] >= '3': csvOpenMode = 'w' csvOpenNewline = '' else: csvOpenMode = 'wb' # for 2.7 csvOpenNewline = None oimFileBase = None if "-facts" in oimFile: oimFileBase = oimFile.partition("-facts")[0] else: for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"): if oimFile.endswith(suffix): oimFileBase = oimFile[:-len(suffix)] break if oimFileBase is None: raise OIMException( "oime:missingCSVTables", _("Unable to identify CSV tables file name pattern")) if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or not os.path.exists(oimFileBase + "-prefixes.csv")): raise OIMException( "oime:missingCSVTables", _("Unable to identify CSV tables for dtsReferences or prefixes" )) instanceFileName = oimFileBase + ".xbrl" currentAction = "loading CSV dtsReferences table" dtsReferences = [] with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: dtsReferences.append( dict( (header[j], col) for j, col in enumerate(row))) currentAction = "loading CSV prefixes table" prefixesList = [] with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = dict((col, i) for i, col in enumerate(row)) else: prefixesList.append( (row[header["prefix"]], row[header["URI"]])) defaults = {} if os.path.exists(oimFileBase + "-defaults.csv"): currentAction = "loading CSV defaults table" with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row fileCol = row.index("file") else: defaults[row[fileCol]] = dict( (header[j], col) for j, col in enumerate(row) if j != fileCol) currentAction = "loading CSV facts tables" facts = [] _dir = os.path.dirname(oimFileBase) factsFileBasename = os.path.basename(oimFileBase) + "-facts" for filename in os.listdir(_dir): filepath = os.path.join(_dir, filename) if filename.startswith(factsFileBasename): currentAction = "loading CSV facts table {}".format( filename) tableDefaults = defaults.get(filename, {}) with io.open(filepath, 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col is not None: if header[ j]: # skip cols with no header if header[j].endswith("Value"): if col: # ignore empty columns (= null CSV value) fact["value"] = col else: fact[header[j]] = col facts.append(fact) footnotes = [] if os.path.exists(oimFileBase + "-footnotes.csv"): currentAction = "loading CSV footnotes table" with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: footnotes.append( dict((header[j], col) for j, col in enumerate(row) if col)) elif isXL: currentAction = "identifying workbook input worksheets" oimWb = load_workbook(oimFile, read_only=True, data_only=True) sheetNames = oimWb.get_sheet_names() if (not any(sheetName == "prefixes" for sheetName in sheetNames) or not any(sheetName == "dtsReferences" for sheetName in sheetNames) or not any("facts" in sheetName for sheetName in sheetNames)): raise OIMException( "oime:missingWorkbookWorksheets", _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts" )) currentAction = "loading worksheet: dtsReferences" dtsReferences = [] for i, row in enumerate(oimWb["dtsReferences"]): if i == 0: header = [col.value for col in row] else: dtsReferences.append( dict((header[j], col.value) for j, col in enumerate(row))) currentAction = "loading worksheet: prefixes" prefixesList = [] for i, row in enumerate(oimWb["prefixes"]): if i == 0: header = dict((col.value, i) for i, col in enumerate(row)) else: prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value)) defaults = {} if "defaults" in sheetNames: currentAction = "loading worksheet: defaults" for i, row in enumerate(oimWb["defaults"]): if i == 0: header = dict( (col.value, i) for i, col in enumerate(row)) fileCol = header["file"] else: defaults[row[fileCol].value] = dict( (header[j], col.value) for j, col in enumerate(row) if j != fileCol) facts = [] for sheetName in sheetNames: if sheetName == "facts" or "-facts" in sheetName: currentAction = "loading worksheet: {}".format(sheetName) tableDefaults = defaults.get(sheetName, {}) for i, row in enumerate(oimWb[sheetName]): if i == 0: header = [col.value for col in row] else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col.value is not None: if header[j]: # skip cols with no header if header[j].endswith("Value"): fact["value"] = str(col.value) else: fact[header[j]] = str(col.value) facts.append(fact) footnotes = [] if "footnotes" in sheetNames: currentAction = "loading worksheet: footnotes" for i, row in enumerate(oimWb["footnotes"]): if i == 0: header = dict((col.value, i) for i, col in enumerate(row) if col.value) else: footnotes.append( dict((header[j], col.value) for j, col in enumerate(row) if col.value)) currentAction = "identifying default dimensions" ValidateXbrlDimensions.loadDimensionDefaults( modelXbrl) # needs dimension defaults currentAction = "validating OIM" prefixes = {} prefixedUris = {} for _prefix, _uri in prefixesList: if not _prefix: modelXbrl.error( "oime:emptyPrefix", _("The empty string must not be used as a prefix, uri %(uri)s" ), modelObject=modelXbrl, uri=_uri) elif not NCNamePattern.match(_prefix): modelXbrl.error( "oime:prefixPattern", _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s" ), modelObject=modelXbrl, prefix=_prefix, uri=_uri) elif _prefix in prefixes: modelXbrl.error( "oime:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s" ), modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri) elif _uri in prefixedUris: modelXbrl.error( "oime:duplicatedUri", _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s" ), modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix) else: prefixes[_prefix] = _uri prefixedUris[_uri] = _prefix oimPrefix = None for _nsOim in nsOim: if _nsOim in prefixedUris: oimPrefix = prefixedUris[_nsOim] if not oimPrefix: raise OIMException( "oime:noOimPrefix", _("The oim namespace must have a declared prefix")) oimConcept = "{}:concept".format(oimPrefix) oimEntity = "{}:entity".format(oimPrefix) oimPeriod = "{}:period".format(oimPrefix) oimPeriodStart = "{}:periodStart".format(oimPrefix) oimPeriodEnd = "{}:periodEnd".format(oimPrefix) oimUnit = "{}:unit".format(oimPrefix) oimPrefix = "{}:".format(oimPrefix) # create the instance document currentAction = "creating instance document" modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument( modelXbrl, Type.INSTANCE, instanceFileName, schemaRefs=[ dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema" ], isEntry=True, initialComment="extracted from OIM {}".format(mappedUri), documentEncoding="utf-8") cntxTbl = {} unitTbl = {} currentAction = "creating facts" for fact in facts: conceptQn = qname(fact[oimConcept], prefixes) concept = modelXbrl.qnameConcepts.get(conceptQn) entityAsQn = qname(fact[oimEntity], prefixes) if oimPeriod in fact: periodStart = fact[oimPeriod]["start"] periodEnd = fact[oimPeriod]["end"] else: periodStart = fact[oimPeriodStart] periodEnd = fact[oimPeriodEnd] cntxKey = ( # hashable context key ("periodType", concept.periodType), ("entity", entityAsQn), ("periodStart", periodStart), ("periodEnd", periodEnd)) + tuple( sorted( (dimName, dimVal) for dimName, dimVal in fact.items() if ":" in dimName and not dimName.startswith(oimPrefix))) if cntxKey in cntxTbl: _cntx = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) qnameDims = {} for dimName, dimVal in fact.items(): if ":" in dimName and not dimName.startswith( oimPrefix) and dimVal: dimQname = qname(dimName, prefixes) dimConcept = modelXbrl.qnameConcepts.get(dimQname) if ":" in dimVal and dimVal.partition( ':')[0] in prefixes: mem = qname(dimVal, prefixes) # explicit dim elif dimConcept.isTypedDimension: # a modelObject xml element is needed for all of the instance functions to manage the typed dim mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False) qnameDims[dimQname] = DimValuePrototype( modelXbrl, None, dimQname, mem, "segment") _cntx = modelXbrl.createContext( entityAsQn.namespaceURI, entityAsQn.localName, concept.periodType, None if concept.periodType == "instant" else dateTime( periodStart, type=DATETIME), dateTime(periodEnd, type=DATETIME), None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) cntxTbl[cntxKey] = _cntx if oimUnit in fact: unitKey = fact[oimUnit] if unitKey in unitTbl: _unit = unitTbl[unitKey] else: _unit = None # validate unit unitKeySub = PrefixedQName.sub( UnitPrefixedQNameSubstitutionChar, unitKey) if not UnitPattern.match(unitKeySub): modelXbrl.error( "oime:unitStringRepresentation", _("Unit string representation is lexically invalid, %(unit)s" ), modelObject=modelXbrl, unit=unitKey) else: _mul, _sep, _div = unitKey.partition('/') if _mul.startswith('('): _mul = _mul[1:-1] _muls = [u for u in _mul.split('*') if u] if _div.startswith('('): _div = _div[1:-1] _divs = [u for u in _div.split('*') if u] if _muls != sorted(_muls) or _divs != sorted(_divs): modelXbrl.error( "oime:unitStringRepresentation", _("Unit string representation measures are not in alphabetical order, %(unit)s" ), modelObject=modelXbrl, unit=unitKey) try: mulQns = [ qname( u, prefixes, prefixException=OIMException( "oime:unitPrefix", _("Unit prefix is not declared: %(unit)s" ), unit=u)) for u in _muls ] divQns = [ qname( u, prefixes, prefixException=OIMException( "oime:unitPrefix", _("Unit prefix is not declared: %(unit)s" ), unit=u)) for u in _divs ] unitId = 'u-{:02}'.format(len(unitTbl) + 1) for _measures in mulQns, divQns: for _measure in _measures: addQnameValue(modelXbrl.modelDocument, _measure) _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId) except OIMException as ex: modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) unitTbl[unitKey] = _unit else: _unit = None attrs = {"contextRef": _cntx.id} if fact.get("value") is None: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact["value"] if fact.get("id"): attrs["id"] = fact["id"] if concept.isNumeric: if _unit is None: continue # skip creating fact because unit was invalid attrs["unitRef"] = _unit.id if "accuracy" in fact: attrs["decimals"] = fact["accuracy"] # is value a QName? if concept.baseXbrliType == "QName": addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes)) f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text) currentAction = "creating footnotes" footnoteLinks = {} # ELR elements factLocs = {} # index by (linkrole, factId) footnoteNbr = 0 locNbr = 0 for factOrFootnote in footnotes: if "factId" in factOrFootnote: factId = factOrFootnote["factId"] factFootnotes = (factOrFootnote, ) # CSV or XL elif "id" in factOrFootnote and "footnotes" in factOrFootnote: factId = factOrFootnote["id"] factFootnotes = factOrFootnote["footnotes"] else: factFootnotes = () for footnote in factFootnotes: linkrole = footnote.get("group") arcrole = footnote.get("footnoteType") if not factId or not linkrole or not arcrole or not ( footnote.get("factRef") or footnote.get("footnote")): # invalid footnote continue if linkrole not in footnoteLinks: footnoteLinks[linkrole] = addChild( modelXbrl.modelDocument.xmlRootElement, XbrlConst.qnLinkFootnoteLink, attributes={ "{http://www.w3.org/1999/xlink}type": "extended", "{http://www.w3.org/1999/xlink}role": linkrole }) footnoteLink = footnoteLinks[linkrole] if (linkrole, factId) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factId)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={ XLINKTYPE: "locator", XLINKHREF: "#" + factId, XLINKLABEL: locLabel }) locLabel = factLocs[(linkrole, factId)] if footnote.get("footnote"): footnoteNbr += 1 footnoteLabel = "f_{:02}".format(footnoteNbr) attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel} if footnote.get("language"): attrs[XMLLANG] = footnote["language"] # note, for HTML will need to build an element structure addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"]) elif footnote.get("factRef"): factRef = footnote.get("factRef") if (linkrole, factRef) not in factLocs: locNbr += 1 locLabel = "f_{:02}".format(footnoteNbr) factLoc[(linkrole, factRef)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={ XLINKTYPE: "locator", XLINKHREF: "#" + factRef, XLINKLABEL: locLabel }) footnoteLabel = factLoc[(linkrole, factId)] footnoteArc = addChild(footnoteLink, XbrlConst.qnLinkFootnoteArc, attributes={ XLINKTYPE: "arc", XLINKARCROLE: arcrole, XLINKFROM: locLabel, XLINKTO: footnoteLabel }) currentAction = "done loading facts and footnotes" #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") except Exception as ex: if isinstance(ex, OIMException): modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) else: modelXbrl.error( "arelleOIMloader:error", "Error while %(action)s, error %(error)s\ntraceback %(traceback)s", modelObject=modelXbrl, action=currentAction, error=ex, traceback=traceback.format_tb(sys.exc_info()[2])) if priorCWD: os.chdir( priorCWD ) # restore prior current working directory startingErrorCount = len(modelXbrl.errors) if startingErrorCount < len(modelXbrl.errors): # had errors, don't allow ModelDocument.load to continue return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors") return getattr(modelXbrl, "modelDocument", None) # none if returning from exception