def htmlLoader(modelXbrl, mappedUri, filepath, *args, **kwargs): if filepath != lastFilePath or not lastFilePathIsHTML: return None # not an HTML file cntlr = modelXbrl.modelManager.cntlr cntlr.showStatus( _("Loading HTML file: {0}").format(os.path.basename(filepath))) # parse html try: if (modelXbrl.modelManager.validateDisclosureSystem and modelXbrl.modelManager.disclosureSystem.validateFileText): file, _encoding = ValidateFilingText.checkfile(modelXbrl, filepath) else: file, _encoding = modelXbrl.fileSource.file(filepath, stripDeclaration=False) _parser = HTMLParser() htmlTree = parse(file, _parser, base_url=filepath) for error in _parser.error_log: if not (error.type_name == "HTML_UNKNOWN_TAG" and error.message.startswith("Tag ") and error.message.lower()[4:].partition(" ")[0] in edgarAdditionalTags): modelXbrl.error( "html:syntax", _("%(error)s, %(fileName)s, line %(line)s, column %(column)s" ), fileName=os.path.basename(mappedUri), error=error.message, line=error.line, column=error.column) file.close() except Exception as err: modelXbrl.error(type(err).__name__, _("Unrecoverable error: %(error)s, %(fileName)s"), fileName=os.path.basename(mappedUri), error=str(err), exc_info=True) return None if modelXbrl: # pull loader implementation modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = doc = createModelDocument( modelXbrl, Type.HTML, filepath, isEntry=True, documentEncoding="utf-8", base=filepath) else: # API implementation modelXbrl = ModelXbrl.create(cntlr.modelManager, Type.HTML, filepath, isEntry=True, base=filepath) doc = modelXbrl.modelDocument doc.xmlRootElement = htmlTree.getroot() if doc is None: return None # not an HTML file modelXbrl.loadedFromHTML = True return doc
def xfLoader(modelXbrl, mappedUri, filepath, *args, **kwargs): if os.path.splitext(filepath)[1] != ".xf": return None # not an XBRL formula syntax file cntlr = modelXbrl.modelManager.cntlr cntlr.showStatus(_("Loading XBRL Formula file: {0}").format(os.path.basename(filepath))) try: import tatsu.exceptions except ImportError: modelXbrl.error("xf:missingTatsu", "Python library module Tatsu must be installed.") from .xf import XFParser with open(filepath, "r") as f: xf = f.read() parser = XFParser() try: ast = parser.parse(xf, rule_name='module') except tatsu.exceptions.FailedParse as err: modelXbrl.error("xf:syntax", "Unrecoverable error: %(error)s", modelObject=modelXbrl, error=err) # create dummy modelDocument for successful plugin execution by ModelDopcument from arelle.ModelDocument import Type, create as createModelDocument doc = createModelDocument(modelXbrl, Type.LINKBASE, filepath, documentEncoding="utf-8", initialXml=''' <!-- Dummy linkbase --> <link:linkbase xmlns:link="http://www.xbrl.org/2003/linkbase" xsi:schemaLocation="http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd" /> ''') return doc
def xfLoader(modelXbrl, mappedUri, filepath, *args, **kwargs): if os.path.splitext(filepath)[1] != ".xf": return None # not an XBRL formula syntax file cntlr = modelXbrl.modelManager.cntlr cntlr.showStatus( _("Loading XBRL Formula file: {0}").format(os.path.basename(filepath))) try: import tatsu.exceptions except ImportError: modelXbrl.error("xf:missingTatsu", "Python library module Tatsu must be installed.") from .xf import XFParser with open(filepath, "r") as f: xf = f.read() parser = XFParser() try: ast = parser.parse(xf, rule_name='module') except tatsu.exceptions.FailedParse as err: modelXbrl.error("xf:syntax", "Unrecoverable error: %(error)s", modelObject=modelXbrl, error=err) # create dummy modelDocument for successful plugin execution by ModelDopcument from arelle.ModelDocument import Type, create as createModelDocument doc = createModelDocument(modelXbrl, Type.LINKBASE, filepath, documentEncoding="utf-8", initialXml=''' <!-- Dummy linkbase --> <link:linkbase xmlns:link="http://www.xbrl.org/2003/linkbase" xsi:schemaLocation="http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd" /> ''') return doc
def loadXbrlFromDB(self, loadDBsaveToFile): # load from database modelXbrl = self.modelXbrl # find instance in DB instanceURI = os.path.basename(loadDBsaveToFile) results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant" " FROM dInstance WHERE FileName = '{}'" .format(instanceURI)) instanceId = moduleId = None for instanceId, moduleId, entScheme, entId, datePerEnd in results: break # find module in DB results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId)) xbrlSchemaRef = None for result in results: xbrlSchemaRef = result[0] break if not instanceId or not xbrlSchemaRef: raise XPDBException("sqlDB:MissingDTS", _("The instance and module were not found for %(instanceURI)"), instanceURI = instanceURI) if modelXbrl.skipDTS: # find prefixes and namespaces in DB results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]") dpmPrefixedNamespaces = dict((prefix, namespace) for owner, prefix, namespace in results) # create the instance document and resulting filing modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument(modelXbrl, Type.INSTANCE, loadDBsaveToFile, schemaRefs=[xbrlSchemaRef], isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement, 'xbrl-streamable-instance', 'version="1.0" contextBuffer="1"') # add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue) # filing indicator code IDs # get filing indicators results = self.execute("SELECT mToT.TemplateOrTableCode " " FROM dFilingIndicator dFI, mTemplateOrTable mToT " " WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID" .format(instanceId)) filingIndicatorCodes = [code[0] for code in results] if filingIndicatorCodes: modelXbrl.createContext(entScheme, entId, 'instant', None, datePerEnd, None, # no dimensional validity checking (like formula does) {}, [], [], id='c') filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators) for filingIndicatorCode in filingIndicatorCodes: modelXbrl.createFact(qnFindFilingIndicator, parent=filingIndicatorsTuple, attributes={"contextRef": "c"}, text=filingIndicatorCode) # facts in this instance factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards," " Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue " "FROM dFact WHERE InstanceID = {} " "ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL " " THEN DataPointSignature" " ELSE DataPointSignatureWithValuesForWildcards" " END, instr(DataPointSignature,'|') + 1)" .format(instanceId)) # results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal # get typed dimension values prefixedNamespaces = modelXbrl.prefixedNamespaces prefixedNamespaces["iso4217"] = XbrlConst.iso4217 if modelXbrl.skipDTS: prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed cntxTbl = {} # index by d unitTbl = {} def typedDimElt(s): # add xmlns into s for known qnames tag, angleBrkt, rest = s[1:].partition('>') text, angleBrkt, rest = rest.partition("<") qn = qname(tag, prefixedNamespaces) # a modelObject xml element is needed for all of the instance functions to manage the typed dim return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False) # contexts and facts for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl: metric, sep, dims = (dpSigTypedDims or dpSig).partition('|') conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces) concept = modelXbrl.qnameConcepts.get(conceptQn) isNumeric = isBool = isDateTime = isQName = isText = False if concept is not None: if concept.isNumeric: isNumeric = True else: baseXbrliType = concept.baseXbrliType if baseXbrliType == "booleanItemType": isBool = True elif baseXbrliType == "dateTimeItemType": # also is dateItemType? isDateTime = True elif baseXbrliType == "QNameItemType": isQName = True else: c = conceptQn.localName[0] if c == 'm': isNumeric = True elif c == 'd': isDateTime = True elif c == 'b': isBool = True elif c == 'e': isQName = True isText = not (isNumeric or isBool or isDateTime or isQName) if isinstance(datePerEnd, _STR_BASE): datePerEnd = datetimeValue(datePerEnd, addOneDay=True) cntxKey = (dims, entId, datePerEnd) if cntxKey in cntxTbl: cntxId = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) cntxTbl[cntxKey] = cntxId qnameDims = {} if dims: for dim in dims.split('|'): dQn, sep, dVal = dim[:-1].partition('(') dimQname = qname(dQn, prefixedNamespaces) if dVal.startswith('<'): mem = typedDimElt(dVal) # typed dim else: mem = qname(dVal, prefixedNamespaces) # explicit dim (even if treat-as-typed) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario") modelXbrl.createContext(entScheme, entId, 'instant', None, datePerEnd, None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) if unit: if unit in unitTbl: unitId = unitTbl[unit] else: unitQn = qname(unit, prefixedNamespaces) unitId = 'u{}'.format(unitQn.localName) unitTbl[unit] = unitId modelXbrl.createUnit([unitQn], [], id=unitId) else: unitId = None attrs = {"contextRef": cntxId} if unitId: attrs["unitRef"] = unitId if dec is not None: if isinstance(dec, float): # must be an integer dec = int(dec) elif isinstance(dec, _STR_BASE) and '.' in dec: dec = dec.partition('.')[0] # drop .0 from any SQLite string attrs["decimals"] = str(dec) # somehow it is float from the database if False: # fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None elif numVal is not None: num = roundValue(numVal, None, dec) # round using reported decimals if dec is None or dec == "INF": # show using decimals or reported format dec = len(numVal.partition(".")[2]) else: # max decimals at 28 dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num)) elif dateVal is not None: text = dateVal elif boolVal is not None: text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false' else: if isQName: # declare namespace addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces)) text = textVal modelXbrl.createFact(conceptQn, attributes=attrs, text=text) # add footnotes if any # save to file modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000) return modelXbrl.modelDocument
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri): from openpyxl import load_workbook from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname _return = None # modelDocument or an exception try: currentAction = "initializing" startingErrorCount = len(modelXbrl.errors) startedAt = time.time() if os.path.isabs(oimFile): # allow relative filenames to loading directory priorCWD = os.getcwd() os.chdir(os.path.dirname(oimFile)) else: priorCWD = None currentAction = "determining file type" isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") # this option is not currently supported isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls") isCSVorXL = isCSV or isXL instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl" _csvwContext = None if isJSON: errPrefix = "xbrlje" currentAction = "loading and parsing JSON OIM file" def loadDict(keyValuePairs): _dict = OrderedDict() # preserve fact order in resulting instance for key, value in keyValuePairs: if isinstance(value, dict): if DUPJSONKEY in value: for _errKey, _errValue, _otherValue in value[DUPJSONKEY]: if key == "prefixes": modelXbrl.error("xbrlje:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"), modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue) del value[DUPJSONKEY] if key in _dict: if DUPJSONKEY not in _dict: _dict[DUPJSONKEY] = [] _dict[DUPJSONKEY].append((key, value, _dict[key])) else: _dict[key] = value return _dict with io.open(oimFile, 'rt', encoding='utf-8') as f: oimObject = json.load(f, object_pairs_hook=loadDict) # check if it's a CSVW metadata _csvwContext = oimObject.get("@context") if _csvwContext == "http://www.w3.org/ns/csvw" or ( isinstance(_csvwContext, list) and "http://www.w3.org/ns/csvw" in _csvwContext): isJSON = False isCSV = True else: if oimObject.get("documentType", None) != "http://www.xbrl.org/WGWD/YYYY-MM-DD/xbrl-json": # is it likely to be an OIM document? if any(t in oimObject for t in ("dtsReferences", "prefixes", "facts")): raise OIMException("xbrlje:missingJSONdocumentType", _("Required documentType is missing from JSON input")) else: raise NotOIMException() # return None, not an OIM document missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject] if missing: raise OIMException("xbrlje:missingJSONElements", _("Required element(s) are missing from JSON input: %(missing)s"), missing = ", ".join(missing)) currentAction = "identifying JSON objects" dtsReferences = oimObject["dtsReferences"] prefixesList = oimObject["prefixes"].items() facts = oimObject["facts"] footnotes = oimObject["facts"] # shares this object if isCSV: errPrefix = "xbrlce" currentAction = "compiling metadata" if sys.version[0] >= '3': csvOpenMode = 'w' csvOpenNewline = '' else: csvOpenMode = 'wb' # for 2.7 csvOpenNewline = None if not _csvwContext: # json metadata wasn't specified, try to find it raise OIMException("xbrlce:missingCSVMetadata", _("Unable to identify CSV metadata file")) # process CSV metadata # mandatory sections of metadata file oimMetadata = oimObject.get(CSVmetadata) if not oimMetadata: raise OIMException("xbrlce:missingOIMMetadata", _("Unable to identify OIM metadata infile")) if oimMetadata.get("documentType") != CSVdocumentType: raise OIMException("xbrlce:documentType", _("Document type %(documentType)s not recognized, expecting %(expectedDocumentType)s"), documentType=oimMetadata.get("documentType"), expectedDocumentType=CSVdocumentType) dtsReferences = oimMetadata.get("dtsReferences", {}) prefixesList = oimMetadata.get("prefixes", {}).items() topLevelAspects = oimObject.get(CSVaspects, {}) currentAction = "loading CSV facts tables" facts = [] footnotes = [] _dir = os.path.dirname(oimFile) for oimTable in oimObject.get("tables", []): tableType = oimTable.get(CSVtableType) tableLevelAspects = oimTable.get(CSVaspects, {}) tableUrl = oimTable.get("url") tableColumns = oimTable.get("tableSchema",{}).get("columns", []) # compile column dependencies aspectCols = [] factCols = [] for iCol, col in enumerate(tableColumns): if CSVcolumnAspect in col: aspectCols.append(iCol) if CSVtupleFactAspects in col or CSVsimpleFactAspects in col: factCols.append(iCol) if tableType not in ("fact", "footnote"): modelXbrl.error("xbrlce:unrecognizedTableType", _("Table type %(tableType)s was not recognized, table URI %(uri)s."), modelObject=modelXbrl, uri=oimFile, tableType=tableType) continue if not tableColumns: modelXbrl.error("xbrlce:noTableColumns", _("Table has no columns, table URI %(uri)s."), modelObject=modelXbrl, uri=_uri) continue tableAspects = topLevelAspects.copy() tableAspects.update(tableLevelAspects) filepath = os.path.join(_dir, tableUrl) tupleIds = set() with io.open(filepath, 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: if tableType == "fact": colAspects = tableAspects.copy() specificColAspects = defaultdict(dict) for iCol in aspectCols: value = row[iCol] aspect = tableColumns[iCol][CSVcolumnAspect] if isinstance(aspect, str): # applies to all cols colAspects[aspect] = value elif isinstance(aspect, dict): # applies to specific cols for _aspect, _colNames in aspect.items(): for _colName in _colNames: specificColAspects[_colName][_aspect] = value for iCol in factCols: fact = {"aspects": {}} cellValue = row[iCol] if cellValue == "": # no fact produced for this cell continue tableCol = tableColumns[iCol] if CSVtupleFactAspects in tableCol: aspectsObjectName = CSVtupleFactAspects else: aspectsObjectName = CSVsimpleFactAspects cellAspects = (colAspects, specificColAspects.get(tableCol.get("name"), EMPTYDICT), tableColumns[iCol].get(aspectsObjectName), EMPTYDICT) inapplicableAspects = set() if tableCol.get(CSVtupleReferenceId) == "true": if cellValue: if cellValue in tupleIds: continue # don't duplicate parent tuple fact["id"] = cellValue tupleIds.add(cellValue) # prevent tuple duplication elif CSVsimpleFactAspects in tableCol: fact["value"] = csvCellValue(cellValue) if CSVtupleFactAspects in tableCol: inapplicableAspects.update(oimSimpleFactAspects) for _aspects in cellAspects: for aspectName, aspectValue in _aspects.items(): if not aspectName.startswith(oimPrefix): inapplicableAspects.add(aspectName) # block any row aspect produced by this column from this column's fact _colAspect = tableCol.get(CSVcolumnAspect) if isinstance(_colAspect, str): # applies to all cols inapplicableAspects.add(_colAspect) for _aspects in cellAspects: if _aspects: for aspectName, aspectValue in _aspects.items(): if aspectName not in inapplicableAspects and aspectValue != "": if ":" in aspectName: fact["aspects"][aspectName] = csvCellValue(aspectValue) else: fact[aspectName] = aspectValue facts.append(fact) elif tableType == "footnote": footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col)) del tupleIds elif isXL: errPrefix = "xbrlwe" currentAction = "identifying workbook input worksheets" oimWb = load_workbook(oimFile, read_only=True, data_only=True) sheetNames = oimWb.get_sheet_names() if (not any(sheetName == "prefixes" for sheetName in sheetNames) or not any(sheetName == "dtsReferences" for sheetName in sheetNames) or not any("facts" in sheetName for sheetName in sheetNames)): raise OIMException("xbrlwe:missingWorkbookWorksheets", _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts")) currentAction = "loading worksheet: dtsReferences" dtsReferences = [] for i, row in enumerate(oimWb["dtsReferences"]): if i == 0: header = [col.value for col in row] else: dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row))) currentAction = "loading worksheet: prefixes" prefixesList = [] for i, row in enumerate(oimWb["prefixes"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) else: prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value)) defaults = {} if "defaults" in sheetNames: currentAction = "loading worksheet: defaults" for i, row in enumerate(oimWb["defaults"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) fileCol = header["file"] else: defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol) facts = [] for sheetName in sheetNames: if sheetName == "facts" or "-facts" in sheetName: currentAction = "loading worksheet: {}".format(sheetName) tableDefaults = defaults.get(sheetName, {}) for i, row in enumerate(oimWb[sheetName]): if i == 0: header = [col.value for col in row] else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col.value is not None: if header[j]: # skip cols with no header if header[j].endswith("Value"): fact["value"] = str(col.value) else: fact[header[j]] = str(col.value) facts.append(fact) footnotes = [] if "footnotes" in sheetNames: currentAction = "loading worksheet: footnotes" for i, row in enumerate(oimWb["footnotes"]): if i == 0: header = [col.value for j,col in enumerate(row) if col.value] else: footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value)) currentAction = "identifying default dimensions" ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults currentAction = "validating OIM" prefixes = {} prefixedUris = {} for _prefix, _uri in prefixesList: if not _prefix: modelXbrl.error("{}:emptyPrefix".format(errPrefix), _("The empty string must not be used as a prefix, uri %(uri)s"), modelObject=modelXbrl, uri=_uri) elif not NCNamePattern.match(_prefix): modelXbrl.error("oime:prefixPattern", _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"), modelObject=modelXbrl, prefix=_prefix, uri=_uri) elif _prefix in prefixes: modelXbrl.error("{}:duplicatedPrefix".format(errPrefix), _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"), modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri) elif _uri in prefixedUris: modelXbrl.error("{}:duplicatedUri".format(errPrefix), _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"), modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix) else: prefixes[_prefix] = _uri prefixedUris[_uri] = _prefix for _nsOim in nsOim: if _nsOim in prefixedUris: _oimPrefix = prefixedUris[_nsOim] if not _oimPrefix: raise OIMException("oime:noOimPrefix", _("The oim namespace must have a declared prefix")) # create the instance document currentAction = "creating instance document" modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = _return = createModelDocument( modelXbrl, Type.INSTANCE, instanceFileName, schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef.get("type") == "schema" and dtsRef.get("href")], isEntry=True, initialComment="extracted from OIM {}".format(mappedUri), documentEncoding="utf-8") modelXbrl.modelDocument.inDTS = True # add linkbase, role and arcrole refs for refType in ("linkbase", "role", "arcrole"): for dtsRef in dtsReferences: if dtsRef.get("type") == refType and dtsRef.get("href"): elt = addChild(modelXbrl.modelDocument.xmlRootElement, qname(link, refType+"Ref"), attributes=(("{http://www.w3.org/1999/xlink}href", dtsRef["href"]), ("{http://www.w3.org/1999/xlink}type", "simple"))) href = modelXbrl.modelDocument.discoverHref(elt) if href: _elt, hrefDoc, hrefId = href _defElt = hrefDoc.idObjects.get(hrefId) _uriAttrName = refType + "URI" _uriAttrValue = _defElt.get(_uriAttrName) if _defElt is not None and _uriAttrValue: elt.set(_uriAttrName, _uriAttrValue) cntxTbl = {} unitTbl = {} # determine tuple dependency order idFacts = {} parentedFacts = defaultdict(list) for i, fact in enumerate(facts): aspects = fact["aspects"] id = fact.get("id") tupleParent = fact.get("aspects", EMPTYDICT).get(oimTupleParent) if id is not None: idFacts[id] = i parentedFacts[tupleParent].append(i) # root facts have tupleParent None currentAction = "creating facts" def createModelFact(fact, parentModelFact, topTupleFact): aspects = fact.get("aspects", EMPTYDICT) if oimConcept not in aspects: modelXbrl.error("{}:conceptQName".format(errPrefix), _("The concept QName could not be determined"), modelObject=modelXbrl) return conceptQn = qname(aspects[oimConcept], prefixes) concept = modelXbrl.qnameConcepts.get(conceptQn) attrs = {} if concept.isItem: missingAspects = [] if oimEntity not in aspects: missingAspects.append(oimEntity) if oimPeriod not in aspects and (oimPeriodStart not in aspects or oimPeriodEnd not in aspects): missingAspects.append(oimPeriod) if missingAspects: modelXbrl.error("{}:missingAspects".format(errPrefix), _("The concept %(element)s is missing aspects %(missingAspects)s"), modelObject=modelXbrl, element=conceptQn, missingAspects=", ".join(missingAspects)) return entityAsQn = qname(aspects[oimEntity], prefixes) or qname("error",fact[oimEntity]) if oimPeriod in aspects: periodStart = periodEnd = aspects[oimPeriod] if oimPeriodStart in aspects and oimPeriodEnd in aspects: periodStart = aspects[oimPeriodStart] periodEnd = aspects[oimPeriodEnd] cntxKey = ( # hashable context key ("periodType", concept.periodType), ("entity", entityAsQn), ("periodStart", periodStart), ("periodEnd", periodEnd)) + tuple(sorted( (dimName, dimVal["value"] if isinstance(dimVal,dict) else dimVal) for dimName, dimVal in aspects.items() if ":" in dimName and not dimName.startswith(oimPrefix))) if cntxKey in cntxTbl: _cntx = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) qnameDims = {} for dimName, dimVal in aspects.items(): if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal: dimQname = qname(dimName, prefixes) dimConcept = modelXbrl.qnameConcepts.get(dimQname) if dimConcept is None: modelXbrl.error("{}:taxonomyDefinedAspectQName".format(errPrefix), _("The taxonomy defined aspect concept QName %(qname)s could not be determined"), modelObject=modelXbrl, qname=dimQname) continue if isinstance(dimVal, dict): dimVal = dimVal["value"] else: dimVal = str(dimVal) # may be int or boolean if isinstance(dimVal,str) and ":" in dimVal and dimVal.partition(':')[0] in prefixes: mem = qname(dimVal, prefixes) # explicit dim elif dimConcept.isTypedDimension: # a modelObject xml element is needed for all of the instance functions to manage the typed dim mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment") _cntx = modelXbrl.createContext( entityAsQn.namespaceURI, entityAsQn.localName, concept.periodType, None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME), dateTime(periodEnd, type=DATETIME), None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId, beforeSibling=topTupleFact) cntxTbl[cntxKey] = _cntx if oimUnit in aspects and concept.isNumeric: unitKey = aspects[oimUnit] if unitKey in unitTbl: _unit = unitTbl[unitKey] else: _unit = None # validate unit unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey) if not UnitPattern.match(unitKeySub): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation is lexically invalid, %(unit)s"), modelObject=modelXbrl, unit=unitKey) else: _mul, _sep, _div = unitKey.partition('/') if _mul.startswith('('): _mul = _mul[1:-1] _muls = [u for u in _mul.split('*') if u] if _div.startswith('('): _div = _div[1:-1] _divs = [u for u in _div.split('*') if u] if _muls != sorted(_muls) or _divs != sorted(_divs): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation measures are not in alphabetical order, %(unit)s"), modelObject=modelXbrl, unit=unitKey) try: mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _muls] divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _divs] unitId = 'u-{:02}'.format(len(unitTbl) + 1) for _measures in mulQns, divQns: for _measure in _measures: addQnameValue(modelXbrl.modelDocument, _measure) _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId, beforeSibling=topTupleFact) except OIMException as ex: modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) unitTbl[unitKey] = _unit else: _unit = None attrs["contextRef"] = _cntx.id if fact.get("value") is None: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact["value"] if concept.isNumeric: if _unit is None: return # skip creating fact because unit was invalid attrs["unitRef"] = _unit.id if "accuracy" in attrs or attrs.get(XbrlConst.qnXsiNil, "false") != "true": attrs["decimals"] = fact.get("accuracy", "INF") else: text = None #tuple id = fact.get("id") if id is not None: attrs["id"] = fact["id"] # is value a QName? if concept.baseXbrliType == "QName": addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes)) f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text, parent=parentModelFact, validate=False) if id is not None and id in parentedFacts: # create child facts for i in sorted(parentedFacts[id], key=lambda j: facts[j].get("aspects", EMPTYDICT).get(oimTupleOrder,0)): createModelFact(facts[i], f, f if topTupleFact is None else topTupleFact) # validate after creating tuple contents xmlValidate(modelXbrl, f) for i in parentedFacts[None]: createModelFact(facts[i], None, None) currentAction = "creating footnotes" footnoteLinks = OrderedDict() # ELR elements factLocs = {} # index by (linkrole, factId) footnoteNbr = 0 locNbr = 0 for factOrFootnote in footnotes: if "factId" in factOrFootnote: factId = factOrFootnote["factId"] factFootnotes = (factOrFootnote,) # CSV or XL elif "id" in factOrFootnote and "footnotes" in factOrFootnote: factId = factOrFootnote["id"] factFootnotes = factOrFootnote["footnotes"] else: factFootnotes = () for footnote in factFootnotes: linkrole = footnote.get("group") arcrole = footnote.get("footnoteType") if not factId or not linkrole or not arcrole or not ( footnote.get("factRef") or footnote.get("footnote")): # invalid footnote continue if linkrole not in footnoteLinks: footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, XbrlConst.qnLinkFootnoteLink, attributes={"{http://www.w3.org/1999/xlink}type": "extended", "{http://www.w3.org/1999/xlink}role": linkrole}) footnoteLink = footnoteLinks[linkrole] if (linkrole, factId) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factId)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factId, XLINKLABEL: locLabel}) locFromLabel = factLocs[(linkrole, factId)] if footnote.get("footnote"): footnoteNbr += 1 footnoteToLabel = "f_{:02}".format(footnoteNbr) attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteToLabel} if footnote.get("language"): attrs[XMLLANG] = footnote["language"] # note, for HTML will need to build an element structure addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"]) elif footnote.get("factRef"): factRef = footnote.get("factRef") if (linkrole, factRef) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factRef)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factRef, XLINKLABEL: locLabel}) footnoteToLabel = factLocs[(linkrole, factRef)] footnoteArc = addChild(footnoteLink, XbrlConst.qnLinkFootnoteArc, attributes={XLINKTYPE: "arc", XLINKARCROLE: arcrole, XLINKFROM: locFromLabel, XLINKTO: footnoteToLabel}) if footnoteLinks: modelXbrl.modelDocument.linkbaseDiscover(footnoteLinks.values(), inInstance=True) currentAction = "done loading facts and footnotes" #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") except NotOIMException as ex: _return = ex # not an OIM document except Exception as ex: _return = ex if isinstance(ex, OIMException): modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) else: modelXbrl.error("arelleOIMloader:error", "Error while %(action)s, error %(error)s\ntraceback %(traceback)s", modelObject=modelXbrl, action=currentAction, error=ex, traceback=traceback.format_tb(sys.exc_info()[2])) if priorCWD: os.chdir(priorCWD) # restore prior current working directory startingErrorCount = len(modelXbrl.errors) #if startingErrorCount < len(modelXbrl.errors): # # had errors, don't allow ModelDocument.load to continue # return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors") return _return
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri): from openpyxl import load_workbook from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname try: currentAction = "initializing" startedAt = time.time() if os.path.isabs(oimFile): # allow relative filenames to loading directory priorCWD = os.getcwd() os.chdir(os.path.dirname(oimFile)) else: priorCWD = None currentAction = "determining file type" isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json") isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json") isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls") isCSVorXL = isCSV or isXL instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl" if isJSON: currentAction = "loading and parsing JSON OIM file" with io.open(oimFile, 'rt', encoding='utf-8') as f: oimObject = json.load(f) missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject] if missing: raise OIMException("oime:missingJSONelements", _("The required elements %(missing)s {} in JSON input"), missing = ", ".join(missing)) currentAction = "identifying JSON objects" dtsReferences = oimObject["dtsReferences"] prefixes = oimObject["prefixes"] facts = oimObject["facts"] footnotes = oimOject["facts"] # shares this object elif isCSV: currentAction = "identifying CSV input tables" if sys.version[0] >= '3': csvOpenMode = 'w' csvOpenNewline = '' else: csvOpenMode = 'wb' # for 2.7 csvOpenNewline = None oimFileBase = None if "-facts" in oimFile: oimFileBase = oimFile.partition("-facts")[0] else: for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"): if oimFile.endswith(suffix): oimFileBase = oimFile[:-length(suffix)] break if oimFileBase is None: raise OIMException("oime:missingCSVtables", _("Unable to identify CSV tables file name pattern")) if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or not os.path.exists(oimFileBase + "-prefixes.csv")): raise OIMException("oime:missingCSVtables", _("Unable to identify CSV tables for dtsReferences or prefixes")) instanceFileName = oimFileBase + ".xbrl" currentAction = "loading CSV dtsReferences table" dtsReferences = [] with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: dtsReferences.append(dict((header[j], col) for j, col in enumerate(row))) currentAction = "loading CSV prefixes table" prefixes = {} with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = dict((col,i) for i,col in enumerate(row)) else: prefixes[row[header["prefix"]]] = row[header["URI"]] currentAction = "loading CSV defaults table" defaults = {} if os.path.exists(oimFileBase + "-defaults.csv"): with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row fileCol = row.index("file") else: defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol) currentAction = "loading CSV facts tables" facts = [] _dir = os.path.dirname(oimFileBase) for filename in os.listdir(_dir): filepath = os.path.join(_dir, filename) if "-facts" in filename and filepath.startswith(oimFileBase): tableDefaults = defaults.get(filename, {}) with io.open(filepath, 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col is not None: if header[j].endswith("Value"): if col: # ignore empty columns (= null CSV value) fact["value"] = col else: fact[header[j]] = col facts.append(fact) footnotes = [] if os.path.exists(oimFileBase + "-footnotes.csv"): with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col)) elif isXL: oimWb = load_workbook(oimFile, read_only=True, data_only=True) sheetNames = oimWb.get_sheet_names() if (not any(sheetName == "prefixes" for sheetName in sheetNames) or not any(sheetName == "dtsReferences" for sheetName in sheetNames) or not any("facts" in sheetName for sheetName in sheetNames)): if priorCWD: os.chdir(priorCWD) return None try: dtsReferences = [] for i, row in oimWb["dtsReferences"]: if i == 0: header = [col.value for col in row] else: dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row))) prefixes = {} for i, row in oimWb["dtsReferences"]: if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) else: prefixes[row[header["prefix"]].value] = row[header["URI"].value] defaults = {} for i, row in oimW.get("defaults", ()): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) fileCol = header["file"] else: defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol) facts = [] _dir = os.path.dirpart(oimFileBase) for sheetName in sheetNames: if sheetName == "facts" or "-facts" in sheetName: for i, row in oimWb[sheetName]: if i == 0: header = [col.value for col in row] else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col.value is not None: if header[j].endswith("Value"): fact["value"] = str(col.value) else: fact[header[j]] = str(col.value) facts.append(fact) footnotes = [] for i, row in oimWb.get("footnotes", ()): if i == 0: header = dict((col.value,i) for i,col in enumerate(row) if col.value) else: footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value)) except Exception as ex: if priorCWD: os.chdir(priorCWD) return None ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults # create the instance document modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument( modelXbrl, Type.INSTANCE, instanceFileName, schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"], isEntry=True, initialComment="extracted from OIM {}".format(mappedUri), documentEncoding="utf-8") cntxTbl = {} unitTbl = {} for fact in facts: conceptQn = qname(fact["oim:concept"], prefixes) concept = modelXbrl.qnameConcepts.get(conceptQn) entityAsQn = qname(fact["oim:entity"], prefixes) if "oim:period" in fact: periodStart = fact["oim:period"]["start"] periodEnd = fact["oim:period"]["end"] else: periodStart = fact["oim:periodStart"] periodEnd = fact["oim:periodEnd"] cntxKey = ( # hashable context key ("periodType", concept.periodType), ("entity", entityAsQn), ("periodStart", periodStart), ("periodEnd", periodEnd)) + tuple(sorted( (dimName, dimVal) for dimName, dimVal in fact.items() if ":" in dimName and not dimName.startswith("oim:"))) if cntxKey in cntxTbl: _cntx = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) qnameDims = {} for dimName, dimVal in fact.items(): if ":" in dimName and not dimName.startswith("oim:"): dimQname = qname(dimName, prefixes) dimConcept = modelXbrl.qnameConcepts.get(dimQname) if ":" in dimVal and dimVal.partition[':'][0] in prefixes: mem = qname(dimVal, prefixes) # explicit dim elif dimConcept.isTypedDimension: # a modelObject xml element is needed for all of the instance functions to manage the typed dim mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment") _cntx = modelXbrl.createContext( entityAsQn.namespaceURI, entityAsQn.localName, concept.periodType, None if concept.periodType == "instnat" else dateTime(periodStart, type=DATETIME), dateTime(periodEnd, type=DATETIME), None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) cntxTbl[cntxKey] = _cntx if "oim:unit" in fact: unitKey = fact["oim:unit"] if unitKey in unitTbl: _unit = unitTbl[unitKey] else: _mul, _sep, _div = unitKey.partition('/') if _mul.startswith('('): _mul = _mul[1:-1] mulQns = [qname(u, prefixes) for u in _mul.split('*') if u] if _div.startswith('('): _div = _div[1:-1] divQns = [qname(u, prefixes) for u in _div.split('*') if u] unitId = 'u-{:02}'.format(len(unitTbl) + 1) for _measures in mulQns, divQns: for _measure in _measures: addQnameValue(modelXbrl.modelDocument, _measure) _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId) unitTbl[unitKey] = _unit else: _unit = None attrs = {"contextRef": _cntx.id} if fact.get("value") is None: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact["value"] if fact.get("id"): attrs["id"] = fact["id"] if concept.isNumeric: if _unit is not None: attrs["unitRef"] = _unit.id if "accuracy" in fact: attrs["decimals"] = fact["accuracy"] # is value a QName? if concept.baseXbrliType == "QName": addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes)) f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text) footnoteLinks = {} # ELR elements factLocs = {} # index by (linkrole, factId) footnoteNbr = 0 locNbr = 0 for factOrFootnote in footnotes: if "factId" in factOrFootnote: factId = factOrFootnote["factId"] factFootnotes = (factOrFootnote,) # CSV or XL elif "id" in factOrFootnote and "footnotes" in factOrFootnote: factId = factOrFootnote["id"] factFootnotes = factOrFootnote["footnotes"] else: factFootnotes = () for footnote in factFootnotes: linkrole = footnote.get("group") arcrole = footnote.get("footnoteType") if not factId or not linkrole or not arcrole or not ( footnote.get("factRef") or footnote.get("footnote")): # invalid footnote continue if linkrole not in footnoteLinks: footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, XbrlConst.qnLinkFootnoteLink, attributes={"{http://www.w3.org/1999/xlink}type": "extended", "{http://www.w3.org/1999/xlink}role": linkrole}) footnoteLink = footnoteLinks[linkrole] if (linkrole, factId) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factId)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factId, XLINKLABEL: locLabel}) locLabel = factLocs[(linkrole, factId)] if footnote.get("footnote"): footnoteNbr += 1 footnoteLabel = "f_{:02}".format(footnoteNbr) attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel} if footnote.get("language"): attrs[XMLLANG] = footnote["language"] # note, for HTML will need to build an element structure addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"]) elif footnote.get("factRef"): factRef = footnote.get("factRef") if (linkrole, factRef) not in factLocs: locNbr += 1 locLabel = "f_{:02}".format(footnoteNbr) factLoc[(linkrole, factRef)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factRef, XLINKLABEL: locLabel}) footnoteLabel = factLoc[(linkrole, factId)] footnoteArc = addChild(footnoteLink, XbrlConst.qnLinkFootnoteArc, attributes={XLINKTYPE: "arc", XLINKARCROLE: arcrole, XLINKFROM: locLabel, XLINKTO: footnoteLabel}) #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") except Exception as ex: if ex is OIMException: modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) else: modelXbrl.error("Error while %(action)s, error %(error)s", modelObject=modelXbrl, action=currentAction, error=ex) if priorCWD: os.chdir(priorCWD) # restore prior current working directory return modelXbrl.modelDocument
def loadXbrlFromDB(self, loadDBsaveToFile): # load from database modelXbrl = self.modelXbrl # find instance in DB instanceURI = os.path.basename(loadDBsaveToFile) results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant" " FROM dInstance WHERE FileName = '{}'" .format(instanceURI)) instanceId = moduleId = None for instanceId, moduleId, entScheme, entId, datePerEnd in results: break # find module in DB results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId)) xbrlSchemaRef = None for result in results: xbrlSchemaRef = result[0] break if not instanceId or not xbrlSchemaRef: raise XPDBException("sqlDB:MissingDTS", _("The instance and module were not found for %(instanceURI)"), instanceURI = instanceURI) if modelXbrl.skipDTS: # find prefixes and namespaces in DB results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]") dpmPrefixedNamespaces = dict((prefix, namespace) for owner, prefix, namespace in results) # create the instance document and resulting filing modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument(modelXbrl, Type.INSTANCE, loadDBsaveToFile, schemaRefs=[xbrlSchemaRef], isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement, 'xbrl-streamable-instance', 'version="1.0" contextBuffer="1"') # add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue) # filing indicator code IDs # get filing indicators results = self.execute("SELECT mToT.TemplateOrTableCode " " FROM dFilingIndicator dFI, mTemplateOrTable mToT " " WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID" .format(instanceId)) filingIndicatorCodes = [code[0] for code in results] if filingIndicatorCodes: modelXbrl.createContext(entScheme, entId, 'instant', None, datePerEnd, None, # no dimensional validity checking (like formula does) {}, [], [], id='c') filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators) for filingIndicatorCode in filingIndicatorCodes: modelXbrl.createFact(qnFindFilingIndicator, parent=filingIndicatorsTuple, attributes={"contextRef": "c"}, text=filingIndicatorCode) # facts in this instance factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards," " Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue " "FROM dFact WHERE InstanceID = {} " "ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL " " THEN DataPointSignature" " ELSE DataPointSignatureWithValuesForWildcards" " END, instr(DataPointSignature,'|') + 1)" .format(instanceId)) # results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal # get typed dimension values prefixedNamespaces = modelXbrl.prefixedNamespaces prefixedNamespaces["iso4217"] = XbrlConst.iso4217 if modelXbrl.skipDTS: prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed cntxTbl = {} # index by d unitTbl = {} def typedDimElt(s): # add xmlns into s for known qnames tag, angleBrkt, rest = s[1:].partition('>') text, angleBrkt, rest = rest.partition("<") qn = qname(tag, prefixedNamespaces) # a modelObject xml element is needed for all of the instance functions to manage the typed dim return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False) # contexts and facts for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl: metric, sep, dims = (dpSigTypedDims or dpSig).partition('|') conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces) concept = modelXbrl.qnameConcepts.get(conceptQn) isNumeric = isBool = isDateTime = isQName = isText = False if concept is not None: if concept.isNumeric: isNumeric = True else: baseXbrliType = concept.baseXbrliType if baseXbrliType == "booleanItemType": isBool = True elif baseXbrliType == "dateTimeItemType": # also is dateItemType? isDateTime = True elif baseXbrliType == "QNameItemType": isQName = True else: c = conceptQn.localName[0] if c == 'm': isNumeric = True elif c == 'd': isDateTime = True elif c == 'b': isBool = True elif c == 'e': isQName = True isText = not (isNumeric or isBool or isDateTime or isQName) if isinstance(datePerEnd, _STR_BASE): datePerEnd = datetimeValue(datePerEnd, addOneDay=True) cntxKey = (dims, entId, datePerEnd) if cntxKey in cntxTbl: cntxId = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) cntxTbl[cntxKey] = cntxId qnameDims = {} if dims: for dim in dims.split('|'): dQn, sep, dVal = dim[:-1].partition('(') dimQname = qname(dQn, prefixedNamespaces) if dVal.startswith('<'): # typed dim mem = typedDimElt(dVal) else: mem = qname(dVal, prefixedNamespaces) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario") modelXbrl.createContext(entScheme, entId, 'instant', None, datePerEnd, None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) if unit: if unit in unitTbl: unitId = unitTbl[unit] else: unitQn = qname(unit, prefixedNamespaces) unitId = 'u{}'.format(unitQn.localName) unitTbl[unit] = unitId modelXbrl.createUnit([unitQn], [], id=unitId) else: unitId = None attrs = {"contextRef": cntxId} if unitId: attrs["unitRef"] = unitId if dec is not None: if isinstance(dec, float): # must be an integer dec = int(dec) elif isinstance(dec, _STR_BASE) and '.' in dec: dec = dec.partition('.')[0] # drop .0 from any SQLite string attrs["decimals"] = str(dec) # somehow it is float from the database if False: # fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None elif numVal is not None: num = roundValue(numVal, None, dec) # round using reported decimals if dec is None or dec == "INF": # show using decimals or reported format dec = len(numVal.partition(".")[2]) else: # max decimals at 28 dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num)) elif dateVal is not None: text = dateVal elif boolVal is not None: text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false' else: if isQName: # declare namespace addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces)) text = textVal modelXbrl.createFact(conceptQn, attributes=attrs, text=text) # add footnotes if any # save to file modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000) return modelXbrl.modelDocument
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri): from openpyxl import load_workbook from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname try: currentAction = "initializing" startingErrorCount = len(modelXbrl.errors) startedAt = time.time() if os.path.isabs(oimFile): # allow relative filenames to loading directory priorCWD = os.getcwd() os.chdir(os.path.dirname(oimFile)) else: priorCWD = None currentAction = "determining file type" isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json") isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json") isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls") isCSVorXL = isCSV or isXL instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl" if isJSON: currentAction = "loading and parsing JSON OIM file" def loadDict(keyValuePairs): _dict = OrderedDict() # preserve fact order in resulting instance for key, value in keyValuePairs: if isinstance(value, dict): if DUPJSONKEY in value: for _errKey, _errValue, _otherValue in value[DUPJSONKEY]: if key == "prefixes": modelXbrl.error("oime:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"), modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue) del value[DUPJSONKEY] if key in _dict: if DUPJSONKEY not in _dict: _dict[DUPJSONKEY] = [] _dict[DUPJSONKEY].append((key, value, _dict[key])) else: _dict[key] = value return _dict with io.open(oimFile, 'rt', encoding='utf-8') as f: oimObject = json.load(f, object_pairs_hook=loadDict) missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject] if missing: raise OIMException("oime:missingJSONElements", _("Required element(s) are missing from JSON input: %(missing)s"), missing = ", ".join(missing)) currentAction = "identifying JSON objects" dtsReferences = oimObject["dtsReferences"] prefixesList = oimObject["prefixes"].items() facts = oimObject["facts"] footnotes = oimObject["facts"] # shares this object elif isCSV: currentAction = "identifying CSV input tables" if sys.version[0] >= '3': csvOpenMode = 'w' csvOpenNewline = '' else: csvOpenMode = 'wb' # for 2.7 csvOpenNewline = None oimFileBase = None if "-facts" in oimFile: oimFileBase = oimFile.partition("-facts")[0] else: for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"): if oimFile.endswith(suffix): oimFileBase = oimFile[:-len(suffix)] break if oimFileBase is None: raise OIMException("oime:missingCSVTables", _("Unable to identify CSV tables file name pattern")) if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or not os.path.exists(oimFileBase + "-prefixes.csv")): raise OIMException("oime:missingCSVTables", _("Unable to identify CSV tables for dtsReferences or prefixes")) instanceFileName = oimFileBase + ".xbrl" currentAction = "loading CSV dtsReferences table" dtsReferences = [] with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: dtsReferences.append(dict((header[j], col) for j, col in enumerate(row))) currentAction = "loading CSV prefixes table" prefixesList = [] with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = dict((col,i) for i,col in enumerate(row)) else: prefixesList.append((row[header["prefix"]], row[header["URI"]])) defaults = {} if os.path.exists(oimFileBase + "-defaults.csv"): currentAction = "loading CSV defaults table" with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row fileCol = row.index("file") else: defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol) currentAction = "loading CSV facts tables" facts = [] _dir = os.path.dirname(oimFileBase) factsFileBasename = os.path.basename(oimFileBase) + "-facts" for filename in os.listdir(_dir): filepath = os.path.join(_dir, filename) if filename.startswith(factsFileBasename): currentAction = "loading CSV facts table {}".format(filename) tableDefaults = defaults.get(filename, {}) with io.open(filepath, 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col is not None: if header[j]: # skip cols with no header if header[j].endswith("Value"): if col: # ignore empty columns (= null CSV value) fact["value"] = col else: fact[header[j]] = col facts.append(fact) footnotes = [] if os.path.exists(oimFileBase + "-footnotes.csv"): currentAction = "loading CSV footnotes table" with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col)) elif isXL: currentAction = "identifying workbook input worksheets" oimWb = load_workbook(oimFile, read_only=True, data_only=True) sheetNames = oimWb.get_sheet_names() if (not any(sheetName == "prefixes" for sheetName in sheetNames) or not any(sheetName == "dtsReferences" for sheetName in sheetNames) or not any("facts" in sheetName for sheetName in sheetNames)): raise OIMException("oime:missingWorkbookWorksheets", _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts")) currentAction = "loading worksheet: dtsReferences" dtsReferences = [] for i, row in enumerate(oimWb["dtsReferences"]): if i == 0: header = [col.value for col in row] else: dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row))) currentAction = "loading worksheet: prefixes" prefixesList = [] for i, row in enumerate(oimWb["prefixes"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) else: prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value)) defaults = {} if "defaults" in sheetNames: currentAction = "loading worksheet: defaults" for i, row in enumerate(oimWb["defaults"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row)) fileCol = header["file"] else: defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol) facts = [] for sheetName in sheetNames: if sheetName == "facts" or "-facts" in sheetName: currentAction = "loading worksheet: {}".format(sheetName) tableDefaults = defaults.get(sheetName, {}) for i, row in enumerate(oimWb[sheetName]): if i == 0: header = [col.value for col in row] else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col.value is not None: if header[j]: # skip cols with no header if header[j].endswith("Value"): fact["value"] = str(col.value) else: fact[header[j]] = str(col.value) facts.append(fact) footnotes = [] if "footnotes" in sheetNames: currentAction = "loading worksheet: footnotes" for i, row in enumerate(oimWb["footnotes"]): if i == 0: header = dict((col.value,i) for i,col in enumerate(row) if col.value) else: footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value)) currentAction = "identifying default dimensions" ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults currentAction = "validating OIM" prefixes = {} prefixedUris = {} for _prefix, _uri in prefixesList: if not _prefix: modelXbrl.error("oime:emptyPrefix", _("The empty string must not be used as a prefix, uri %(uri)s"), modelObject=modelXbrl, uri=_uri) elif not NCNamePattern.match(_prefix): modelXbrl.error("oime:prefixPattern", _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"), modelObject=modelXbrl, prefix=_prefix, uri=_uri) elif _prefix in prefixes: modelXbrl.error("oime:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"), modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri) elif _uri in prefixedUris: modelXbrl.error("oime:duplicatedUri", _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"), modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix) else: prefixes[_prefix] = _uri prefixedUris[_uri] = _prefix oimPrefix = None for _nsOim in nsOim: if _nsOim in prefixedUris: oimPrefix = prefixedUris[_nsOim] if not oimPrefix: raise OIMException("oime:noOimPrefix", _("The oim namespace must have a declared prefix")) oimConcept = "{}:concept".format(oimPrefix) oimEntity = "{}:entity".format(oimPrefix) oimPeriod = "{}:period".format(oimPrefix) oimPeriodStart = "{}:periodStart".format(oimPrefix) oimPeriodEnd = "{}:periodEnd".format(oimPrefix) oimUnit = "{}:unit".format(oimPrefix) oimPrefix = "{}:".format(oimPrefix) # create the instance document currentAction = "creating instance document" modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument( modelXbrl, Type.INSTANCE, instanceFileName, schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"], isEntry=True, initialComment="extracted from OIM {}".format(mappedUri), documentEncoding="utf-8") cntxTbl = {} unitTbl = {} currentAction = "creating facts" for fact in facts: conceptQn = qname(fact[oimConcept], prefixes) concept = modelXbrl.qnameConcepts.get(conceptQn) entityAsQn = qname(fact[oimEntity], prefixes) if oimPeriod in fact: periodStart = fact[oimPeriod]["start"] periodEnd = fact[oimPeriod]["end"] else: periodStart = fact[oimPeriodStart] periodEnd = fact[oimPeriodEnd] cntxKey = ( # hashable context key ("periodType", concept.periodType), ("entity", entityAsQn), ("periodStart", periodStart), ("periodEnd", periodEnd)) + tuple(sorted( (dimName, dimVal) for dimName, dimVal in fact.items() if ":" in dimName and not dimName.startswith(oimPrefix))) if cntxKey in cntxTbl: _cntx = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) qnameDims = {} for dimName, dimVal in fact.items(): if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal: dimQname = qname(dimName, prefixes) dimConcept = modelXbrl.qnameConcepts.get(dimQname) if ":" in dimVal and dimVal.partition(':')[0] in prefixes: mem = qname(dimVal, prefixes) # explicit dim elif dimConcept.isTypedDimension: # a modelObject xml element is needed for all of the instance functions to manage the typed dim mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment") _cntx = modelXbrl.createContext( entityAsQn.namespaceURI, entityAsQn.localName, concept.periodType, None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME), dateTime(periodEnd, type=DATETIME), None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) cntxTbl[cntxKey] = _cntx if oimUnit in fact: unitKey = fact[oimUnit] if unitKey in unitTbl: _unit = unitTbl[unitKey] else: _unit = None # validate unit unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey) if not UnitPattern.match(unitKeySub): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation is lexically invalid, %(unit)s"), modelObject=modelXbrl, unit=unitKey) else: _mul, _sep, _div = unitKey.partition('/') if _mul.startswith('('): _mul = _mul[1:-1] _muls = [u for u in _mul.split('*') if u] if _div.startswith('('): _div = _div[1:-1] _divs = [u for u in _div.split('*') if u] if _muls != sorted(_muls) or _divs != sorted(_divs): modelXbrl.error("oime:unitStringRepresentation", _("Unit string representation measures are not in alphabetical order, %(unit)s"), modelObject=modelXbrl, unit=unitKey) try: mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _muls] divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix", _("Unit prefix is not declared: %(unit)s"), unit=u)) for u in _divs] unitId = 'u-{:02}'.format(len(unitTbl) + 1) for _measures in mulQns, divQns: for _measure in _measures: addQnameValue(modelXbrl.modelDocument, _measure) _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId) except OIMException as ex: modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) unitTbl[unitKey] = _unit else: _unit = None attrs = {"contextRef": _cntx.id} if fact.get("value") is None: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact["value"] if fact.get("id"): attrs["id"] = fact["id"] if concept.isNumeric: if _unit is None: continue # skip creating fact because unit was invalid attrs["unitRef"] = _unit.id if "accuracy" in fact: attrs["decimals"] = fact["accuracy"] # is value a QName? if concept.baseXbrliType == "QName": addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes)) f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text) currentAction = "creating footnotes" footnoteLinks = {} # ELR elements factLocs = {} # index by (linkrole, factId) footnoteNbr = 0 locNbr = 0 for factOrFootnote in footnotes: if "factId" in factOrFootnote: factId = factOrFootnote["factId"] factFootnotes = (factOrFootnote,) # CSV or XL elif "id" in factOrFootnote and "footnotes" in factOrFootnote: factId = factOrFootnote["id"] factFootnotes = factOrFootnote["footnotes"] else: factFootnotes = () for footnote in factFootnotes: linkrole = footnote.get("group") arcrole = footnote.get("footnoteType") if not factId or not linkrole or not arcrole or not ( footnote.get("factRef") or footnote.get("footnote")): # invalid footnote continue if linkrole not in footnoteLinks: footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement, XbrlConst.qnLinkFootnoteLink, attributes={"{http://www.w3.org/1999/xlink}type": "extended", "{http://www.w3.org/1999/xlink}role": linkrole}) footnoteLink = footnoteLinks[linkrole] if (linkrole, factId) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factId)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factId, XLINKLABEL: locLabel}) locLabel = factLocs[(linkrole, factId)] if footnote.get("footnote"): footnoteNbr += 1 footnoteLabel = "f_{:02}".format(footnoteNbr) attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel} if footnote.get("language"): attrs[XMLLANG] = footnote["language"] # note, for HTML will need to build an element structure addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"]) elif footnote.get("factRef"): factRef = footnote.get("factRef") if (linkrole, factRef) not in factLocs: locNbr += 1 locLabel = "f_{:02}".format(footnoteNbr) factLoc[(linkrole, factRef)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={XLINKTYPE: "locator", XLINKHREF: "#" + factRef, XLINKLABEL: locLabel}) footnoteLabel = factLoc[(linkrole, factId)] footnoteArc = addChild(footnoteLink, XbrlConst.qnLinkFootnoteArc, attributes={XLINKTYPE: "arc", XLINKARCROLE: arcrole, XLINKFROM: locLabel, XLINKTO: footnoteLabel}) currentAction = "done loading facts and footnotes" #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") except Exception as ex: if isinstance(ex, OIMException): modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) else: modelXbrl.error("arelleOIMloader:error", "Error while %(action)s, error %(error)s\ntraceback %(traceback)s", modelObject=modelXbrl, action=currentAction, error=ex, traceback=traceback.format_tb(sys.exc_info()[2])) if priorCWD: os.chdir(priorCWD) # restore prior current working directory startingErrorCount = len(modelXbrl.errors) if startingErrorCount < len(modelXbrl.errors): # had errors, don't allow ModelDocument.load to continue return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors") return getattr(modelXbrl, "modelDocument", None) # none if returning from exception
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri): from openpyxl import load_workbook from arelle import ModelDocument, ModelXbrl, XmlUtil from arelle.ModelDocument import ModelDocumentReference from arelle.ModelValue import qname try: currentAction = "initializing" startingErrorCount = len(modelXbrl.errors) startedAt = time.time() if os.path.isabs(oimFile): # allow relative filenames to loading directory priorCWD = os.getcwd() os.chdir(os.path.dirname(oimFile)) else: priorCWD = None currentAction = "determining file type" isJSON = oimFile.endswith( ".json") and not oimFile.endswith("-metadata.json") isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json") isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls") isCSVorXL = isCSV or isXL instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl" if isJSON: currentAction = "loading and parsing JSON OIM file" def loadDict(keyValuePairs): _dict = OrderedDict( ) # preserve fact order in resulting instance for key, value in keyValuePairs: if isinstance(value, dict): if DUPJSONKEY in value: for _errKey, _errValue, _otherValue in value[ DUPJSONKEY]: if key == "prefixes": modelXbrl.error( "oime:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s" ), modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue) del value[DUPJSONKEY] if key in _dict: if DUPJSONKEY not in _dict: _dict[DUPJSONKEY] = [] _dict[DUPJSONKEY].append((key, value, _dict[key])) else: _dict[key] = value return _dict with io.open(oimFile, 'rt', encoding='utf-8') as f: oimObject = json.load(f, object_pairs_hook=loadDict) missing = [ t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject ] if missing: raise OIMException( "oime:missingJSONElements", _("Required element(s) are missing from JSON input: %(missing)s" ), missing=", ".join(missing)) currentAction = "identifying JSON objects" dtsReferences = oimObject["dtsReferences"] prefixesList = oimObject["prefixes"].items() facts = oimObject["facts"] footnotes = oimObject["facts"] # shares this object elif isCSV: currentAction = "identifying CSV input tables" if sys.version[0] >= '3': csvOpenMode = 'w' csvOpenNewline = '' else: csvOpenMode = 'wb' # for 2.7 csvOpenNewline = None oimFileBase = None if "-facts" in oimFile: oimFileBase = oimFile.partition("-facts")[0] else: for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"): if oimFile.endswith(suffix): oimFileBase = oimFile[:-len(suffix)] break if oimFileBase is None: raise OIMException( "oime:missingCSVTables", _("Unable to identify CSV tables file name pattern")) if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or not os.path.exists(oimFileBase + "-prefixes.csv")): raise OIMException( "oime:missingCSVTables", _("Unable to identify CSV tables for dtsReferences or prefixes" )) instanceFileName = oimFileBase + ".xbrl" currentAction = "loading CSV dtsReferences table" dtsReferences = [] with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: dtsReferences.append( dict( (header[j], col) for j, col in enumerate(row))) currentAction = "loading CSV prefixes table" prefixesList = [] with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = dict((col, i) for i, col in enumerate(row)) else: prefixesList.append( (row[header["prefix"]], row[header["URI"]])) defaults = {} if os.path.exists(oimFileBase + "-defaults.csv"): currentAction = "loading CSV defaults table" with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row fileCol = row.index("file") else: defaults[row[fileCol]] = dict( (header[j], col) for j, col in enumerate(row) if j != fileCol) currentAction = "loading CSV facts tables" facts = [] _dir = os.path.dirname(oimFileBase) factsFileBasename = os.path.basename(oimFileBase) + "-facts" for filename in os.listdir(_dir): filepath = os.path.join(_dir, filename) if filename.startswith(factsFileBasename): currentAction = "loading CSV facts table {}".format( filename) tableDefaults = defaults.get(filename, {}) with io.open(filepath, 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col is not None: if header[ j]: # skip cols with no header if header[j].endswith("Value"): if col: # ignore empty columns (= null CSV value) fact["value"] = col else: fact[header[j]] = col facts.append(fact) footnotes = [] if os.path.exists(oimFileBase + "-footnotes.csv"): currentAction = "loading CSV footnotes table" with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f: csvReader = csv.reader(f) for i, row in enumerate(csvReader): if i == 0: header = row else: footnotes.append( dict((header[j], col) for j, col in enumerate(row) if col)) elif isXL: currentAction = "identifying workbook input worksheets" oimWb = load_workbook(oimFile, read_only=True, data_only=True) sheetNames = oimWb.get_sheet_names() if (not any(sheetName == "prefixes" for sheetName in sheetNames) or not any(sheetName == "dtsReferences" for sheetName in sheetNames) or not any("facts" in sheetName for sheetName in sheetNames)): raise OIMException( "oime:missingWorkbookWorksheets", _("Unable to identify worksheet tabs for dtsReferences, prefixes or facts" )) currentAction = "loading worksheet: dtsReferences" dtsReferences = [] for i, row in enumerate(oimWb["dtsReferences"]): if i == 0: header = [col.value for col in row] else: dtsReferences.append( dict((header[j], col.value) for j, col in enumerate(row))) currentAction = "loading worksheet: prefixes" prefixesList = [] for i, row in enumerate(oimWb["prefixes"]): if i == 0: header = dict((col.value, i) for i, col in enumerate(row)) else: prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value)) defaults = {} if "defaults" in sheetNames: currentAction = "loading worksheet: defaults" for i, row in enumerate(oimWb["defaults"]): if i == 0: header = dict( (col.value, i) for i, col in enumerate(row)) fileCol = header["file"] else: defaults[row[fileCol].value] = dict( (header[j], col.value) for j, col in enumerate(row) if j != fileCol) facts = [] for sheetName in sheetNames: if sheetName == "facts" or "-facts" in sheetName: currentAction = "loading worksheet: {}".format(sheetName) tableDefaults = defaults.get(sheetName, {}) for i, row in enumerate(oimWb[sheetName]): if i == 0: header = [col.value for col in row] else: fact = {} fact.update(tableDefaults) for j, col in enumerate(row): if col.value is not None: if header[j]: # skip cols with no header if header[j].endswith("Value"): fact["value"] = str(col.value) else: fact[header[j]] = str(col.value) facts.append(fact) footnotes = [] if "footnotes" in sheetNames: currentAction = "loading worksheet: footnotes" for i, row in enumerate(oimWb["footnotes"]): if i == 0: header = dict((col.value, i) for i, col in enumerate(row) if col.value) else: footnotes.append( dict((header[j], col.value) for j, col in enumerate(row) if col.value)) currentAction = "identifying default dimensions" ValidateXbrlDimensions.loadDimensionDefaults( modelXbrl) # needs dimension defaults currentAction = "validating OIM" prefixes = {} prefixedUris = {} for _prefix, _uri in prefixesList: if not _prefix: modelXbrl.error( "oime:emptyPrefix", _("The empty string must not be used as a prefix, uri %(uri)s" ), modelObject=modelXbrl, uri=_uri) elif not NCNamePattern.match(_prefix): modelXbrl.error( "oime:prefixPattern", _("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s" ), modelObject=modelXbrl, prefix=_prefix, uri=_uri) elif _prefix in prefixes: modelXbrl.error( "oime:duplicatedPrefix", _("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s" ), modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri) elif _uri in prefixedUris: modelXbrl.error( "oime:duplicatedUri", _("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s" ), modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix) else: prefixes[_prefix] = _uri prefixedUris[_uri] = _prefix oimPrefix = None for _nsOim in nsOim: if _nsOim in prefixedUris: oimPrefix = prefixedUris[_nsOim] if not oimPrefix: raise OIMException( "oime:noOimPrefix", _("The oim namespace must have a declared prefix")) oimConcept = "{}:concept".format(oimPrefix) oimEntity = "{}:entity".format(oimPrefix) oimPeriod = "{}:period".format(oimPrefix) oimPeriodStart = "{}:periodStart".format(oimPrefix) oimPeriodEnd = "{}:periodEnd".format(oimPrefix) oimUnit = "{}:unit".format(oimPrefix) oimPrefix = "{}:".format(oimPrefix) # create the instance document currentAction = "creating instance document" modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument( modelXbrl, Type.INSTANCE, instanceFileName, schemaRefs=[ dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema" ], isEntry=True, initialComment="extracted from OIM {}".format(mappedUri), documentEncoding="utf-8") cntxTbl = {} unitTbl = {} currentAction = "creating facts" for fact in facts: conceptQn = qname(fact[oimConcept], prefixes) concept = modelXbrl.qnameConcepts.get(conceptQn) entityAsQn = qname(fact[oimEntity], prefixes) if oimPeriod in fact: periodStart = fact[oimPeriod]["start"] periodEnd = fact[oimPeriod]["end"] else: periodStart = fact[oimPeriodStart] periodEnd = fact[oimPeriodEnd] cntxKey = ( # hashable context key ("periodType", concept.periodType), ("entity", entityAsQn), ("periodStart", periodStart), ("periodEnd", periodEnd)) + tuple( sorted( (dimName, dimVal) for dimName, dimVal in fact.items() if ":" in dimName and not dimName.startswith(oimPrefix))) if cntxKey in cntxTbl: _cntx = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) qnameDims = {} for dimName, dimVal in fact.items(): if ":" in dimName and not dimName.startswith( oimPrefix) and dimVal: dimQname = qname(dimName, prefixes) dimConcept = modelXbrl.qnameConcepts.get(dimQname) if ":" in dimVal and dimVal.partition( ':')[0] in prefixes: mem = qname(dimVal, prefixes) # explicit dim elif dimConcept.isTypedDimension: # a modelObject xml element is needed for all of the instance functions to manage the typed dim mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False) qnameDims[dimQname] = DimValuePrototype( modelXbrl, None, dimQname, mem, "segment") _cntx = modelXbrl.createContext( entityAsQn.namespaceURI, entityAsQn.localName, concept.periodType, None if concept.periodType == "instant" else dateTime( periodStart, type=DATETIME), dateTime(periodEnd, type=DATETIME), None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) cntxTbl[cntxKey] = _cntx if oimUnit in fact: unitKey = fact[oimUnit] if unitKey in unitTbl: _unit = unitTbl[unitKey] else: _unit = None # validate unit unitKeySub = PrefixedQName.sub( UnitPrefixedQNameSubstitutionChar, unitKey) if not UnitPattern.match(unitKeySub): modelXbrl.error( "oime:unitStringRepresentation", _("Unit string representation is lexically invalid, %(unit)s" ), modelObject=modelXbrl, unit=unitKey) else: _mul, _sep, _div = unitKey.partition('/') if _mul.startswith('('): _mul = _mul[1:-1] _muls = [u for u in _mul.split('*') if u] if _div.startswith('('): _div = _div[1:-1] _divs = [u for u in _div.split('*') if u] if _muls != sorted(_muls) or _divs != sorted(_divs): modelXbrl.error( "oime:unitStringRepresentation", _("Unit string representation measures are not in alphabetical order, %(unit)s" ), modelObject=modelXbrl, unit=unitKey) try: mulQns = [ qname( u, prefixes, prefixException=OIMException( "oime:unitPrefix", _("Unit prefix is not declared: %(unit)s" ), unit=u)) for u in _muls ] divQns = [ qname( u, prefixes, prefixException=OIMException( "oime:unitPrefix", _("Unit prefix is not declared: %(unit)s" ), unit=u)) for u in _divs ] unitId = 'u-{:02}'.format(len(unitTbl) + 1) for _measures in mulQns, divQns: for _measure in _measures: addQnameValue(modelXbrl.modelDocument, _measure) _unit = modelXbrl.createUnit(mulQns, divQns, id=unitId) except OIMException as ex: modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) unitTbl[unitKey] = _unit else: _unit = None attrs = {"contextRef": _cntx.id} if fact.get("value") is None: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact["value"] if fact.get("id"): attrs["id"] = fact["id"] if concept.isNumeric: if _unit is None: continue # skip creating fact because unit was invalid attrs["unitRef"] = _unit.id if "accuracy" in fact: attrs["decimals"] = fact["accuracy"] # is value a QName? if concept.baseXbrliType == "QName": addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes)) f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text) currentAction = "creating footnotes" footnoteLinks = {} # ELR elements factLocs = {} # index by (linkrole, factId) footnoteNbr = 0 locNbr = 0 for factOrFootnote in footnotes: if "factId" in factOrFootnote: factId = factOrFootnote["factId"] factFootnotes = (factOrFootnote, ) # CSV or XL elif "id" in factOrFootnote and "footnotes" in factOrFootnote: factId = factOrFootnote["id"] factFootnotes = factOrFootnote["footnotes"] else: factFootnotes = () for footnote in factFootnotes: linkrole = footnote.get("group") arcrole = footnote.get("footnoteType") if not factId or not linkrole or not arcrole or not ( footnote.get("factRef") or footnote.get("footnote")): # invalid footnote continue if linkrole not in footnoteLinks: footnoteLinks[linkrole] = addChild( modelXbrl.modelDocument.xmlRootElement, XbrlConst.qnLinkFootnoteLink, attributes={ "{http://www.w3.org/1999/xlink}type": "extended", "{http://www.w3.org/1999/xlink}role": linkrole }) footnoteLink = footnoteLinks[linkrole] if (linkrole, factId) not in factLocs: locNbr += 1 locLabel = "l_{:02}".format(locNbr) factLocs[(linkrole, factId)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={ XLINKTYPE: "locator", XLINKHREF: "#" + factId, XLINKLABEL: locLabel }) locLabel = factLocs[(linkrole, factId)] if footnote.get("footnote"): footnoteNbr += 1 footnoteLabel = "f_{:02}".format(footnoteNbr) attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel} if footnote.get("language"): attrs[XMLLANG] = footnote["language"] # note, for HTML will need to build an element structure addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"]) elif footnote.get("factRef"): factRef = footnote.get("factRef") if (linkrole, factRef) not in factLocs: locNbr += 1 locLabel = "f_{:02}".format(footnoteNbr) factLoc[(linkrole, factRef)] = locLabel addChild(footnoteLink, XbrlConst.qnLinkLoc, attributes={ XLINKTYPE: "locator", XLINKHREF: "#" + factRef, XLINKLABEL: locLabel }) footnoteLabel = factLoc[(linkrole, factId)] footnoteArc = addChild(footnoteLink, XbrlConst.qnLinkFootnoteArc, attributes={ XLINKTYPE: "arc", XLINKARCROLE: arcrole, XLINKFROM: locLabel, XLINKTO: footnoteLabel }) currentAction = "done loading facts and footnotes" #cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt), # messageCode="loadFromExcel:info") except Exception as ex: if isinstance(ex, OIMException): modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs) else: modelXbrl.error( "arelleOIMloader:error", "Error while %(action)s, error %(error)s\ntraceback %(traceback)s", modelObject=modelXbrl, action=currentAction, error=ex, traceback=traceback.format_tb(sys.exc_info()[2])) if priorCWD: os.chdir( priorCWD ) # restore prior current working directory startingErrorCount = len(modelXbrl.errors) if startingErrorCount < len(modelXbrl.errors): # had errors, don't allow ModelDocument.load to continue return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors") return getattr(modelXbrl, "modelDocument", None) # none if returning from exception