def htmlLoader(modelXbrl, mappedUri, filepath, *args, **kwargs):
if filepath != lastFilePath or not lastFilePathIsHTML:
return None # not an HTML file
cntlr = modelXbrl.modelManager.cntlr
cntlr.showStatus(
_("Loading HTML file: {0}").format(os.path.basename(filepath)))
# parse html
try:
if (modelXbrl.modelManager.validateDisclosureSystem
and modelXbrl.modelManager.disclosureSystem.validateFileText):
file, _encoding = ValidateFilingText.checkfile(modelXbrl, filepath)
else:
file, _encoding = modelXbrl.fileSource.file(filepath,
stripDeclaration=False)
_parser = HTMLParser()
htmlTree = parse(file, _parser, base_url=filepath)
for error in _parser.error_log:
if not (error.type_name == "HTML_UNKNOWN_TAG"
and error.message.startswith("Tag ")
and error.message.lower()[4:].partition(" ")[0]
in edgarAdditionalTags):
modelXbrl.error(
"html:syntax",
_("%(error)s, %(fileName)s, line %(line)s, column %(column)s"
),
fileName=os.path.basename(mappedUri),
error=error.message,
line=error.line,
column=error.column)
file.close()
except Exception as err:
modelXbrl.error(type(err).__name__,
_("Unrecoverable error: %(error)s, %(fileName)s"),
fileName=os.path.basename(mappedUri),
error=str(err),
exc_info=True)
return None
if modelXbrl: # pull loader implementation
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = doc = createModelDocument(
modelXbrl,
Type.HTML,
filepath,
isEntry=True,
documentEncoding="utf-8",
base=filepath)
else: # API implementation
modelXbrl = ModelXbrl.create(cntlr.modelManager,
Type.HTML,
filepath,
isEntry=True,
base=filepath)
doc = modelXbrl.modelDocument
doc.xmlRootElement = htmlTree.getroot()
if doc is None:
return None # not an HTML file
modelXbrl.loadedFromHTML = True
return doc
def xfLoader(modelXbrl, mappedUri, filepath, *args, **kwargs):
if os.path.splitext(filepath)[1] != ".xf":
return None # not an XBRL formula syntax file
cntlr = modelXbrl.modelManager.cntlr
cntlr.showStatus(_("Loading XBRL Formula file: {0}").format(os.path.basename(filepath)))
try:
import tatsu.exceptions
except ImportError:
modelXbrl.error("xf:missingTatsu",
"Python library module Tatsu must be installed.")
from .xf import XFParser
with open(filepath, "r") as f:
xf = f.read()
parser = XFParser()
try:
ast = parser.parse(xf, rule_name='module')
except tatsu.exceptions.FailedParse as err:
modelXbrl.error("xf:syntax",
"Unrecoverable error: %(error)s",
modelObject=modelXbrl, error=err)
# create dummy modelDocument for successful plugin execution by ModelDopcument
from arelle.ModelDocument import Type, create as createModelDocument
doc = createModelDocument(modelXbrl, Type.LINKBASE, filepath, documentEncoding="utf-8", initialXml='''
<!-- Dummy linkbase -->
<link:linkbase
xmlns:link="http://www.xbrl.org/2003/linkbase"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
/>
''')
return doc
def xfLoader(modelXbrl, mappedUri, filepath, *args, **kwargs):
if os.path.splitext(filepath)[1] != ".xf":
return None # not an XBRL formula syntax file
cntlr = modelXbrl.modelManager.cntlr
cntlr.showStatus(
_("Loading XBRL Formula file: {0}").format(os.path.basename(filepath)))
try:
import tatsu.exceptions
except ImportError:
modelXbrl.error("xf:missingTatsu",
"Python library module Tatsu must be installed.")
from .xf import XFParser
with open(filepath, "r") as f:
xf = f.read()
parser = XFParser()
try:
ast = parser.parse(xf, rule_name='module')
except tatsu.exceptions.FailedParse as err:
modelXbrl.error("xf:syntax",
"Unrecoverable error: %(error)s",
modelObject=modelXbrl,
error=err)
# create dummy modelDocument for successful plugin execution by ModelDopcument
from arelle.ModelDocument import Type, create as createModelDocument
doc = createModelDocument(modelXbrl,
Type.LINKBASE,
filepath,
documentEncoding="utf-8",
initialXml='''
<!-- Dummy linkbase -->
<link:linkbase
xmlns:link="http://www.xbrl.org/2003/linkbase"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
/>
''')
return doc
def loadXbrlFromDB(self, loadDBsaveToFile):
# load from database
modelXbrl = self.modelXbrl
# find instance in DB
instanceURI = os.path.basename(loadDBsaveToFile)
results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant"
" FROM dInstance WHERE FileName = '{}'"
.format(instanceURI))
instanceId = moduleId = None
for instanceId, moduleId, entScheme, entId, datePerEnd in results:
break
# find module in DB
results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId))
xbrlSchemaRef = None
for result in results:
xbrlSchemaRef = result[0]
break
if not instanceId or not xbrlSchemaRef:
raise XPDBException("sqlDB:MissingDTS",
_("The instance and module were not found for %(instanceURI)"),
instanceURI = instanceURI)
if modelXbrl.skipDTS:
# find prefixes and namespaces in DB
results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]")
dpmPrefixedNamespaces = dict((prefix, namespace)
for owner, prefix, namespace in results)
# create the instance document and resulting filing
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(modelXbrl,
Type.INSTANCE,
loadDBsaveToFile,
schemaRefs=[xbrlSchemaRef],
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement,
'xbrl-streamable-instance',
'version="1.0" contextBuffer="1"')
# add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue)
# filing indicator code IDs
# get filing indicators
results = self.execute("SELECT mToT.TemplateOrTableCode "
" FROM dFilingIndicator dFI, mTemplateOrTable mToT "
" WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID"
.format(instanceId))
filingIndicatorCodes = [code[0] for code in results]
if filingIndicatorCodes:
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
{}, [], [],
id='c')
filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators)
for filingIndicatorCode in filingIndicatorCodes:
modelXbrl.createFact(qnFindFilingIndicator,
parent=filingIndicatorsTuple,
attributes={"contextRef": "c"},
text=filingIndicatorCode)
# facts in this instance
factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards,"
" Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue "
"FROM dFact WHERE InstanceID = {} "
"ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL "
" THEN DataPointSignature"
" ELSE DataPointSignatureWithValuesForWildcards"
" END, instr(DataPointSignature,'|') + 1)"
.format(instanceId))
# results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal
# get typed dimension values
prefixedNamespaces = modelXbrl.prefixedNamespaces
prefixedNamespaces["iso4217"] = XbrlConst.iso4217
if modelXbrl.skipDTS:
prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed
cntxTbl = {} # index by d
unitTbl = {}
def typedDimElt(s):
# add xmlns into s for known qnames
tag, angleBrkt, rest = s[1:].partition('>')
text, angleBrkt, rest = rest.partition("<")
qn = qname(tag, prefixedNamespaces)
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
# contexts and facts
for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl:
metric, sep, dims = (dpSigTypedDims or dpSig).partition('|')
conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces)
concept = modelXbrl.qnameConcepts.get(conceptQn)
isNumeric = isBool = isDateTime = isQName = isText = False
if concept is not None:
if concept.isNumeric:
isNumeric = True
else:
baseXbrliType = concept.baseXbrliType
if baseXbrliType == "booleanItemType":
isBool = True
elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
isDateTime = True
elif baseXbrliType == "QNameItemType":
isQName = True
else:
c = conceptQn.localName[0]
if c == 'm':
isNumeric = True
elif c == 'd':
isDateTime = True
elif c == 'b':
isBool = True
elif c == 'e':
isQName = True
isText = not (isNumeric or isBool or isDateTime or isQName)
if isinstance(datePerEnd, _STR_BASE):
datePerEnd = datetimeValue(datePerEnd, addOneDay=True)
cntxKey = (dims, entId, datePerEnd)
if cntxKey in cntxTbl:
cntxId = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
cntxTbl[cntxKey] = cntxId
qnameDims = {}
if dims:
for dim in dims.split('|'):
dQn, sep, dVal = dim[:-1].partition('(')
dimQname = qname(dQn, prefixedNamespaces)
if dVal.startswith('<'):
mem = typedDimElt(dVal) # typed dim
else:
mem = qname(dVal, prefixedNamespaces) # explicit dim (even if treat-as-typed)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario")
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId)
if unit:
if unit in unitTbl:
unitId = unitTbl[unit]
else:
unitQn = qname(unit, prefixedNamespaces)
unitId = 'u{}'.format(unitQn.localName)
unitTbl[unit] = unitId
modelXbrl.createUnit([unitQn], [], id=unitId)
else:
unitId = None
attrs = {"contextRef": cntxId}
if unitId:
attrs["unitRef"] = unitId
if dec is not None:
if isinstance(dec, float): # must be an integer
dec = int(dec)
elif isinstance(dec, _STR_BASE) and '.' in dec:
dec = dec.partition('.')[0] # drop .0 from any SQLite string
attrs["decimals"] = str(dec) # somehow it is float from the database
if False: # fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
elif numVal is not None:
num = roundValue(numVal, None, dec) # round using reported decimals
if dec is None or dec == "INF": # show using decimals or reported format
dec = len(numVal.partition(".")[2])
else: # max decimals at 28
dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num))
elif dateVal is not None:
text = dateVal
elif boolVal is not None:
text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false'
else:
if isQName: # declare namespace
addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces))
text = textVal
modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
# add footnotes if any
# save to file
modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
return modelXbrl.modelDocument
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
from openpyxl import load_workbook
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
_return = None # modelDocument or an exception
try:
currentAction = "initializing"
startingErrorCount = len(modelXbrl.errors)
startedAt = time.time()
if os.path.isabs(oimFile):
# allow relative filenames to loading directory
priorCWD = os.getcwd()
os.chdir(os.path.dirname(oimFile))
else:
priorCWD = None
currentAction = "determining file type"
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv") # this option is not currently supported
isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
isCSVorXL = isCSV or isXL
instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"
_csvwContext = None
if isJSON:
errPrefix = "xbrlje"
currentAction = "loading and parsing JSON OIM file"
def loadDict(keyValuePairs):
_dict = OrderedDict() # preserve fact order in resulting instance
for key, value in keyValuePairs:
if isinstance(value, dict):
if DUPJSONKEY in value:
for _errKey, _errValue, _otherValue in value[DUPJSONKEY]:
if key == "prefixes":
modelXbrl.error("xbrlje:duplicatedPrefix",
_("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"),
modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue)
del value[DUPJSONKEY]
if key in _dict:
if DUPJSONKEY not in _dict:
_dict[DUPJSONKEY] = []
_dict[DUPJSONKEY].append((key, value, _dict[key]))
else:
_dict[key] = value
return _dict
with io.open(oimFile, 'rt', encoding='utf-8') as f:
oimObject = json.load(f, object_pairs_hook=loadDict)
# check if it's a CSVW metadata
_csvwContext = oimObject.get("@context")
if _csvwContext == "http://www.w3.org/ns/csvw" or (
isinstance(_csvwContext, list) and "http://www.w3.org/ns/csvw" in _csvwContext):
isJSON = False
isCSV = True
else:
if oimObject.get("documentType", None) != "http://www.xbrl.org/WGWD/YYYY-MM-DD/xbrl-json":
# is it likely to be an OIM document?
if any(t in oimObject for t in ("dtsReferences", "prefixes", "facts")):
raise OIMException("xbrlje:missingJSONdocumentType",
_("Required documentType is missing from JSON input"))
else:
raise NotOIMException() # return None, not an OIM document
missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject]
if missing:
raise OIMException("xbrlje:missingJSONElements",
_("Required element(s) are missing from JSON input: %(missing)s"),
missing = ", ".join(missing))
currentAction = "identifying JSON objects"
dtsReferences = oimObject["dtsReferences"]
prefixesList = oimObject["prefixes"].items()
facts = oimObject["facts"]
footnotes = oimObject["facts"] # shares this object
if isCSV:
errPrefix = "xbrlce"
currentAction = "compiling metadata"
if sys.version[0] >= '3':
csvOpenMode = 'w'
csvOpenNewline = ''
else:
csvOpenMode = 'wb' # for 2.7
csvOpenNewline = None
if not _csvwContext: # json metadata wasn't specified, try to find it
raise OIMException("xbrlce:missingCSVMetadata",
_("Unable to identify CSV metadata file"))
# process CSV metadata
# mandatory sections of metadata file
oimMetadata = oimObject.get(CSVmetadata)
if not oimMetadata:
raise OIMException("xbrlce:missingOIMMetadata",
_("Unable to identify OIM metadata infile"))
if oimMetadata.get("documentType") != CSVdocumentType:
raise OIMException("xbrlce:documentType",
_("Document type %(documentType)s not recognized, expecting %(expectedDocumentType)s"),
documentType=oimMetadata.get("documentType"), expectedDocumentType=CSVdocumentType)
dtsReferences = oimMetadata.get("dtsReferences", {})
prefixesList = oimMetadata.get("prefixes", {}).items()
topLevelAspects = oimObject.get(CSVaspects, {})
currentAction = "loading CSV facts tables"
facts = []
footnotes = []
_dir = os.path.dirname(oimFile)
for oimTable in oimObject.get("tables", []):
tableType = oimTable.get(CSVtableType)
tableLevelAspects = oimTable.get(CSVaspects, {})
tableUrl = oimTable.get("url")
tableColumns = oimTable.get("tableSchema",{}).get("columns", [])
# compile column dependencies
aspectCols = []
factCols = []
for iCol, col in enumerate(tableColumns):
if CSVcolumnAspect in col:
aspectCols.append(iCol)
if CSVtupleFactAspects in col or CSVsimpleFactAspects in col:
factCols.append(iCol)
if tableType not in ("fact", "footnote"):
modelXbrl.error("xbrlce:unrecognizedTableType",
_("Table type %(tableType)s was not recognized, table URI %(uri)s."),
modelObject=modelXbrl, uri=oimFile, tableType=tableType)
continue
if not tableColumns:
modelXbrl.error("xbrlce:noTableColumns",
_("Table has no columns, table URI %(uri)s."),
modelObject=modelXbrl, uri=_uri)
continue
tableAspects = topLevelAspects.copy()
tableAspects.update(tableLevelAspects)
filepath = os.path.join(_dir, tableUrl)
tupleIds = set()
with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
if tableType == "fact":
colAspects = tableAspects.copy()
specificColAspects = defaultdict(dict)
for iCol in aspectCols:
value = row[iCol]
aspect = tableColumns[iCol][CSVcolumnAspect]
if isinstance(aspect, str): # applies to all cols
colAspects[aspect] = value
elif isinstance(aspect, dict): # applies to specific cols
for _aspect, _colNames in aspect.items():
for _colName in _colNames:
specificColAspects[_colName][_aspect] = value
for iCol in factCols:
fact = {"aspects": {}}
cellValue = row[iCol]
if cellValue == "": # no fact produced for this cell
continue
tableCol = tableColumns[iCol]
if CSVtupleFactAspects in tableCol:
aspectsObjectName = CSVtupleFactAspects
else:
aspectsObjectName = CSVsimpleFactAspects
cellAspects = (colAspects,
specificColAspects.get(tableCol.get("name"), EMPTYDICT),
tableColumns[iCol].get(aspectsObjectName), EMPTYDICT)
inapplicableAspects = set()
if tableCol.get(CSVtupleReferenceId) == "true":
if cellValue:
if cellValue in tupleIds:
continue # don't duplicate parent tuple
fact["id"] = cellValue
tupleIds.add(cellValue) # prevent tuple duplication
elif CSVsimpleFactAspects in tableCol:
fact["value"] = csvCellValue(cellValue)
if CSVtupleFactAspects in tableCol:
inapplicableAspects.update(oimSimpleFactAspects)
for _aspects in cellAspects:
for aspectName, aspectValue in _aspects.items():
if not aspectName.startswith(oimPrefix):
inapplicableAspects.add(aspectName)
# block any row aspect produced by this column from this column's fact
_colAspect = tableCol.get(CSVcolumnAspect)
if isinstance(_colAspect, str): # applies to all cols
inapplicableAspects.add(_colAspect)
for _aspects in cellAspects:
if _aspects:
for aspectName, aspectValue in _aspects.items():
if aspectName not in inapplicableAspects and aspectValue != "":
if ":" in aspectName:
fact["aspects"][aspectName] = csvCellValue(aspectValue)
else:
fact[aspectName] = aspectValue
facts.append(fact)
elif tableType == "footnote":
footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col))
del tupleIds
elif isXL:
errPrefix = "xbrlwe"
currentAction = "identifying workbook input worksheets"
oimWb = load_workbook(oimFile, read_only=True, data_only=True)
sheetNames = oimWb.get_sheet_names()
if (not any(sheetName == "prefixes" for sheetName in sheetNames) or
not any(sheetName == "dtsReferences" for sheetName in sheetNames) or
not any("facts" in sheetName for sheetName in sheetNames)):
raise OIMException("xbrlwe:missingWorkbookWorksheets",
_("Unable to identify worksheet tabs for dtsReferences, prefixes or facts"))
currentAction = "loading worksheet: dtsReferences"
dtsReferences = []
for i, row in enumerate(oimWb["dtsReferences"]):
if i == 0:
header = [col.value for col in row]
else:
dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row)))
currentAction = "loading worksheet: prefixes"
prefixesList = []
for i, row in enumerate(oimWb["prefixes"]):
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row))
else:
prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value))
defaults = {}
if "defaults" in sheetNames:
currentAction = "loading worksheet: defaults"
for i, row in enumerate(oimWb["defaults"]):
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row))
fileCol = header["file"]
else:
defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol)
facts = []
for sheetName in sheetNames:
if sheetName == "facts" or "-facts" in sheetName:
currentAction = "loading worksheet: {}".format(sheetName)
tableDefaults = defaults.get(sheetName, {})
for i, row in enumerate(oimWb[sheetName]):
if i == 0:
header = [col.value for col in row]
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col.value is not None:
if header[j]: # skip cols with no header
if header[j].endswith("Value"):
fact["value"] = str(col.value)
else:
fact[header[j]] = str(col.value)
facts.append(fact)
footnotes = []
if "footnotes" in sheetNames:
currentAction = "loading worksheet: footnotes"
for i, row in enumerate(oimWb["footnotes"]):
if i == 0:
header = [col.value for j,col in enumerate(row) if col.value]
else:
footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value))
currentAction = "identifying default dimensions"
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
currentAction = "validating OIM"
prefixes = {}
prefixedUris = {}
for _prefix, _uri in prefixesList:
if not _prefix:
modelXbrl.error("{}:emptyPrefix".format(errPrefix),
_("The empty string must not be used as a prefix, uri %(uri)s"),
modelObject=modelXbrl, uri=_uri)
elif not NCNamePattern.match(_prefix):
modelXbrl.error("oime:prefixPattern",
_("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"),
modelObject=modelXbrl, prefix=_prefix, uri=_uri)
elif _prefix in prefixes:
modelXbrl.error("{}:duplicatedPrefix".format(errPrefix),
_("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"),
modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri)
elif _uri in prefixedUris:
modelXbrl.error("{}:duplicatedUri".format(errPrefix),
_("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"),
modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix)
else:
prefixes[_prefix] = _uri
prefixedUris[_uri] = _prefix
for _nsOim in nsOim:
if _nsOim in prefixedUris:
_oimPrefix = prefixedUris[_nsOim]
if not _oimPrefix:
raise OIMException("oime:noOimPrefix",
_("The oim namespace must have a declared prefix"))
# create the instance document
currentAction = "creating instance document"
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = _return = createModelDocument(
modelXbrl,
Type.INSTANCE,
instanceFileName,
schemaRefs=[dtsRef["href"]
for dtsRef in dtsReferences
if dtsRef.get("type") == "schema" and dtsRef.get("href")],
isEntry=True,
initialComment="extracted from OIM {}".format(mappedUri),
documentEncoding="utf-8")
modelXbrl.modelDocument.inDTS = True
# add linkbase, role and arcrole refs
for refType in ("linkbase", "role", "arcrole"):
for dtsRef in dtsReferences:
if dtsRef.get("type") == refType and dtsRef.get("href"):
elt = addChild(modelXbrl.modelDocument.xmlRootElement,
qname(link, refType+"Ref"),
attributes=(("{http://www.w3.org/1999/xlink}href", dtsRef["href"]),
("{http://www.w3.org/1999/xlink}type", "simple")))
href = modelXbrl.modelDocument.discoverHref(elt)
if href:
_elt, hrefDoc, hrefId = href
_defElt = hrefDoc.idObjects.get(hrefId)
_uriAttrName = refType + "URI"
_uriAttrValue = _defElt.get(_uriAttrName)
if _defElt is not None and _uriAttrValue:
elt.set(_uriAttrName, _uriAttrValue)
cntxTbl = {}
unitTbl = {}
# determine tuple dependency order
idFacts = {}
parentedFacts = defaultdict(list)
for i, fact in enumerate(facts):
aspects = fact["aspects"]
id = fact.get("id")
tupleParent = fact.get("aspects", EMPTYDICT).get(oimTupleParent)
if id is not None:
idFacts[id] = i
parentedFacts[tupleParent].append(i) # root facts have tupleParent None
currentAction = "creating facts"
def createModelFact(fact, parentModelFact, topTupleFact):
aspects = fact.get("aspects", EMPTYDICT)
if oimConcept not in aspects:
modelXbrl.error("{}:conceptQName".format(errPrefix),
_("The concept QName could not be determined"),
modelObject=modelXbrl)
return
conceptQn = qname(aspects[oimConcept], prefixes)
concept = modelXbrl.qnameConcepts.get(conceptQn)
attrs = {}
if concept.isItem:
missingAspects = []
if oimEntity not in aspects:
missingAspects.append(oimEntity)
if oimPeriod not in aspects and (oimPeriodStart not in aspects or oimPeriodEnd not in aspects):
missingAspects.append(oimPeriod)
if missingAspects:
modelXbrl.error("{}:missingAspects".format(errPrefix),
_("The concept %(element)s is missing aspects %(missingAspects)s"),
modelObject=modelXbrl, element=conceptQn, missingAspects=", ".join(missingAspects))
return
entityAsQn = qname(aspects[oimEntity], prefixes) or qname("error",fact[oimEntity])
if oimPeriod in aspects:
periodStart = periodEnd = aspects[oimPeriod]
if oimPeriodStart in aspects and oimPeriodEnd in aspects:
periodStart = aspects[oimPeriodStart]
periodEnd = aspects[oimPeriodEnd]
cntxKey = ( # hashable context key
("periodType", concept.periodType),
("entity", entityAsQn),
("periodStart", periodStart),
("periodEnd", periodEnd)) + tuple(sorted(
(dimName, dimVal["value"] if isinstance(dimVal,dict) else dimVal)
for dimName, dimVal in aspects.items()
if ":" in dimName and not dimName.startswith(oimPrefix)))
if cntxKey in cntxTbl:
_cntx = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
qnameDims = {}
for dimName, dimVal in aspects.items():
if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal:
dimQname = qname(dimName, prefixes)
dimConcept = modelXbrl.qnameConcepts.get(dimQname)
if dimConcept is None:
modelXbrl.error("{}:taxonomyDefinedAspectQName".format(errPrefix),
_("The taxonomy defined aspect concept QName %(qname)s could not be determined"),
modelObject=modelXbrl, qname=dimQname)
continue
if isinstance(dimVal, dict):
dimVal = dimVal["value"]
else:
dimVal = str(dimVal) # may be int or boolean
if isinstance(dimVal,str) and ":" in dimVal and dimVal.partition(':')[0] in prefixes:
mem = qname(dimVal, prefixes) # explicit dim
elif dimConcept.isTypedDimension:
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
_cntx = modelXbrl.createContext(
entityAsQn.namespaceURI,
entityAsQn.localName,
concept.periodType,
None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME),
dateTime(periodEnd, type=DATETIME),
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId,
beforeSibling=topTupleFact)
cntxTbl[cntxKey] = _cntx
if oimUnit in aspects and concept.isNumeric:
unitKey = aspects[oimUnit]
if unitKey in unitTbl:
_unit = unitTbl[unitKey]
else:
_unit = None
# validate unit
unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey)
if not UnitPattern.match(unitKeySub):
modelXbrl.error("oime:unitStringRepresentation",
_("Unit string representation is lexically invalid, %(unit)s"),
modelObject=modelXbrl, unit=unitKey)
else:
_mul, _sep, _div = unitKey.partition('/')
if _mul.startswith('('):
_mul = _mul[1:-1]
_muls = [u for u in _mul.split('*') if u]
if _div.startswith('('):
_div = _div[1:-1]
_divs = [u for u in _div.split('*') if u]
if _muls != sorted(_muls) or _divs != sorted(_divs):
modelXbrl.error("oime:unitStringRepresentation",
_("Unit string representation measures are not in alphabetical order, %(unit)s"),
modelObject=modelXbrl, unit=unitKey)
try:
mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"),
unit=u))
for u in _muls]
divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"),
unit=u))
for u in _divs]
unitId = 'u-{:02}'.format(len(unitTbl) + 1)
for _measures in mulQns, divQns:
for _measure in _measures:
addQnameValue(modelXbrl.modelDocument, _measure)
_unit = modelXbrl.createUnit(mulQns, divQns, id=unitId, beforeSibling=topTupleFact)
except OIMException as ex:
modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
unitTbl[unitKey] = _unit
else:
_unit = None
attrs["contextRef"] = _cntx.id
if fact.get("value") is None:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact["value"]
if concept.isNumeric:
if _unit is None:
return # skip creating fact because unit was invalid
attrs["unitRef"] = _unit.id
if "accuracy" in attrs or attrs.get(XbrlConst.qnXsiNil, "false") != "true":
attrs["decimals"] = fact.get("accuracy", "INF")
else:
text = None #tuple
id = fact.get("id")
if id is not None:
attrs["id"] = fact["id"]
# is value a QName?
if concept.baseXbrliType == "QName":
addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text, parent=parentModelFact, validate=False)
if id is not None and id in parentedFacts:
# create child facts
for i in sorted(parentedFacts[id],
key=lambda j: facts[j].get("aspects", EMPTYDICT).get(oimTupleOrder,0)):
createModelFact(facts[i], f, f if topTupleFact is None else topTupleFact)
# validate after creating tuple contents
xmlValidate(modelXbrl, f)
for i in parentedFacts[None]:
createModelFact(facts[i], None, None)
currentAction = "creating footnotes"
footnoteLinks = OrderedDict() # ELR elements
factLocs = {} # index by (linkrole, factId)
footnoteNbr = 0
locNbr = 0
for factOrFootnote in footnotes:
if "factId" in factOrFootnote:
factId = factOrFootnote["factId"]
factFootnotes = (factOrFootnote,) # CSV or XL
elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
factId = factOrFootnote["id"]
factFootnotes = factOrFootnote["footnotes"]
else:
factFootnotes = ()
for footnote in factFootnotes:
linkrole = footnote.get("group")
arcrole = footnote.get("footnoteType")
if not factId or not linkrole or not arcrole or not (
footnote.get("factRef") or footnote.get("footnote")):
# invalid footnote
continue
if linkrole not in footnoteLinks:
footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement,
XbrlConst.qnLinkFootnoteLink,
attributes={"{http://www.w3.org/1999/xlink}type": "extended",
"{http://www.w3.org/1999/xlink}role": linkrole})
footnoteLink = footnoteLinks[linkrole]
if (linkrole, factId) not in factLocs:
locNbr += 1
locLabel = "l_{:02}".format(locNbr)
factLocs[(linkrole, factId)] = locLabel
addChild(footnoteLink, XbrlConst.qnLinkLoc,
attributes={XLINKTYPE: "locator",
XLINKHREF: "#" + factId,
XLINKLABEL: locLabel})
locFromLabel = factLocs[(linkrole, factId)]
if footnote.get("footnote"):
footnoteNbr += 1
footnoteToLabel = "f_{:02}".format(footnoteNbr)
attrs = {XLINKTYPE: "resource",
XLINKLABEL: footnoteToLabel}
if footnote.get("language"):
attrs[XMLLANG] = footnote["language"]
# note, for HTML will need to build an element structure
addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"])
elif footnote.get("factRef"):
factRef = footnote.get("factRef")
if (linkrole, factRef) not in factLocs:
locNbr += 1
locLabel = "l_{:02}".format(locNbr)
factLocs[(linkrole, factRef)] = locLabel
addChild(footnoteLink, XbrlConst.qnLinkLoc,
attributes={XLINKTYPE: "locator",
XLINKHREF: "#" + factRef,
XLINKLABEL: locLabel})
footnoteToLabel = factLocs[(linkrole, factRef)]
footnoteArc = addChild(footnoteLink,
XbrlConst.qnLinkFootnoteArc,
attributes={XLINKTYPE: "arc",
XLINKARCROLE: arcrole,
XLINKFROM: locFromLabel,
XLINKTO: footnoteToLabel})
if footnoteLinks:
modelXbrl.modelDocument.linkbaseDiscover(footnoteLinks.values(), inInstance=True)
currentAction = "done loading facts and footnotes"
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
except NotOIMException as ex:
_return = ex # not an OIM document
except Exception as ex:
_return = ex
if isinstance(ex, OIMException):
modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
else:
modelXbrl.error("arelleOIMloader:error",
"Error while %(action)s, error %(error)s\ntraceback %(traceback)s",
modelObject=modelXbrl, action=currentAction, error=ex,
traceback=traceback.format_tb(sys.exc_info()[2]))
if priorCWD:
os.chdir(priorCWD) # restore prior current working directory startingErrorCount = len(modelXbrl.errors)
#if startingErrorCount < len(modelXbrl.errors):
# # had errors, don't allow ModelDocument.load to continue
# return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors")
return _return
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
from openpyxl import load_workbook
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
try:
currentAction = "initializing"
startedAt = time.time()
if os.path.isabs(oimFile):
# allow relative filenames to loading directory
priorCWD = os.getcwd()
os.chdir(os.path.dirname(oimFile))
else:
priorCWD = None
currentAction = "determining file type"
isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json")
isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
isCSVorXL = isCSV or isXL
instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"
if isJSON:
currentAction = "loading and parsing JSON OIM file"
with io.open(oimFile, 'rt', encoding='utf-8') as f:
oimObject = json.load(f)
missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject]
if missing:
raise OIMException("oime:missingJSONelements",
_("The required elements %(missing)s {} in JSON input"),
missing = ", ".join(missing))
currentAction = "identifying JSON objects"
dtsReferences = oimObject["dtsReferences"]
prefixes = oimObject["prefixes"]
facts = oimObject["facts"]
footnotes = oimOject["facts"] # shares this object
elif isCSV:
currentAction = "identifying CSV input tables"
if sys.version[0] >= '3':
csvOpenMode = 'w'
csvOpenNewline = ''
else:
csvOpenMode = 'wb' # for 2.7
csvOpenNewline = None
oimFileBase = None
if "-facts" in oimFile:
oimFileBase = oimFile.partition("-facts")[0]
else:
for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"):
if oimFile.endswith(suffix):
oimFileBase = oimFile[:-length(suffix)]
break
if oimFileBase is None:
raise OIMException("oime:missingCSVtables",
_("Unable to identify CSV tables file name pattern"))
if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or
not os.path.exists(oimFileBase + "-prefixes.csv")):
raise OIMException("oime:missingCSVtables",
_("Unable to identify CSV tables for dtsReferences or prefixes"))
instanceFileName = oimFileBase + ".xbrl"
currentAction = "loading CSV dtsReferences table"
dtsReferences = []
with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
dtsReferences.append(dict((header[j], col) for j, col in enumerate(row)))
currentAction = "loading CSV prefixes table"
prefixes = {}
with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = dict((col,i) for i,col in enumerate(row))
else:
prefixes[row[header["prefix"]]] = row[header["URI"]]
currentAction = "loading CSV defaults table"
defaults = {}
if os.path.exists(oimFileBase + "-defaults.csv"):
with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
fileCol = row.index("file")
else:
defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol)
currentAction = "loading CSV facts tables"
facts = []
_dir = os.path.dirname(oimFileBase)
for filename in os.listdir(_dir):
filepath = os.path.join(_dir, filename)
if "-facts" in filename and filepath.startswith(oimFileBase):
tableDefaults = defaults.get(filename, {})
with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col is not None:
if header[j].endswith("Value"):
if col: # ignore empty columns (= null CSV value)
fact["value"] = col
else:
fact[header[j]] = col
facts.append(fact)
footnotes = []
if os.path.exists(oimFileBase + "-footnotes.csv"):
with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col))
elif isXL:
oimWb = load_workbook(oimFile, read_only=True, data_only=True)
sheetNames = oimWb.get_sheet_names()
if (not any(sheetName == "prefixes" for sheetName in sheetNames) or
not any(sheetName == "dtsReferences" for sheetName in sheetNames) or
not any("facts" in sheetName for sheetName in sheetNames)):
if priorCWD: os.chdir(priorCWD)
return None
try:
dtsReferences = []
for i, row in oimWb["dtsReferences"]:
if i == 0:
header = [col.value for col in row]
else:
dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row)))
prefixes = {}
for i, row in oimWb["dtsReferences"]:
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row))
else:
prefixes[row[header["prefix"]].value] = row[header["URI"].value]
defaults = {}
for i, row in oimW.get("defaults", ()):
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row))
fileCol = header["file"]
else:
defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol)
facts = []
_dir = os.path.dirpart(oimFileBase)
for sheetName in sheetNames:
if sheetName == "facts" or "-facts" in sheetName:
for i, row in oimWb[sheetName]:
if i == 0:
header = [col.value for col in row]
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col.value is not None:
if header[j].endswith("Value"):
fact["value"] = str(col.value)
else:
fact[header[j]] = str(col.value)
facts.append(fact)
footnotes = []
for i, row in oimWb.get("footnotes", ()):
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row) if col.value)
else:
footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value))
except Exception as ex:
if priorCWD: os.chdir(priorCWD)
return None
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
# create the instance document
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(
modelXbrl,
Type.INSTANCE,
instanceFileName,
schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"],
isEntry=True,
initialComment="extracted from OIM {}".format(mappedUri),
documentEncoding="utf-8")
cntxTbl = {}
unitTbl = {}
for fact in facts:
conceptQn = qname(fact["oim:concept"], prefixes)
concept = modelXbrl.qnameConcepts.get(conceptQn)
entityAsQn = qname(fact["oim:entity"], prefixes)
if "oim:period" in fact:
periodStart = fact["oim:period"]["start"]
periodEnd = fact["oim:period"]["end"]
else:
periodStart = fact["oim:periodStart"]
periodEnd = fact["oim:periodEnd"]
cntxKey = ( # hashable context key
("periodType", concept.periodType),
("entity", entityAsQn),
("periodStart", periodStart),
("periodEnd", periodEnd)) + tuple(sorted(
(dimName, dimVal)
for dimName, dimVal in fact.items()
if ":" in dimName and not dimName.startswith("oim:")))
if cntxKey in cntxTbl:
_cntx = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
qnameDims = {}
for dimName, dimVal in fact.items():
if ":" in dimName and not dimName.startswith("oim:"):
dimQname = qname(dimName, prefixes)
dimConcept = modelXbrl.qnameConcepts.get(dimQname)
if ":" in dimVal and dimVal.partition[':'][0] in prefixes:
mem = qname(dimVal, prefixes) # explicit dim
elif dimConcept.isTypedDimension:
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
_cntx = modelXbrl.createContext(
entityAsQn.namespaceURI,
entityAsQn.localName,
concept.periodType,
None if concept.periodType == "instnat" else dateTime(periodStart, type=DATETIME),
dateTime(periodEnd, type=DATETIME),
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId)
cntxTbl[cntxKey] = _cntx
if "oim:unit" in fact:
unitKey = fact["oim:unit"]
if unitKey in unitTbl:
_unit = unitTbl[unitKey]
else:
_mul, _sep, _div = unitKey.partition('/')
if _mul.startswith('('):
_mul = _mul[1:-1]
mulQns = [qname(u, prefixes) for u in _mul.split('*') if u]
if _div.startswith('('):
_div = _div[1:-1]
divQns = [qname(u, prefixes) for u in _div.split('*') if u]
unitId = 'u-{:02}'.format(len(unitTbl) + 1)
for _measures in mulQns, divQns:
for _measure in _measures:
addQnameValue(modelXbrl.modelDocument, _measure)
_unit = modelXbrl.createUnit(mulQns, divQns, id=unitId)
unitTbl[unitKey] = _unit
else:
_unit = None
attrs = {"contextRef": _cntx.id}
if fact.get("value") is None:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact["value"]
if fact.get("id"):
attrs["id"] = fact["id"]
if concept.isNumeric:
if _unit is not None:
attrs["unitRef"] = _unit.id
if "accuracy" in fact:
attrs["decimals"] = fact["accuracy"]
# is value a QName?
if concept.baseXbrliType == "QName":
addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
footnoteLinks = {} # ELR elements
factLocs = {} # index by (linkrole, factId)
footnoteNbr = 0
locNbr = 0
for factOrFootnote in footnotes:
if "factId" in factOrFootnote:
factId = factOrFootnote["factId"]
factFootnotes = (factOrFootnote,) # CSV or XL
elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
factId = factOrFootnote["id"]
factFootnotes = factOrFootnote["footnotes"]
else:
factFootnotes = ()
for footnote in factFootnotes:
linkrole = footnote.get("group")
arcrole = footnote.get("footnoteType")
if not factId or not linkrole or not arcrole or not (
footnote.get("factRef") or footnote.get("footnote")):
# invalid footnote
continue
if linkrole not in footnoteLinks:
footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement,
XbrlConst.qnLinkFootnoteLink,
attributes={"{http://www.w3.org/1999/xlink}type": "extended",
"{http://www.w3.org/1999/xlink}role": linkrole})
footnoteLink = footnoteLinks[linkrole]
if (linkrole, factId) not in factLocs:
locNbr += 1
locLabel = "l_{:02}".format(locNbr)
factLocs[(linkrole, factId)] = locLabel
addChild(footnoteLink, XbrlConst.qnLinkLoc,
attributes={XLINKTYPE: "locator",
XLINKHREF: "#" + factId,
XLINKLABEL: locLabel})
locLabel = factLocs[(linkrole, factId)]
if footnote.get("footnote"):
footnoteNbr += 1
footnoteLabel = "f_{:02}".format(footnoteNbr)
attrs = {XLINKTYPE: "resource",
XLINKLABEL: footnoteLabel}
if footnote.get("language"):
attrs[XMLLANG] = footnote["language"]
# note, for HTML will need to build an element structure
addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"])
elif footnote.get("factRef"):
factRef = footnote.get("factRef")
if (linkrole, factRef) not in factLocs:
locNbr += 1
locLabel = "f_{:02}".format(footnoteNbr)
factLoc[(linkrole, factRef)] = locLabel
addChild(footnoteLink, XbrlConst.qnLinkLoc,
attributes={XLINKTYPE: "locator",
XLINKHREF: "#" + factRef,
XLINKLABEL: locLabel})
footnoteLabel = factLoc[(linkrole, factId)]
footnoteArc = addChild(footnoteLink,
XbrlConst.qnLinkFootnoteArc,
attributes={XLINKTYPE: "arc",
XLINKARCROLE: arcrole,
XLINKFROM: locLabel,
XLINKTO: footnoteLabel})
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
except Exception as ex:
if ex is OIMException:
modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
else:
modelXbrl.error("Error while %(action)s, error %(error)s",
modelObject=modelXbrl, action=currentAction, error=ex)
if priorCWD:
os.chdir(priorCWD) # restore prior current working directory
return modelXbrl.modelDocument
def loadXbrlFromDB(self, loadDBsaveToFile):
# load from database
modelXbrl = self.modelXbrl
# find instance in DB
instanceURI = os.path.basename(loadDBsaveToFile)
results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant"
" FROM dInstance WHERE FileName = '{}'"
.format(instanceURI))
instanceId = moduleId = None
for instanceId, moduleId, entScheme, entId, datePerEnd in results:
break
# find module in DB
results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId))
xbrlSchemaRef = None
for result in results:
xbrlSchemaRef = result[0]
break
if not instanceId or not xbrlSchemaRef:
raise XPDBException("sqlDB:MissingDTS",
_("The instance and module were not found for %(instanceURI)"),
instanceURI = instanceURI)
if modelXbrl.skipDTS:
# find prefixes and namespaces in DB
results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]")
dpmPrefixedNamespaces = dict((prefix, namespace)
for owner, prefix, namespace in results)
# create the instance document and resulting filing
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(modelXbrl,
Type.INSTANCE,
loadDBsaveToFile,
schemaRefs=[xbrlSchemaRef],
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement,
'xbrl-streamable-instance',
'version="1.0" contextBuffer="1"')
# add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue)
# filing indicator code IDs
# get filing indicators
results = self.execute("SELECT mToT.TemplateOrTableCode "
" FROM dFilingIndicator dFI, mTemplateOrTable mToT "
" WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID"
.format(instanceId))
filingIndicatorCodes = [code[0] for code in results]
if filingIndicatorCodes:
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
{}, [], [],
id='c')
filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators)
for filingIndicatorCode in filingIndicatorCodes:
modelXbrl.createFact(qnFindFilingIndicator,
parent=filingIndicatorsTuple,
attributes={"contextRef": "c"},
text=filingIndicatorCode)
# facts in this instance
factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards,"
" Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue "
"FROM dFact WHERE InstanceID = {} "
"ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL "
" THEN DataPointSignature"
" ELSE DataPointSignatureWithValuesForWildcards"
" END, instr(DataPointSignature,'|') + 1)"
.format(instanceId))
# results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal
# get typed dimension values
prefixedNamespaces = modelXbrl.prefixedNamespaces
prefixedNamespaces["iso4217"] = XbrlConst.iso4217
if modelXbrl.skipDTS:
prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed
cntxTbl = {} # index by d
unitTbl = {}
def typedDimElt(s):
# add xmlns into s for known qnames
tag, angleBrkt, rest = s[1:].partition('>')
text, angleBrkt, rest = rest.partition("<")
qn = qname(tag, prefixedNamespaces)
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
# contexts and facts
for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl:
metric, sep, dims = (dpSigTypedDims or dpSig).partition('|')
conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces)
concept = modelXbrl.qnameConcepts.get(conceptQn)
isNumeric = isBool = isDateTime = isQName = isText = False
if concept is not None:
if concept.isNumeric:
isNumeric = True
else:
baseXbrliType = concept.baseXbrliType
if baseXbrliType == "booleanItemType":
isBool = True
elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
isDateTime = True
elif baseXbrliType == "QNameItemType":
isQName = True
else:
c = conceptQn.localName[0]
if c == 'm':
isNumeric = True
elif c == 'd':
isDateTime = True
elif c == 'b':
isBool = True
elif c == 'e':
isQName = True
isText = not (isNumeric or isBool or isDateTime or isQName)
if isinstance(datePerEnd, _STR_BASE):
datePerEnd = datetimeValue(datePerEnd, addOneDay=True)
cntxKey = (dims, entId, datePerEnd)
if cntxKey in cntxTbl:
cntxId = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
cntxTbl[cntxKey] = cntxId
qnameDims = {}
if dims:
for dim in dims.split('|'):
dQn, sep, dVal = dim[:-1].partition('(')
dimQname = qname(dQn, prefixedNamespaces)
if dVal.startswith('<'): # typed dim
mem = typedDimElt(dVal)
else:
mem = qname(dVal, prefixedNamespaces)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario")
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId)
if unit:
if unit in unitTbl:
unitId = unitTbl[unit]
else:
unitQn = qname(unit, prefixedNamespaces)
unitId = 'u{}'.format(unitQn.localName)
unitTbl[unit] = unitId
modelXbrl.createUnit([unitQn], [], id=unitId)
else:
unitId = None
attrs = {"contextRef": cntxId}
if unitId:
attrs["unitRef"] = unitId
if dec is not None:
if isinstance(dec, float): # must be an integer
dec = int(dec)
elif isinstance(dec, _STR_BASE) and '.' in dec:
dec = dec.partition('.')[0] # drop .0 from any SQLite string
attrs["decimals"] = str(dec) # somehow it is float from the database
if False: # fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
elif numVal is not None:
num = roundValue(numVal, None, dec) # round using reported decimals
if dec is None or dec == "INF": # show using decimals or reported format
dec = len(numVal.partition(".")[2])
else: # max decimals at 28
dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num))
elif dateVal is not None:
text = dateVal
elif boolVal is not None:
text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false'
else:
if isQName: # declare namespace
addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces))
text = textVal
modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
# add footnotes if any
# save to file
modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
return modelXbrl.modelDocument
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
from openpyxl import load_workbook
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
try:
currentAction = "initializing"
startingErrorCount = len(modelXbrl.errors)
startedAt = time.time()
if os.path.isabs(oimFile):
# allow relative filenames to loading directory
priorCWD = os.getcwd()
os.chdir(os.path.dirname(oimFile))
else:
priorCWD = None
currentAction = "determining file type"
isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json")
isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
isCSVorXL = isCSV or isXL
instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"
if isJSON:
currentAction = "loading and parsing JSON OIM file"
def loadDict(keyValuePairs):
_dict = OrderedDict() # preserve fact order in resulting instance
for key, value in keyValuePairs:
if isinstance(value, dict):
if DUPJSONKEY in value:
for _errKey, _errValue, _otherValue in value[DUPJSONKEY]:
if key == "prefixes":
modelXbrl.error("oime:duplicatedPrefix",
_("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"),
modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue)
del value[DUPJSONKEY]
if key in _dict:
if DUPJSONKEY not in _dict:
_dict[DUPJSONKEY] = []
_dict[DUPJSONKEY].append((key, value, _dict[key]))
else:
_dict[key] = value
return _dict
with io.open(oimFile, 'rt', encoding='utf-8') as f:
oimObject = json.load(f, object_pairs_hook=loadDict)
missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject]
if missing:
raise OIMException("oime:missingJSONElements",
_("Required element(s) are missing from JSON input: %(missing)s"),
missing = ", ".join(missing))
currentAction = "identifying JSON objects"
dtsReferences = oimObject["dtsReferences"]
prefixesList = oimObject["prefixes"].items()
facts = oimObject["facts"]
footnotes = oimObject["facts"] # shares this object
elif isCSV:
currentAction = "identifying CSV input tables"
if sys.version[0] >= '3':
csvOpenMode = 'w'
csvOpenNewline = ''
else:
csvOpenMode = 'wb' # for 2.7
csvOpenNewline = None
oimFileBase = None
if "-facts" in oimFile:
oimFileBase = oimFile.partition("-facts")[0]
else:
for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"):
if oimFile.endswith(suffix):
oimFileBase = oimFile[:-len(suffix)]
break
if oimFileBase is None:
raise OIMException("oime:missingCSVTables",
_("Unable to identify CSV tables file name pattern"))
if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or
not os.path.exists(oimFileBase + "-prefixes.csv")):
raise OIMException("oime:missingCSVTables",
_("Unable to identify CSV tables for dtsReferences or prefixes"))
instanceFileName = oimFileBase + ".xbrl"
currentAction = "loading CSV dtsReferences table"
dtsReferences = []
with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
dtsReferences.append(dict((header[j], col) for j, col in enumerate(row)))
currentAction = "loading CSV prefixes table"
prefixesList = []
with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = dict((col,i) for i,col in enumerate(row))
else:
prefixesList.append((row[header["prefix"]], row[header["URI"]]))
defaults = {}
if os.path.exists(oimFileBase + "-defaults.csv"):
currentAction = "loading CSV defaults table"
with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
fileCol = row.index("file")
else:
defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol)
currentAction = "loading CSV facts tables"
facts = []
_dir = os.path.dirname(oimFileBase)
factsFileBasename = os.path.basename(oimFileBase) + "-facts"
for filename in os.listdir(_dir):
filepath = os.path.join(_dir, filename)
if filename.startswith(factsFileBasename):
currentAction = "loading CSV facts table {}".format(filename)
tableDefaults = defaults.get(filename, {})
with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col is not None:
if header[j]: # skip cols with no header
if header[j].endswith("Value"):
if col: # ignore empty columns (= null CSV value)
fact["value"] = col
else:
fact[header[j]] = col
facts.append(fact)
footnotes = []
if os.path.exists(oimFileBase + "-footnotes.csv"):
currentAction = "loading CSV footnotes table"
with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col))
elif isXL:
currentAction = "identifying workbook input worksheets"
oimWb = load_workbook(oimFile, read_only=True, data_only=True)
sheetNames = oimWb.get_sheet_names()
if (not any(sheetName == "prefixes" for sheetName in sheetNames) or
not any(sheetName == "dtsReferences" for sheetName in sheetNames) or
not any("facts" in sheetName for sheetName in sheetNames)):
raise OIMException("oime:missingWorkbookWorksheets",
_("Unable to identify worksheet tabs for dtsReferences, prefixes or facts"))
currentAction = "loading worksheet: dtsReferences"
dtsReferences = []
for i, row in enumerate(oimWb["dtsReferences"]):
if i == 0:
header = [col.value for col in row]
else:
dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row)))
currentAction = "loading worksheet: prefixes"
prefixesList = []
for i, row in enumerate(oimWb["prefixes"]):
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row))
else:
prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value))
defaults = {}
if "defaults" in sheetNames:
currentAction = "loading worksheet: defaults"
for i, row in enumerate(oimWb["defaults"]):
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row))
fileCol = header["file"]
else:
defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol)
facts = []
for sheetName in sheetNames:
if sheetName == "facts" or "-facts" in sheetName:
currentAction = "loading worksheet: {}".format(sheetName)
tableDefaults = defaults.get(sheetName, {})
for i, row in enumerate(oimWb[sheetName]):
if i == 0:
header = [col.value for col in row]
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col.value is not None:
if header[j]: # skip cols with no header
if header[j].endswith("Value"):
fact["value"] = str(col.value)
else:
fact[header[j]] = str(col.value)
facts.append(fact)
footnotes = []
if "footnotes" in sheetNames:
currentAction = "loading worksheet: footnotes"
for i, row in enumerate(oimWb["footnotes"]):
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row) if col.value)
else:
footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value))
currentAction = "identifying default dimensions"
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
currentAction = "validating OIM"
prefixes = {}
prefixedUris = {}
for _prefix, _uri in prefixesList:
if not _prefix:
modelXbrl.error("oime:emptyPrefix",
_("The empty string must not be used as a prefix, uri %(uri)s"),
modelObject=modelXbrl, uri=_uri)
elif not NCNamePattern.match(_prefix):
modelXbrl.error("oime:prefixPattern",
_("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"),
modelObject=modelXbrl, prefix=_prefix, uri=_uri)
elif _prefix in prefixes:
modelXbrl.error("oime:duplicatedPrefix",
_("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"),
modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri)
elif _uri in prefixedUris:
modelXbrl.error("oime:duplicatedUri",
_("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"),
modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix)
else:
prefixes[_prefix] = _uri
prefixedUris[_uri] = _prefix
oimPrefix = None
for _nsOim in nsOim:
if _nsOim in prefixedUris:
oimPrefix = prefixedUris[_nsOim]
if not oimPrefix:
raise OIMException("oime:noOimPrefix",
_("The oim namespace must have a declared prefix"))
oimConcept = "{}:concept".format(oimPrefix)
oimEntity = "{}:entity".format(oimPrefix)
oimPeriod = "{}:period".format(oimPrefix)
oimPeriodStart = "{}:periodStart".format(oimPrefix)
oimPeriodEnd = "{}:periodEnd".format(oimPrefix)
oimUnit = "{}:unit".format(oimPrefix)
oimPrefix = "{}:".format(oimPrefix)
# create the instance document
currentAction = "creating instance document"
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(
modelXbrl,
Type.INSTANCE,
instanceFileName,
schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"],
isEntry=True,
initialComment="extracted from OIM {}".format(mappedUri),
documentEncoding="utf-8")
cntxTbl = {}
unitTbl = {}
currentAction = "creating facts"
for fact in facts:
conceptQn = qname(fact[oimConcept], prefixes)
concept = modelXbrl.qnameConcepts.get(conceptQn)
entityAsQn = qname(fact[oimEntity], prefixes)
if oimPeriod in fact:
periodStart = fact[oimPeriod]["start"]
periodEnd = fact[oimPeriod]["end"]
else:
periodStart = fact[oimPeriodStart]
periodEnd = fact[oimPeriodEnd]
cntxKey = ( # hashable context key
("periodType", concept.periodType),
("entity", entityAsQn),
("periodStart", periodStart),
("periodEnd", periodEnd)) + tuple(sorted(
(dimName, dimVal)
for dimName, dimVal in fact.items()
if ":" in dimName and not dimName.startswith(oimPrefix)))
if cntxKey in cntxTbl:
_cntx = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
qnameDims = {}
for dimName, dimVal in fact.items():
if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal:
dimQname = qname(dimName, prefixes)
dimConcept = modelXbrl.qnameConcepts.get(dimQname)
if ":" in dimVal and dimVal.partition(':')[0] in prefixes:
mem = qname(dimVal, prefixes) # explicit dim
elif dimConcept.isTypedDimension:
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
_cntx = modelXbrl.createContext(
entityAsQn.namespaceURI,
entityAsQn.localName,
concept.periodType,
None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME),
dateTime(periodEnd, type=DATETIME),
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId)
cntxTbl[cntxKey] = _cntx
if oimUnit in fact:
unitKey = fact[oimUnit]
if unitKey in unitTbl:
_unit = unitTbl[unitKey]
else:
_unit = None
# validate unit
unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey)
if not UnitPattern.match(unitKeySub):
modelXbrl.error("oime:unitStringRepresentation",
_("Unit string representation is lexically invalid, %(unit)s"),
modelObject=modelXbrl, unit=unitKey)
else:
_mul, _sep, _div = unitKey.partition('/')
if _mul.startswith('('):
_mul = _mul[1:-1]
_muls = [u for u in _mul.split('*') if u]
if _div.startswith('('):
_div = _div[1:-1]
_divs = [u for u in _div.split('*') if u]
if _muls != sorted(_muls) or _divs != sorted(_divs):
modelXbrl.error("oime:unitStringRepresentation",
_("Unit string representation measures are not in alphabetical order, %(unit)s"),
modelObject=modelXbrl, unit=unitKey)
try:
mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"),
unit=u))
for u in _muls]
divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"),
unit=u))
for u in _divs]
unitId = 'u-{:02}'.format(len(unitTbl) + 1)
for _measures in mulQns, divQns:
for _measure in _measures:
addQnameValue(modelXbrl.modelDocument, _measure)
_unit = modelXbrl.createUnit(mulQns, divQns, id=unitId)
except OIMException as ex:
modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
unitTbl[unitKey] = _unit
else:
_unit = None
attrs = {"contextRef": _cntx.id}
if fact.get("value") is None:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact["value"]
if fact.get("id"):
attrs["id"] = fact["id"]
if concept.isNumeric:
if _unit is None:
continue # skip creating fact because unit was invalid
attrs["unitRef"] = _unit.id
if "accuracy" in fact:
attrs["decimals"] = fact["accuracy"]
# is value a QName?
if concept.baseXbrliType == "QName":
addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
currentAction = "creating footnotes"
footnoteLinks = {} # ELR elements
factLocs = {} # index by (linkrole, factId)
footnoteNbr = 0
locNbr = 0
for factOrFootnote in footnotes:
if "factId" in factOrFootnote:
factId = factOrFootnote["factId"]
factFootnotes = (factOrFootnote,) # CSV or XL
elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
factId = factOrFootnote["id"]
factFootnotes = factOrFootnote["footnotes"]
else:
factFootnotes = ()
for footnote in factFootnotes:
linkrole = footnote.get("group")
arcrole = footnote.get("footnoteType")
if not factId or not linkrole or not arcrole or not (
footnote.get("factRef") or footnote.get("footnote")):
# invalid footnote
continue
if linkrole not in footnoteLinks:
footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement,
XbrlConst.qnLinkFootnoteLink,
attributes={"{http://www.w3.org/1999/xlink}type": "extended",
"{http://www.w3.org/1999/xlink}role": linkrole})
footnoteLink = footnoteLinks[linkrole]
if (linkrole, factId) not in factLocs:
locNbr += 1
locLabel = "l_{:02}".format(locNbr)
factLocs[(linkrole, factId)] = locLabel
addChild(footnoteLink, XbrlConst.qnLinkLoc,
attributes={XLINKTYPE: "locator",
XLINKHREF: "#" + factId,
XLINKLABEL: locLabel})
locLabel = factLocs[(linkrole, factId)]
if footnote.get("footnote"):
footnoteNbr += 1
footnoteLabel = "f_{:02}".format(footnoteNbr)
attrs = {XLINKTYPE: "resource",
XLINKLABEL: footnoteLabel}
if footnote.get("language"):
attrs[XMLLANG] = footnote["language"]
# note, for HTML will need to build an element structure
addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"])
elif footnote.get("factRef"):
factRef = footnote.get("factRef")
if (linkrole, factRef) not in factLocs:
locNbr += 1
locLabel = "f_{:02}".format(footnoteNbr)
factLoc[(linkrole, factRef)] = locLabel
addChild(footnoteLink, XbrlConst.qnLinkLoc,
attributes={XLINKTYPE: "locator",
XLINKHREF: "#" + factRef,
XLINKLABEL: locLabel})
footnoteLabel = factLoc[(linkrole, factId)]
footnoteArc = addChild(footnoteLink,
XbrlConst.qnLinkFootnoteArc,
attributes={XLINKTYPE: "arc",
XLINKARCROLE: arcrole,
XLINKFROM: locLabel,
XLINKTO: footnoteLabel})
currentAction = "done loading facts and footnotes"
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
except Exception as ex:
if isinstance(ex, OIMException):
modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
else:
modelXbrl.error("arelleOIMloader:error",
"Error while %(action)s, error %(error)s\ntraceback %(traceback)s",
modelObject=modelXbrl, action=currentAction, error=ex,
traceback=traceback.format_tb(sys.exc_info()[2]))
if priorCWD:
os.chdir(priorCWD) # restore prior current working directory startingErrorCount = len(modelXbrl.errors)
if startingErrorCount < len(modelXbrl.errors):
# had errors, don't allow ModelDocument.load to continue
return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors")
return getattr(modelXbrl, "modelDocument", None) # none if returning from exception
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
from openpyxl import load_workbook
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
try:
currentAction = "initializing"
startingErrorCount = len(modelXbrl.errors)
startedAt = time.time()
if os.path.isabs(oimFile):
# allow relative filenames to loading directory
priorCWD = os.getcwd()
os.chdir(os.path.dirname(oimFile))
else:
priorCWD = None
currentAction = "determining file type"
isJSON = oimFile.endswith(
".json") and not oimFile.endswith("-metadata.json")
isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
isCSVorXL = isCSV or isXL
instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"
if isJSON:
currentAction = "loading and parsing JSON OIM file"
def loadDict(keyValuePairs):
_dict = OrderedDict(
) # preserve fact order in resulting instance
for key, value in keyValuePairs:
if isinstance(value, dict):
if DUPJSONKEY in value:
for _errKey, _errValue, _otherValue in value[
DUPJSONKEY]:
if key == "prefixes":
modelXbrl.error(
"oime:duplicatedPrefix",
_("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"
),
modelObject=modelXbrl,
prefix=_errKey,
uri1=_errValue,
uri2=_otherValue)
del value[DUPJSONKEY]
if key in _dict:
if DUPJSONKEY not in _dict:
_dict[DUPJSONKEY] = []
_dict[DUPJSONKEY].append((key, value, _dict[key]))
else:
_dict[key] = value
return _dict
with io.open(oimFile, 'rt', encoding='utf-8') as f:
oimObject = json.load(f, object_pairs_hook=loadDict)
missing = [
t for t in ("dtsReferences", "prefixes", "facts")
if t not in oimObject
]
if missing:
raise OIMException(
"oime:missingJSONElements",
_("Required element(s) are missing from JSON input: %(missing)s"
),
missing=", ".join(missing))
currentAction = "identifying JSON objects"
dtsReferences = oimObject["dtsReferences"]
prefixesList = oimObject["prefixes"].items()
facts = oimObject["facts"]
footnotes = oimObject["facts"] # shares this object
elif isCSV:
currentAction = "identifying CSV input tables"
if sys.version[0] >= '3':
csvOpenMode = 'w'
csvOpenNewline = ''
else:
csvOpenMode = 'wb' # for 2.7
csvOpenNewline = None
oimFileBase = None
if "-facts" in oimFile:
oimFileBase = oimFile.partition("-facts")[0]
else:
for suffix in ("-dtsReferences.csv", "-defaults.csv",
"-prefixes.csv", "-footnotes.csv",
"-metadata.json"):
if oimFile.endswith(suffix):
oimFileBase = oimFile[:-len(suffix)]
break
if oimFileBase is None:
raise OIMException(
"oime:missingCSVTables",
_("Unable to identify CSV tables file name pattern"))
if (not os.path.exists(oimFileBase + "-dtsReferences.csv")
or not os.path.exists(oimFileBase + "-prefixes.csv")):
raise OIMException(
"oime:missingCSVTables",
_("Unable to identify CSV tables for dtsReferences or prefixes"
))
instanceFileName = oimFileBase + ".xbrl"
currentAction = "loading CSV dtsReferences table"
dtsReferences = []
with io.open(oimFileBase + "-dtsReferences.csv",
'rt',
encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
dtsReferences.append(
dict(
(header[j], col) for j, col in enumerate(row)))
currentAction = "loading CSV prefixes table"
prefixesList = []
with io.open(oimFileBase + "-prefixes.csv",
'rt',
encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = dict((col, i) for i, col in enumerate(row))
else:
prefixesList.append(
(row[header["prefix"]], row[header["URI"]]))
defaults = {}
if os.path.exists(oimFileBase + "-defaults.csv"):
currentAction = "loading CSV defaults table"
with io.open(oimFileBase + "-defaults.csv",
'rt',
encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
fileCol = row.index("file")
else:
defaults[row[fileCol]] = dict(
(header[j], col) for j, col in enumerate(row)
if j != fileCol)
currentAction = "loading CSV facts tables"
facts = []
_dir = os.path.dirname(oimFileBase)
factsFileBasename = os.path.basename(oimFileBase) + "-facts"
for filename in os.listdir(_dir):
filepath = os.path.join(_dir, filename)
if filename.startswith(factsFileBasename):
currentAction = "loading CSV facts table {}".format(
filename)
tableDefaults = defaults.get(filename, {})
with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col is not None:
if header[
j]: # skip cols with no header
if header[j].endswith("Value"):
if col: # ignore empty columns (= null CSV value)
fact["value"] = col
else:
fact[header[j]] = col
facts.append(fact)
footnotes = []
if os.path.exists(oimFileBase + "-footnotes.csv"):
currentAction = "loading CSV footnotes table"
with io.open(oimFileBase + "-footnotes.csv",
'rt',
encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
footnotes.append(
dict((header[j], col)
for j, col in enumerate(row) if col))
elif isXL:
currentAction = "identifying workbook input worksheets"
oimWb = load_workbook(oimFile, read_only=True, data_only=True)
sheetNames = oimWb.get_sheet_names()
if (not any(sheetName == "prefixes" for sheetName in sheetNames)
or not any(sheetName == "dtsReferences"
for sheetName in sheetNames)
or not any("facts" in sheetName
for sheetName in sheetNames)):
raise OIMException(
"oime:missingWorkbookWorksheets",
_("Unable to identify worksheet tabs for dtsReferences, prefixes or facts"
))
currentAction = "loading worksheet: dtsReferences"
dtsReferences = []
for i, row in enumerate(oimWb["dtsReferences"]):
if i == 0:
header = [col.value for col in row]
else:
dtsReferences.append(
dict((header[j], col.value)
for j, col in enumerate(row)))
currentAction = "loading worksheet: prefixes"
prefixesList = []
for i, row in enumerate(oimWb["prefixes"]):
if i == 0:
header = dict((col.value, i) for i, col in enumerate(row))
else:
prefixesList.append((row[header["prefix"]].value,
row[header["URI"]].value))
defaults = {}
if "defaults" in sheetNames:
currentAction = "loading worksheet: defaults"
for i, row in enumerate(oimWb["defaults"]):
if i == 0:
header = dict(
(col.value, i) for i, col in enumerate(row))
fileCol = header["file"]
else:
defaults[row[fileCol].value] = dict(
(header[j], col.value) for j, col in enumerate(row)
if j != fileCol)
facts = []
for sheetName in sheetNames:
if sheetName == "facts" or "-facts" in sheetName:
currentAction = "loading worksheet: {}".format(sheetName)
tableDefaults = defaults.get(sheetName, {})
for i, row in enumerate(oimWb[sheetName]):
if i == 0:
header = [col.value for col in row]
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col.value is not None:
if header[j]: # skip cols with no header
if header[j].endswith("Value"):
fact["value"] = str(col.value)
else:
fact[header[j]] = str(col.value)
facts.append(fact)
footnotes = []
if "footnotes" in sheetNames:
currentAction = "loading worksheet: footnotes"
for i, row in enumerate(oimWb["footnotes"]):
if i == 0:
header = dict((col.value, i)
for i, col in enumerate(row)
if col.value)
else:
footnotes.append(
dict((header[j], col.value)
for j, col in enumerate(row) if col.value))
currentAction = "identifying default dimensions"
ValidateXbrlDimensions.loadDimensionDefaults(
modelXbrl) # needs dimension defaults
currentAction = "validating OIM"
prefixes = {}
prefixedUris = {}
for _prefix, _uri in prefixesList:
if not _prefix:
modelXbrl.error(
"oime:emptyPrefix",
_("The empty string must not be used as a prefix, uri %(uri)s"
),
modelObject=modelXbrl,
uri=_uri)
elif not NCNamePattern.match(_prefix):
modelXbrl.error(
"oime:prefixPattern",
_("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"
),
modelObject=modelXbrl,
prefix=_prefix,
uri=_uri)
elif _prefix in prefixes:
modelXbrl.error(
"oime:duplicatedPrefix",
_("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"
),
modelObject=modelXbrl,
prefix=_prefix,
uri1=prefixes[_prefix],
uri2=_uri)
elif _uri in prefixedUris:
modelXbrl.error(
"oime:duplicatedUri",
_("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"
),
modelObject=modelXbrl,
uri=_uri,
prefix1=prefixedUris[_uri],
prefix2=_prefix)
else:
prefixes[_prefix] = _uri
prefixedUris[_uri] = _prefix
oimPrefix = None
for _nsOim in nsOim:
if _nsOim in prefixedUris:
oimPrefix = prefixedUris[_nsOim]
if not oimPrefix:
raise OIMException(
"oime:noOimPrefix",
_("The oim namespace must have a declared prefix"))
oimConcept = "{}:concept".format(oimPrefix)
oimEntity = "{}:entity".format(oimPrefix)
oimPeriod = "{}:period".format(oimPrefix)
oimPeriodStart = "{}:periodStart".format(oimPrefix)
oimPeriodEnd = "{}:periodEnd".format(oimPrefix)
oimUnit = "{}:unit".format(oimPrefix)
oimPrefix = "{}:".format(oimPrefix)
# create the instance document
currentAction = "creating instance document"
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(
modelXbrl,
Type.INSTANCE,
instanceFileName,
schemaRefs=[
dtsRef["href"] for dtsRef in dtsReferences
if dtsRef["type"] == "schema"
],
isEntry=True,
initialComment="extracted from OIM {}".format(mappedUri),
documentEncoding="utf-8")
cntxTbl = {}
unitTbl = {}
currentAction = "creating facts"
for fact in facts:
conceptQn = qname(fact[oimConcept], prefixes)
concept = modelXbrl.qnameConcepts.get(conceptQn)
entityAsQn = qname(fact[oimEntity], prefixes)
if oimPeriod in fact:
periodStart = fact[oimPeriod]["start"]
periodEnd = fact[oimPeriod]["end"]
else:
periodStart = fact[oimPeriodStart]
periodEnd = fact[oimPeriodEnd]
cntxKey = ( # hashable context key
("periodType", concept.periodType), ("entity", entityAsQn),
("periodStart", periodStart),
("periodEnd", periodEnd)) + tuple(
sorted(
(dimName, dimVal)
for dimName, dimVal in fact.items() if ":" in dimName
and not dimName.startswith(oimPrefix)))
if cntxKey in cntxTbl:
_cntx = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
qnameDims = {}
for dimName, dimVal in fact.items():
if ":" in dimName and not dimName.startswith(
oimPrefix) and dimVal:
dimQname = qname(dimName, prefixes)
dimConcept = modelXbrl.qnameConcepts.get(dimQname)
if ":" in dimVal and dimVal.partition(
':')[0] in prefixes:
mem = qname(dimVal, prefixes) # explicit dim
elif dimConcept.isTypedDimension:
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
mem = addChild(modelXbrl.modelDocument,
dimConcept.typedDomainElement.qname,
text=dimVal,
appendChild=False)
qnameDims[dimQname] = DimValuePrototype(
modelXbrl, None, dimQname, mem, "segment")
_cntx = modelXbrl.createContext(
entityAsQn.namespaceURI,
entityAsQn.localName,
concept.periodType,
None if concept.periodType == "instant" else dateTime(
periodStart, type=DATETIME),
dateTime(periodEnd, type=DATETIME),
None, # no dimensional validity checking (like formula does)
qnameDims,
[],
[],
id=cntxId)
cntxTbl[cntxKey] = _cntx
if oimUnit in fact:
unitKey = fact[oimUnit]
if unitKey in unitTbl:
_unit = unitTbl[unitKey]
else:
_unit = None
# validate unit
unitKeySub = PrefixedQName.sub(
UnitPrefixedQNameSubstitutionChar, unitKey)
if not UnitPattern.match(unitKeySub):
modelXbrl.error(
"oime:unitStringRepresentation",
_("Unit string representation is lexically invalid, %(unit)s"
),
modelObject=modelXbrl,
unit=unitKey)
else:
_mul, _sep, _div = unitKey.partition('/')
if _mul.startswith('('):
_mul = _mul[1:-1]
_muls = [u for u in _mul.split('*') if u]
if _div.startswith('('):
_div = _div[1:-1]
_divs = [u for u in _div.split('*') if u]
if _muls != sorted(_muls) or _divs != sorted(_divs):
modelXbrl.error(
"oime:unitStringRepresentation",
_("Unit string representation measures are not in alphabetical order, %(unit)s"
),
modelObject=modelXbrl,
unit=unitKey)
try:
mulQns = [
qname(
u,
prefixes,
prefixException=OIMException(
"oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"
),
unit=u)) for u in _muls
]
divQns = [
qname(
u,
prefixes,
prefixException=OIMException(
"oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"
),
unit=u)) for u in _divs
]
unitId = 'u-{:02}'.format(len(unitTbl) + 1)
for _measures in mulQns, divQns:
for _measure in _measures:
addQnameValue(modelXbrl.modelDocument,
_measure)
_unit = modelXbrl.createUnit(mulQns,
divQns,
id=unitId)
except OIMException as ex:
modelXbrl.error(ex.code,
ex.message,
modelObject=modelXbrl,
**ex.msgArgs)
unitTbl[unitKey] = _unit
else:
_unit = None
attrs = {"contextRef": _cntx.id}
if fact.get("value") is None:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact["value"]
if fact.get("id"):
attrs["id"] = fact["id"]
if concept.isNumeric:
if _unit is None:
continue # skip creating fact because unit was invalid
attrs["unitRef"] = _unit.id
if "accuracy" in fact:
attrs["decimals"] = fact["accuracy"]
# is value a QName?
if concept.baseXbrliType == "QName":
addQnameValue(modelXbrl.modelDocument,
qname(text.strip(), prefixes))
f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
currentAction = "creating footnotes"
footnoteLinks = {} # ELR elements
factLocs = {} # index by (linkrole, factId)
footnoteNbr = 0
locNbr = 0
for factOrFootnote in footnotes:
if "factId" in factOrFootnote:
factId = factOrFootnote["factId"]
factFootnotes = (factOrFootnote, ) # CSV or XL
elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
factId = factOrFootnote["id"]
factFootnotes = factOrFootnote["footnotes"]
else:
factFootnotes = ()
for footnote in factFootnotes:
linkrole = footnote.get("group")
arcrole = footnote.get("footnoteType")
if not factId or not linkrole or not arcrole or not (
footnote.get("factRef") or footnote.get("footnote")):
# invalid footnote
continue
if linkrole not in footnoteLinks:
footnoteLinks[linkrole] = addChild(
modelXbrl.modelDocument.xmlRootElement,
XbrlConst.qnLinkFootnoteLink,
attributes={
"{http://www.w3.org/1999/xlink}type": "extended",
"{http://www.w3.org/1999/xlink}role": linkrole
})
footnoteLink = footnoteLinks[linkrole]
if (linkrole, factId) not in factLocs:
locNbr += 1
locLabel = "l_{:02}".format(locNbr)
factLocs[(linkrole, factId)] = locLabel
addChild(footnoteLink,
XbrlConst.qnLinkLoc,
attributes={
XLINKTYPE: "locator",
XLINKHREF: "#" + factId,
XLINKLABEL: locLabel
})
locLabel = factLocs[(linkrole, factId)]
if footnote.get("footnote"):
footnoteNbr += 1
footnoteLabel = "f_{:02}".format(footnoteNbr)
attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel}
if footnote.get("language"):
attrs[XMLLANG] = footnote["language"]
# note, for HTML will need to build an element structure
addChild(footnoteLink,
XbrlConst.qnLinkFootnote,
attributes=attrs,
text=footnote["footnote"])
elif footnote.get("factRef"):
factRef = footnote.get("factRef")
if (linkrole, factRef) not in factLocs:
locNbr += 1
locLabel = "f_{:02}".format(footnoteNbr)
factLoc[(linkrole, factRef)] = locLabel
addChild(footnoteLink,
XbrlConst.qnLinkLoc,
attributes={
XLINKTYPE: "locator",
XLINKHREF: "#" + factRef,
XLINKLABEL: locLabel
})
footnoteLabel = factLoc[(linkrole, factId)]
footnoteArc = addChild(footnoteLink,
XbrlConst.qnLinkFootnoteArc,
attributes={
XLINKTYPE: "arc",
XLINKARCROLE: arcrole,
XLINKFROM: locLabel,
XLINKTO: footnoteLabel
})
currentAction = "done loading facts and footnotes"
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
except Exception as ex:
if isinstance(ex, OIMException):
modelXbrl.error(ex.code,
ex.message,
modelObject=modelXbrl,
**ex.msgArgs)
else:
modelXbrl.error(
"arelleOIMloader:error",
"Error while %(action)s, error %(error)s\ntraceback %(traceback)s",
modelObject=modelXbrl,
action=currentAction,
error=ex,
traceback=traceback.format_tb(sys.exc_info()[2]))
if priorCWD:
os.chdir(
priorCWD
) # restore prior current working directory startingErrorCount = len(modelXbrl.errors)
if startingErrorCount < len(modelXbrl.errors):
# had errors, don't allow ModelDocument.load to continue
return OIMException("arelleOIMloader:unableToLoad",
"Unable to load due to reported errors")
return getattr(modelXbrl, "modelDocument",
None) # none if returning from exception