def loadXbrlFromDB(self, loadDBsaveToFile):
# load from database
modelXbrl = self.modelXbrl
# find instance in DB
instanceURI = os.path.basename(loadDBsaveToFile)
results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant"
" FROM dInstance WHERE FileName = '{}'"
.format(instanceURI))
instanceId = moduleId = None
for instanceId, moduleId, entScheme, entId, datePerEnd in results:
break
# find module in DB
results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId))
xbrlSchemaRef = None
for result in results:
xbrlSchemaRef = result[0]
break
if not instanceId or not xbrlSchemaRef:
raise XPDBException("sqlDB:MissingDTS",
_("The instance and module were not found for %(instanceURI)"),
instanceURI = instanceURI)
if modelXbrl.skipDTS:
# find prefixes and namespaces in DB
results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]")
dpmPrefixedNamespaces = dict((prefix, namespace)
for owner, prefix, namespace in results)
# create the instance document and resulting filing
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(modelXbrl,
Type.INSTANCE,
loadDBsaveToFile,
schemaRefs=[xbrlSchemaRef],
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement,
'xbrl-streamable-instance',
'version="1.0" contextBuffer="1"')
# add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue)
# filing indicator code IDs
# get filing indicators
results = self.execute("SELECT mToT.TemplateOrTableCode "
" FROM dFilingIndicator dFI, mTemplateOrTable mToT "
" WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID"
.format(instanceId))
filingIndicatorCodes = [code[0] for code in results]
if filingIndicatorCodes:
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
{}, [], [],
id='c')
filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators)
for filingIndicatorCode in filingIndicatorCodes:
modelXbrl.createFact(qnFindFilingIndicator,
parent=filingIndicatorsTuple,
attributes={"contextRef": "c"},
text=filingIndicatorCode)
# facts in this instance
factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards,"
" Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue "
"FROM dFact WHERE InstanceID = {} "
"ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL "
" THEN DataPointSignature"
" ELSE DataPointSignatureWithValuesForWildcards"
" END, instr(DataPointSignature,'|') + 1)"
.format(instanceId))
# results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal
# get typed dimension values
prefixedNamespaces = modelXbrl.prefixedNamespaces
prefixedNamespaces["iso4217"] = XbrlConst.iso4217
if modelXbrl.skipDTS:
prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed
cntxTbl = {} # index by d
unitTbl = {}
def typedDimElt(s):
# add xmlns into s for known qnames
tag, angleBrkt, rest = s[1:].partition('>')
text, angleBrkt, rest = rest.partition("<")
qn = qname(tag, prefixedNamespaces)
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
# contexts and facts
for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl:
metric, sep, dims = (dpSigTypedDims or dpSig).partition('|')
conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces)
concept = modelXbrl.qnameConcepts.get(conceptQn)
isNumeric = isBool = isDateTime = isQName = isText = False
if concept is not None:
if concept.isNumeric:
isNumeric = True
else:
baseXbrliType = concept.baseXbrliType
if baseXbrliType == "booleanItemType":
isBool = True
elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
isDateTime = True
elif baseXbrliType == "QNameItemType":
isQName = True
else:
c = conceptQn.localName[0]
if c == 'm':
isNumeric = True
elif c == 'd':
isDateTime = True
elif c == 'b':
isBool = True
elif c == 'e':
isQName = True
isText = not (isNumeric or isBool or isDateTime or isQName)
if isinstance(datePerEnd, _STR_BASE):
datePerEnd = datetimeValue(datePerEnd, addOneDay=True)
cntxKey = (dims, entId, datePerEnd)
if cntxKey in cntxTbl:
cntxId = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
cntxTbl[cntxKey] = cntxId
qnameDims = {}
if dims:
for dim in dims.split('|'):
dQn, sep, dVal = dim[:-1].partition('(')
dimQname = qname(dQn, prefixedNamespaces)
if dVal.startswith('<'):
mem = typedDimElt(dVal) # typed dim
else:
mem = qname(dVal, prefixedNamespaces) # explicit dim (even if treat-as-typed)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario")
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId)
if unit:
if unit in unitTbl:
unitId = unitTbl[unit]
else:
unitQn = qname(unit, prefixedNamespaces)
unitId = 'u{}'.format(unitQn.localName)
unitTbl[unit] = unitId
modelXbrl.createUnit([unitQn], [], id=unitId)
else:
unitId = None
attrs = {"contextRef": cntxId}
if unitId:
attrs["unitRef"] = unitId
if dec is not None:
if isinstance(dec, float): # must be an integer
dec = int(dec)
elif isinstance(dec, _STR_BASE) and '.' in dec:
dec = dec.partition('.')[0] # drop .0 from any SQLite string
attrs["decimals"] = str(dec) # somehow it is float from the database
if False: # fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
elif numVal is not None:
num = roundValue(numVal, None, dec) # round using reported decimals
if dec is None or dec == "INF": # show using decimals or reported format
dec = len(numVal.partition(".")[2])
else: # max decimals at 28
dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num))
elif dateVal is not None:
text = dateVal
elif boolVal is not None:
text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false'
else:
if isQName: # declare namespace
addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces))
text = textVal
modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
# add footnotes if any
# save to file
modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
return modelXbrl.modelDocument
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
from openpyxl import load_workbook
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
try:
currentAction = "initializing"
startedAt = time.time()
if os.path.isabs(oimFile):
# allow relative filenames to loading directory
priorCWD = os.getcwd()
os.chdir(os.path.dirname(oimFile))
else:
priorCWD = None
currentAction = "determining file type"
isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json")
isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
isCSVorXL = isCSV or isXL
instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"
if isJSON:
currentAction = "loading and parsing JSON OIM file"
with io.open(oimFile, 'rt', encoding='utf-8') as f:
oimObject = json.load(f)
missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject]
if missing:
raise OIMException("oime:missingJSONelements",
_("The required elements %(missing)s {} in JSON input"),
missing = ", ".join(missing))
currentAction = "identifying JSON objects"
dtsReferences = oimObject["dtsReferences"]
prefixes = oimObject["prefixes"]
facts = oimObject["facts"]
footnotes = oimOject["facts"] # shares this object
elif isCSV:
currentAction = "identifying CSV input tables"
if sys.version[0] >= '3':
csvOpenMode = 'w'
csvOpenNewline = ''
else:
csvOpenMode = 'wb' # for 2.7
csvOpenNewline = None
oimFileBase = None
if "-facts" in oimFile:
oimFileBase = oimFile.partition("-facts")[0]
else:
for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"):
if oimFile.endswith(suffix):
oimFileBase = oimFile[:-length(suffix)]
break
if oimFileBase is None:
raise OIMException("oime:missingCSVtables",
_("Unable to identify CSV tables file name pattern"))
if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or
not os.path.exists(oimFileBase + "-prefixes.csv")):
raise OIMException("oime:missingCSVtables",
_("Unable to identify CSV tables for dtsReferences or prefixes"))
instanceFileName = oimFileBase + ".xbrl"
currentAction = "loading CSV dtsReferences table"
dtsReferences = []
with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
dtsReferences.append(dict((header[j], col) for j, col in enumerate(row)))
currentAction = "loading CSV prefixes table"
prefixes = {}
with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = dict((col,i) for i,col in enumerate(row))
else:
prefixes[row[header["prefix"]]] = row[header["URI"]]
currentAction = "loading CSV defaults table"
defaults = {}
if os.path.exists(oimFileBase + "-defaults.csv"):
with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
fileCol = row.index("file")
else:
defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol)
currentAction = "loading CSV facts tables"
facts = []
_dir = os.path.dirname(oimFileBase)
for filename in os.listdir(_dir):
filepath = os.path.join(_dir, filename)
if "-facts" in filename and filepath.startswith(oimFileBase):
tableDefaults = defaults.get(filename, {})
with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col is not None:
if header[j].endswith("Value"):
if col: # ignore empty columns (= null CSV value)
fact["value"] = col
else:
fact[header[j]] = col
facts.append(fact)
footnotes = []
if os.path.exists(oimFileBase + "-footnotes.csv"):
with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col))
elif isXL:
oimWb = load_workbook(oimFile, read_only=True, data_only=True)
sheetNames = oimWb.get_sheet_names()
if (not any(sheetName == "prefixes" for sheetName in sheetNames) or
not any(sheetName == "dtsReferences" for sheetName in sheetNames) or
not any("facts" in sheetName for sheetName in sheetNames)):
if priorCWD: os.chdir(priorCWD)
return None
try:
dtsReferences = []
for i, row in oimWb["dtsReferences"]:
if i == 0:
header = [col.value for col in row]
else:
dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row)))
prefixes = {}
for i, row in oimWb["dtsReferences"]:
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row))
else:
prefixes[row[header["prefix"]].value] = row[header["URI"].value]
defaults = {}
for i, row in oimW.get("defaults", ()):
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row))
fileCol = header["file"]
else:
defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol)
facts = []
_dir = os.path.dirpart(oimFileBase)
for sheetName in sheetNames:
if sheetName == "facts" or "-facts" in sheetName:
for i, row in oimWb[sheetName]:
if i == 0:
header = [col.value for col in row]
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col.value is not None:
if header[j].endswith("Value"):
fact["value"] = str(col.value)
else:
fact[header[j]] = str(col.value)
facts.append(fact)
footnotes = []
for i, row in oimWb.get("footnotes", ()):
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row) if col.value)
else:
footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value))
except Exception as ex:
if priorCWD: os.chdir(priorCWD)
return None
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
# create the instance document
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(
modelXbrl,
Type.INSTANCE,
instanceFileName,
schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"],
isEntry=True,
initialComment="extracted from OIM {}".format(mappedUri),
documentEncoding="utf-8")
cntxTbl = {}
unitTbl = {}
for fact in facts:
conceptQn = qname(fact["oim:concept"], prefixes)
concept = modelXbrl.qnameConcepts.get(conceptQn)
entityAsQn = qname(fact["oim:entity"], prefixes)
if "oim:period" in fact:
periodStart = fact["oim:period"]["start"]
periodEnd = fact["oim:period"]["end"]
else:
periodStart = fact["oim:periodStart"]
periodEnd = fact["oim:periodEnd"]
cntxKey = ( # hashable context key
("periodType", concept.periodType),
("entity", entityAsQn),
("periodStart", periodStart),
("periodEnd", periodEnd)) + tuple(sorted(
(dimName, dimVal)
for dimName, dimVal in fact.items()
if ":" in dimName and not dimName.startswith("oim:")))
if cntxKey in cntxTbl:
_cntx = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
qnameDims = {}
for dimName, dimVal in fact.items():
if ":" in dimName and not dimName.startswith("oim:"):
dimQname = qname(dimName, prefixes)
dimConcept = modelXbrl.qnameConcepts.get(dimQname)
if ":" in dimVal and dimVal.partition[':'][0] in prefixes:
mem = qname(dimVal, prefixes) # explicit dim
elif dimConcept.isTypedDimension:
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
_cntx = modelXbrl.createContext(
entityAsQn.namespaceURI,
entityAsQn.localName,
concept.periodType,
None if concept.periodType == "instnat" else dateTime(periodStart, type=DATETIME),
dateTime(periodEnd, type=DATETIME),
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId)
cntxTbl[cntxKey] = _cntx
if "oim:unit" in fact:
unitKey = fact["oim:unit"]
if unitKey in unitTbl:
_unit = unitTbl[unitKey]
else:
_mul, _sep, _div = unitKey.partition('/')
if _mul.startswith('('):
_mul = _mul[1:-1]
mulQns = [qname(u, prefixes) for u in _mul.split('*') if u]
if _div.startswith('('):
_div = _div[1:-1]
divQns = [qname(u, prefixes) for u in _div.split('*') if u]
unitId = 'u-{:02}'.format(len(unitTbl) + 1)
for _measures in mulQns, divQns:
for _measure in _measures:
addQnameValue(modelXbrl.modelDocument, _measure)
_unit = modelXbrl.createUnit(mulQns, divQns, id=unitId)
unitTbl[unitKey] = _unit
else:
_unit = None
attrs = {"contextRef": _cntx.id}
if fact.get("value") is None:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact["value"]
if fact.get("id"):
attrs["id"] = fact["id"]
if concept.isNumeric:
if _unit is not None:
attrs["unitRef"] = _unit.id
if "accuracy" in fact:
attrs["decimals"] = fact["accuracy"]
# is value a QName?
if concept.baseXbrliType == "QName":
addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
footnoteLinks = {} # ELR elements
factLocs = {} # index by (linkrole, factId)
footnoteNbr = 0
locNbr = 0
for factOrFootnote in footnotes:
if "factId" in factOrFootnote:
factId = factOrFootnote["factId"]
factFootnotes = (factOrFootnote,) # CSV or XL
elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
factId = factOrFootnote["id"]
factFootnotes = factOrFootnote["footnotes"]
else:
factFootnotes = ()
for footnote in factFootnotes:
linkrole = footnote.get("group")
arcrole = footnote.get("footnoteType")
if not factId or not linkrole or not arcrole or not (
footnote.get("factRef") or footnote.get("footnote")):
# invalid footnote
continue
if linkrole not in footnoteLinks:
footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement,
XbrlConst.qnLinkFootnoteLink,
attributes={"{http://www.w3.org/1999/xlink}type": "extended",
"{http://www.w3.org/1999/xlink}role": linkrole})
footnoteLink = footnoteLinks[linkrole]
if (linkrole, factId) not in factLocs:
locNbr += 1
locLabel = "l_{:02}".format(locNbr)
factLocs[(linkrole, factId)] = locLabel
addChild(footnoteLink, XbrlConst.qnLinkLoc,
attributes={XLINKTYPE: "locator",
XLINKHREF: "#" + factId,
XLINKLABEL: locLabel})
locLabel = factLocs[(linkrole, factId)]
if footnote.get("footnote"):
footnoteNbr += 1
footnoteLabel = "f_{:02}".format(footnoteNbr)
attrs = {XLINKTYPE: "resource",
XLINKLABEL: footnoteLabel}
if footnote.get("language"):
attrs[XMLLANG] = footnote["language"]
# note, for HTML will need to build an element structure
addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"])
elif footnote.get("factRef"):
factRef = footnote.get("factRef")
if (linkrole, factRef) not in factLocs:
locNbr += 1
locLabel = "f_{:02}".format(footnoteNbr)
factLoc[(linkrole, factRef)] = locLabel
addChild(footnoteLink, XbrlConst.qnLinkLoc,
attributes={XLINKTYPE: "locator",
XLINKHREF: "#" + factRef,
XLINKLABEL: locLabel})
footnoteLabel = factLoc[(linkrole, factId)]
footnoteArc = addChild(footnoteLink,
XbrlConst.qnLinkFootnoteArc,
attributes={XLINKTYPE: "arc",
XLINKARCROLE: arcrole,
XLINKFROM: locLabel,
XLINKTO: footnoteLabel})
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
except Exception as ex:
if ex is OIMException:
modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
else:
modelXbrl.error("Error while %(action)s, error %(error)s",
modelObject=modelXbrl, action=currentAction, error=ex)
if priorCWD:
os.chdir(priorCWD) # restore prior current working directory
return modelXbrl.modelDocument
def createModelFact(fact, parentModelFact, topTupleFact):
aspects = fact.get("aspects", EMPTYDICT)
if oimConcept not in aspects:
modelXbrl.error("{}:conceptQName".format(errPrefix),
_("The concept QName could not be determined"),
modelObject=modelXbrl)
return
conceptQn = qname(aspects[oimConcept], prefixes)
concept = modelXbrl.qnameConcepts.get(conceptQn)
attrs = {}
if concept.isItem:
missingAspects = []
if oimEntity not in aspects:
missingAspects.append(oimEntity)
if oimPeriod not in aspects and (oimPeriodStart not in aspects or oimPeriodEnd not in aspects):
missingAspects.append(oimPeriod)
if missingAspects:
modelXbrl.error("{}:missingAspects".format(errPrefix),
_("The concept %(element)s is missing aspects %(missingAspects)s"),
modelObject=modelXbrl, element=conceptQn, missingAspects=", ".join(missingAspects))
return
entityAsQn = qname(aspects[oimEntity], prefixes) or qname("error",fact[oimEntity])
if oimPeriod in aspects:
periodStart = periodEnd = aspects[oimPeriod]
if oimPeriodStart in aspects and oimPeriodEnd in aspects:
periodStart = aspects[oimPeriodStart]
periodEnd = aspects[oimPeriodEnd]
cntxKey = ( # hashable context key
("periodType", concept.periodType),
("entity", entityAsQn),
("periodStart", periodStart),
("periodEnd", periodEnd)) + tuple(sorted(
(dimName, dimVal["value"] if isinstance(dimVal,dict) else dimVal)
for dimName, dimVal in aspects.items()
if ":" in dimName and not dimName.startswith(oimPrefix)))
if cntxKey in cntxTbl:
_cntx = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
qnameDims = {}
for dimName, dimVal in aspects.items():
if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal:
dimQname = qname(dimName, prefixes)
dimConcept = modelXbrl.qnameConcepts.get(dimQname)
if dimConcept is None:
modelXbrl.error("{}:taxonomyDefinedAspectQName".format(errPrefix),
_("The taxonomy defined aspect concept QName %(qname)s could not be determined"),
modelObject=modelXbrl, qname=dimQname)
continue
if isinstance(dimVal, dict):
dimVal = dimVal["value"]
else:
dimVal = str(dimVal) # may be int or boolean
if isinstance(dimVal,str) and ":" in dimVal and dimVal.partition(':')[0] in prefixes:
mem = qname(dimVal, prefixes) # explicit dim
elif dimConcept.isTypedDimension:
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
_cntx = modelXbrl.createContext(
entityAsQn.namespaceURI,
entityAsQn.localName,
concept.periodType,
None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME),
dateTime(periodEnd, type=DATETIME),
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId,
beforeSibling=topTupleFact)
cntxTbl[cntxKey] = _cntx
if oimUnit in aspects and concept.isNumeric:
unitKey = aspects[oimUnit]
if unitKey in unitTbl:
_unit = unitTbl[unitKey]
else:
_unit = None
# validate unit
unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey)
if not UnitPattern.match(unitKeySub):
modelXbrl.error("oime:unitStringRepresentation",
_("Unit string representation is lexically invalid, %(unit)s"),
modelObject=modelXbrl, unit=unitKey)
else:
_mul, _sep, _div = unitKey.partition('/')
if _mul.startswith('('):
_mul = _mul[1:-1]
_muls = [u for u in _mul.split('*') if u]
if _div.startswith('('):
_div = _div[1:-1]
_divs = [u for u in _div.split('*') if u]
if _muls != sorted(_muls) or _divs != sorted(_divs):
modelXbrl.error("oime:unitStringRepresentation",
_("Unit string representation measures are not in alphabetical order, %(unit)s"),
modelObject=modelXbrl, unit=unitKey)
try:
mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"),
unit=u))
for u in _muls]
divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"),
unit=u))
for u in _divs]
unitId = 'u-{:02}'.format(len(unitTbl) + 1)
for _measures in mulQns, divQns:
for _measure in _measures:
addQnameValue(modelXbrl.modelDocument, _measure)
_unit = modelXbrl.createUnit(mulQns, divQns, id=unitId, beforeSibling=topTupleFact)
except OIMException as ex:
modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
unitTbl[unitKey] = _unit
else:
_unit = None
attrs["contextRef"] = _cntx.id
if fact.get("value") is None:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact["value"]
if concept.isNumeric:
if _unit is None:
return # skip creating fact because unit was invalid
attrs["unitRef"] = _unit.id
if "accuracy" in attrs or attrs.get(XbrlConst.qnXsiNil, "false") != "true":
attrs["decimals"] = fact.get("accuracy", "INF")
else:
text = None #tuple
id = fact.get("id")
if id is not None:
attrs["id"] = fact["id"]
# is value a QName?
if concept.baseXbrliType == "QName":
addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text, parent=parentModelFact, validate=False)
if id is not None and id in parentedFacts:
# create child facts
for i in sorted(parentedFacts[id],
key=lambda j: facts[j].get("aspects", EMPTYDICT).get(oimTupleOrder,0)):
createModelFact(facts[i], f, f if topTupleFact is None else topTupleFact)
# validate after creating tuple contents
xmlValidate(modelXbrl, f)
def loadXbrlFromDB(self, loadDBsaveToFile):
# load from database
modelXbrl = self.modelXbrl
# find instance in DB
instanceURI = os.path.basename(loadDBsaveToFile)
results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant"
" FROM dInstance WHERE FileName = '{}'"
.format(instanceURI))
instanceId = moduleId = None
for instanceId, moduleId, entScheme, entId, datePerEnd in results:
break
# find module in DB
results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId))
xbrlSchemaRef = None
for result in results:
xbrlSchemaRef = result[0]
break
if not instanceId or not xbrlSchemaRef:
raise XPDBException("sqlDB:MissingDTS",
_("The instance and module were not found for %(instanceURI)"),
instanceURI = instanceURI)
if modelXbrl.skipDTS:
# find prefixes and namespaces in DB
results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]")
dpmPrefixedNamespaces = dict((prefix, namespace)
for owner, prefix, namespace in results)
# create the instance document and resulting filing
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(modelXbrl,
Type.INSTANCE,
loadDBsaveToFile,
schemaRefs=[xbrlSchemaRef],
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement,
'xbrl-streamable-instance',
'version="1.0" contextBuffer="1"')
# add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue)
# filing indicator code IDs
# get filing indicators
results = self.execute("SELECT mToT.TemplateOrTableCode "
" FROM dFilingIndicator dFI, mTemplateOrTable mToT "
" WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID"
.format(instanceId))
filingIndicatorCodes = [code[0] for code in results]
if filingIndicatorCodes:
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
{}, [], [],
id='c')
filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators)
for filingIndicatorCode in filingIndicatorCodes:
modelXbrl.createFact(qnFindFilingIndicator,
parent=filingIndicatorsTuple,
attributes={"contextRef": "c"},
text=filingIndicatorCode)
# facts in this instance
factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards,"
" Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue "
"FROM dFact WHERE InstanceID = {} "
"ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL "
" THEN DataPointSignature"
" ELSE DataPointSignatureWithValuesForWildcards"
" END, instr(DataPointSignature,'|') + 1)"
.format(instanceId))
# results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal
# get typed dimension values
prefixedNamespaces = modelXbrl.prefixedNamespaces
prefixedNamespaces["iso4217"] = XbrlConst.iso4217
if modelXbrl.skipDTS:
prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed
cntxTbl = {} # index by d
unitTbl = {}
def typedDimElt(s):
# add xmlns into s for known qnames
tag, angleBrkt, rest = s[1:].partition('>')
text, angleBrkt, rest = rest.partition("<")
qn = qname(tag, prefixedNamespaces)
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
# contexts and facts
for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl:
metric, sep, dims = (dpSigTypedDims or dpSig).partition('|')
conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces)
concept = modelXbrl.qnameConcepts.get(conceptQn)
isNumeric = isBool = isDateTime = isQName = isText = False
if concept is not None:
if concept.isNumeric:
isNumeric = True
else:
baseXbrliType = concept.baseXbrliType
if baseXbrliType == "booleanItemType":
isBool = True
elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
isDateTime = True
elif baseXbrliType == "QNameItemType":
isQName = True
else:
c = conceptQn.localName[0]
if c == 'm':
isNumeric = True
elif c == 'd':
isDateTime = True
elif c == 'b':
isBool = True
elif c == 'e':
isQName = True
isText = not (isNumeric or isBool or isDateTime or isQName)
if isinstance(datePerEnd, _STR_BASE):
datePerEnd = datetimeValue(datePerEnd, addOneDay=True)
cntxKey = (dims, entId, datePerEnd)
if cntxKey in cntxTbl:
cntxId = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
cntxTbl[cntxKey] = cntxId
qnameDims = {}
if dims:
for dim in dims.split('|'):
dQn, sep, dVal = dim[:-1].partition('(')
dimQname = qname(dQn, prefixedNamespaces)
if dVal.startswith('<'): # typed dim
mem = typedDimElt(dVal)
else:
mem = qname(dVal, prefixedNamespaces)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario")
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId)
if unit:
if unit in unitTbl:
unitId = unitTbl[unit]
else:
unitQn = qname(unit, prefixedNamespaces)
unitId = 'u{}'.format(unitQn.localName)
unitTbl[unit] = unitId
modelXbrl.createUnit([unitQn], [], id=unitId)
else:
unitId = None
attrs = {"contextRef": cntxId}
if unitId:
attrs["unitRef"] = unitId
if dec is not None:
if isinstance(dec, float): # must be an integer
dec = int(dec)
elif isinstance(dec, _STR_BASE) and '.' in dec:
dec = dec.partition('.')[0] # drop .0 from any SQLite string
attrs["decimals"] = str(dec) # somehow it is float from the database
if False: # fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
elif numVal is not None:
num = roundValue(numVal, None, dec) # round using reported decimals
if dec is None or dec == "INF": # show using decimals or reported format
dec = len(numVal.partition(".")[2])
else: # max decimals at 28
dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num))
elif dateVal is not None:
text = dateVal
elif boolVal is not None:
text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false'
else:
if isQName: # declare namespace
addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces))
text = textVal
modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
# add footnotes if any
# save to file
modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
return modelXbrl.modelDocument
def yAxisByCol(self, leftCol, row, yParentStructuralNode, childrenFirst, renderNow, atTop):
if yParentStructuralNode is not None:
nestedBottomRow = row
for yStructuralNode in yParentStructuralNode.childStructuralNodes:
nestRow, nextRow = self.yAxisByCol(leftCol + 1, row, yStructuralNode, # nested items before totals
childrenFirst, childrenFirst, False)
isAbstract = (yStructuralNode.isAbstract or
(yStructuralNode.childStructuralNodes and
not isinstance(yStructuralNode.definitionNode, (ModelClosedDefinitionNode, ModelEuAxisCoord))))
isNonAbstract = not isAbstract
label = yStructuralNode.header(lang=self.lang,
returnGenLabel=isinstance(yStructuralNode.definitionNode, (ModelClosedDefinitionNode, ModelEuAxisCoord)))
topRow = row
if childrenFirst and isNonAbstract:
row = nextRow
#print ( "thisCol {0} leftCol {1} rightCol {2} topRow{3} renderNow {4} label {5}".format(thisCol, leftCol, rightCol, topRow, renderNow, label))
if renderNow:
rowspan= nestRow - row + 1
cellElt = etree.Element(tableModelQName("cell"),
attrib={"span": str(rowspan)} if rowspan > 1 else None)
elt = etree.SubElement(cellElt, tableModelQName("label"))
elt.text = label
self.rowHdrElts[leftCol - 1].append(cellElt)
self.structuralNodeModelElements.append((yStructuralNode, cellElt))
for aspect in sorted(yStructuralNode.aspectsCovered(), key=lambda a: aspectStr(a)):
if yStructuralNode.hasAspect(aspect) and aspect != Aspect.DIMENSIONS:
aspectValue = yStructuralNode.aspectValue(aspect)
if aspectValue is None: aspectValue = "(bound dynamically)"
elt = etree.SubElement(cellElt, tableModelQName("constraint"))
etree.SubElement(elt, tableModelQName("aspect")
).text = aspectStr(aspect)
etree.SubElement(elt, tableModelQName("value")
).text = addQnameValue(self.xmlDoc, aspectValue)
for rollUpCol in range(leftCol, self.rowHdrCols - 1):
rollUpElt = etree.Element(tableModelQName("cell"),
attrib={"rollup":"true"})
self.rowHdrElts[rollUpCol].append(rollUpElt)
if isNonAbstract:
cellElt = etree.Element(tableModelQName("cell"),
attrib={"span": str(rowspan)} if rowspan > 1 else None)
for i, role in enumerate(self.rowHdrNonStdRoles):
labelElt = etree.SubElement(cellElt, tableModelQName("label"),
attrib={"role":role,
"lang":self.lang})
labelElt.text = yStructuralNode.header(role=role, lang=self.lang)
self.rowHdrElts[self.rowHdrCols - 1 + i].append(cellElt)
for aspect in sorted(yStructuralNode.aspectsCovered(), key=lambda a: aspectStr(a)):
if yStructuralNode.hasAspect(aspect) and aspect != Aspect.DIMENSIONS:
aspectValue = yStructuralNode.aspectValue(aspect)
if aspectValue is None: aspectValue = "(bound dynamically)"
elt = etree.SubElement(cellElt, tableModelQName("constraint"))
etree.SubElement(elt, tableModelQName("aspect")
).text = aspectStr(aspect)
etree.SubElement(elt, tableModelQName("value")
).text = addQnameValue(self.xmlDoc, aspectValue)
'''
if self.rowHdrDocCol:
labelElt = etree.SubElement(cellElt, tableModelQName("label"),
attrib={"span": str(rowspan)} if rowspan > 1 else None)
elt.text = yStructuralNode.header(role="http://www.xbrl.org/2008/role/documentation",
lang=self.lang)
self.rowHdrElts[self.rowHdrCols - 1].append(elt)
if self.rowHdrCodeCol:
elt = etree.Element(tableModelQName("label"),
attrib={"span": str(rowspan)} if rowspan > 1 else None)
elt.text = yStructuralNode.header(role="http://www.eurofiling.info/role/2010/coordinate-code",
lang=self.lang)
self.rowHdrElts[self.rowHdrCols - 1 + self.rowHdrDocCol].append(elt)
'''
if isNonAbstract:
row += 1
elif childrenFirst:
row = nextRow
if nestRow > nestedBottomRow:
nestedBottomRow = nestRow + (isNonAbstract and not childrenFirst)
if row > nestedBottomRow:
nestedBottomRow = row
#if renderNow and not childrenFirst:
# dummy, row = self.yAxis(leftCol + 1, row, yStructuralNode, childrenFirst, True, False) # render on this pass
if not childrenFirst:
dummy, row = self.yAxisByCol(leftCol + 1, row, yStructuralNode, childrenFirst, renderNow, False) # render on this pass
return (nestedBottomRow, row)
def xAxis(self, leftCol, topRow, rowBelow, xParentStructuralNode, xStructuralNodes, childrenFirst, renderNow, atTop):
if xParentStructuralNode is not None:
parentRow = rowBelow
noDescendants = True
rightCol = leftCol
widthToSpanParent = 0
sideBorder = not xStructuralNodes
for xStructuralNode in xParentStructuralNode.childStructuralNodes:
noDescendants = False
rightCol, row, width, leafNode = self.xAxis(leftCol, topRow + 1, rowBelow, xStructuralNode, xStructuralNodes, # nested items before totals
childrenFirst, childrenFirst, False)
if row - 1 < parentRow:
parentRow = row - 1
#if not leafNode:
# rightCol -= 1
nonAbstract = not xStructuralNode.isAbstract
if nonAbstract:
width += 100 # width for this label
widthToSpanParent += width
label = xStructuralNode.header(lang=self.lang,
returnGenLabel=isinstance(xStructuralNode.definitionNode, (ModelClosedDefinitionNode, ModelEuAxisCoord)))
if childrenFirst:
thisCol = rightCol
else:
thisCol = leftCol
#print ( "thisCol {0} leftCol {1} rightCol {2} topRow{3} renderNow {4} label {5}".format(thisCol, leftCol, rightCol, topRow, renderNow, label))
if renderNow:
columnspan = rightCol - leftCol + (1 if nonAbstract else 0)
if self.type == HTML:
if rightCol == self.dataFirstCol + self.dataCols - 1:
edgeBorder = "border-right:.5pt solid windowtext;"
else:
edgeBorder = ""
attrib = {"class":"xAxisHdr",
"style":"text-align:center;max-width:{0}pt;{1}".format(width,edgeBorder)}
if columnspan > 1:
attrib["colspan"] = str(columnspan)
if leafNode and row > topRow:
attrib["rowspan"] = str(row - topRow + 1)
elt = etree.Element("{http://www.w3.org/1999/xhtml}th",
attrib=attrib)
self.rowElts[topRow-1].insert(leftCol,elt)
elif self.type == XML:
cellElt = etree.Element(tableModelQName("cell"),
attrib={"span": str(columnspan)} if columnspan > 1 else None)
self.colHdrElts[topRow - self.colHdrTopRow].insert(leftCol,cellElt)
self.structuralNodeModelElements.append((xStructuralNode, cellElt))
elt = etree.SubElement(cellElt, tableModelQName("label"))
if nonAbstract or (leafNode and row > topRow):
for rollUpCol in range(topRow - self.colHdrTopRow + 1, self.colHdrRows - 1):
rollUpElt = etree.Element(tableModelQName("label"),
attrib={"rollup":"true"})
if childrenFirst:
self.colHdrElts[rollUpCol].append(rollUpElt)
else:
self.colHdrElts[rollUpCol].insert(leftCol,rollUpElt)
for i, role in enumerate(self.colHdrNonStdRoles):
etree.SubElement(cellElt, tableModelQName("label"),
attrib={"role": role,
"lang": self.lang}
).text = xStructuralNode.header(
role=role, lang=self.lang)
for aspect in sorted(xStructuralNode.aspectsCovered(), key=lambda a: aspectStr(a)):
if xStructuralNode.hasAspect(aspect) and aspect != Aspect.DIMENSIONS:
aspectValue = xStructuralNode.aspectValue(aspect)
if aspectValue is None: aspectValue = "(bound dynamically)"
aspElt = etree.SubElement(cellElt, tableModelQName("constraint"))
etree.SubElement(aspElt, tableModelQName("aspect")
).text = aspectStr(aspect)
etree.SubElement(aspElt, tableModelQName("value")
).text = addQnameValue(self.xmlDoc, aspectValue)
elt.text = label or "\u00A0" #produces
if nonAbstract:
if columnspan > 1 and rowBelow > topRow: # add spanned left leg portion one row down
if self.type == HTML:
attrib= {"class":"xAxisSpanLeg",
"rowspan": str(rowBelow - row)}
if edgeBorder:
attrib["style"] = edgeBorder
elt = etree.Element("{http://www.w3.org/1999/xhtml}th",
attrib=attrib)
elt.text = "\u00A0"
if childrenFirst:
self.rowElts[topRow].append(elt)
else:
self.rowElts[topRow].insert(leftCol,elt)
if self.type == HTML:
for i, role in enumerate(self.colHdrNonStdRoles):
elt = etree.Element("{http://www.w3.org/1999/xhtml}th",
attrib={"class":"xAxisHdr",
"style":"text-align:center;max-width:100pt;{0}".format(edgeBorder)})
self.rowElts[self.dataFirstRow - 1 - len(self.colHdrNonStdRoles) + i].insert(thisCol,elt)
elt.text = xStructuralNode.header(role=role, lang=self.lang) or "\u00A0"
'''
if self.colHdrDocRow:
doc = xStructuralNode.header(role="http://www.xbrl.org/2008/role/documentation", lang=self.lang)
if self.type == HTML:
elt = etree.Element("{http://www.w3.org/1999/xhtml}th",
attrib={"class":"xAxisHdr",
"style":"text-align:center;max-width:100pt;{0}".format(edgeBorder)})
self.rowElts[self.dataFirstRow - 2 - self.rowHdrCodeCol].insert(thisCol,elt)
elif self.type == XML:
elt = etree.Element(tableModelQName("label"))
self.colHdrElts[self.colHdrRows - 1].insert(thisCol,elt)
elt.text = doc or "\u00A0"
if self.colHdrCodeRow:
code = xStructuralNode.header(role="http://www.eurofiling.info/role/2010/coordinate-code")
if self.type == HTML:
elt = etree.Element("{http://www.w3.org/1999/xhtml}th",
attrib={"class":"xAxisHdr",
"style":"text-align:center;max-width:100pt;{0}".format(edgeBorder)})
self.rowElts[self.dataFirstRow - 2].insert(thisCol,elt)
elif self.type == XML:
elt = etree.Element(tableModelQName("label"))
self.colHdrElts[self.colHdrRows - 1 + self.colHdrDocRow].insert(thisCol,elt)
elt.text = code or "\u00A0"
'''
xStructuralNodes.append(xStructuralNode)
if nonAbstract:
rightCol += 1
if renderNow and not childrenFirst:
self.xAxis(leftCol + (1 if nonAbstract else 0), topRow + 1, rowBelow, xStructuralNode, xStructuralNodes, childrenFirst, True, False) # render on this pass
leftCol = rightCol
return (rightCol, parentRow, widthToSpanParent, noDescendants)
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
from openpyxl import load_workbook
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
try:
currentAction = "initializing"
startingErrorCount = len(modelXbrl.errors)
startedAt = time.time()
if os.path.isabs(oimFile):
# allow relative filenames to loading directory
priorCWD = os.getcwd()
os.chdir(os.path.dirname(oimFile))
else:
priorCWD = None
currentAction = "determining file type"
isJSON = oimFile.endswith(".json") and not oimFile.endswith("-metadata.json")
isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
isCSVorXL = isCSV or isXL
instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"
if isJSON:
currentAction = "loading and parsing JSON OIM file"
def loadDict(keyValuePairs):
_dict = OrderedDict() # preserve fact order in resulting instance
for key, value in keyValuePairs:
if isinstance(value, dict):
if DUPJSONKEY in value:
for _errKey, _errValue, _otherValue in value[DUPJSONKEY]:
if key == "prefixes":
modelXbrl.error("oime:duplicatedPrefix",
_("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"),
modelObject=modelXbrl, prefix=_errKey, uri1=_errValue, uri2=_otherValue)
del value[DUPJSONKEY]
if key in _dict:
if DUPJSONKEY not in _dict:
_dict[DUPJSONKEY] = []
_dict[DUPJSONKEY].append((key, value, _dict[key]))
else:
_dict[key] = value
return _dict
with io.open(oimFile, 'rt', encoding='utf-8') as f:
oimObject = json.load(f, object_pairs_hook=loadDict)
missing = [t for t in ("dtsReferences", "prefixes", "facts") if t not in oimObject]
if missing:
raise OIMException("oime:missingJSONElements",
_("Required element(s) are missing from JSON input: %(missing)s"),
missing = ", ".join(missing))
currentAction = "identifying JSON objects"
dtsReferences = oimObject["dtsReferences"]
prefixesList = oimObject["prefixes"].items()
facts = oimObject["facts"]
footnotes = oimObject["facts"] # shares this object
elif isCSV:
currentAction = "identifying CSV input tables"
if sys.version[0] >= '3':
csvOpenMode = 'w'
csvOpenNewline = ''
else:
csvOpenMode = 'wb' # for 2.7
csvOpenNewline = None
oimFileBase = None
if "-facts" in oimFile:
oimFileBase = oimFile.partition("-facts")[0]
else:
for suffix in ("-dtsReferences.csv", "-defaults.csv", "-prefixes.csv", "-footnotes.csv", "-metadata.json"):
if oimFile.endswith(suffix):
oimFileBase = oimFile[:-len(suffix)]
break
if oimFileBase is None:
raise OIMException("oime:missingCSVTables",
_("Unable to identify CSV tables file name pattern"))
if (not os.path.exists(oimFileBase + "-dtsReferences.csv") or
not os.path.exists(oimFileBase + "-prefixes.csv")):
raise OIMException("oime:missingCSVTables",
_("Unable to identify CSV tables for dtsReferences or prefixes"))
instanceFileName = oimFileBase + ".xbrl"
currentAction = "loading CSV dtsReferences table"
dtsReferences = []
with io.open(oimFileBase + "-dtsReferences.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
dtsReferences.append(dict((header[j], col) for j, col in enumerate(row)))
currentAction = "loading CSV prefixes table"
prefixesList = []
with io.open(oimFileBase + "-prefixes.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = dict((col,i) for i,col in enumerate(row))
else:
prefixesList.append((row[header["prefix"]], row[header["URI"]]))
defaults = {}
if os.path.exists(oimFileBase + "-defaults.csv"):
currentAction = "loading CSV defaults table"
with io.open(oimFileBase + "-defaults.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
fileCol = row.index("file")
else:
defaults[row[fileCol]] = dict((header[j], col) for j, col in enumerate(row) if j != fileCol)
currentAction = "loading CSV facts tables"
facts = []
_dir = os.path.dirname(oimFileBase)
factsFileBasename = os.path.basename(oimFileBase) + "-facts"
for filename in os.listdir(_dir):
filepath = os.path.join(_dir, filename)
if filename.startswith(factsFileBasename):
currentAction = "loading CSV facts table {}".format(filename)
tableDefaults = defaults.get(filename, {})
with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col is not None:
if header[j]: # skip cols with no header
if header[j].endswith("Value"):
if col: # ignore empty columns (= null CSV value)
fact["value"] = col
else:
fact[header[j]] = col
facts.append(fact)
footnotes = []
if os.path.exists(oimFileBase + "-footnotes.csv"):
currentAction = "loading CSV footnotes table"
with io.open(oimFileBase + "-footnotes.csv", 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
footnotes.append(dict((header[j], col) for j, col in enumerate(row) if col))
elif isXL:
currentAction = "identifying workbook input worksheets"
oimWb = load_workbook(oimFile, read_only=True, data_only=True)
sheetNames = oimWb.get_sheet_names()
if (not any(sheetName == "prefixes" for sheetName in sheetNames) or
not any(sheetName == "dtsReferences" for sheetName in sheetNames) or
not any("facts" in sheetName for sheetName in sheetNames)):
raise OIMException("oime:missingWorkbookWorksheets",
_("Unable to identify worksheet tabs for dtsReferences, prefixes or facts"))
currentAction = "loading worksheet: dtsReferences"
dtsReferences = []
for i, row in enumerate(oimWb["dtsReferences"]):
if i == 0:
header = [col.value for col in row]
else:
dtsReferences.append(dict((header[j], col.value) for j, col in enumerate(row)))
currentAction = "loading worksheet: prefixes"
prefixesList = []
for i, row in enumerate(oimWb["prefixes"]):
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row))
else:
prefixesList.append((row[header["prefix"]].value, row[header["URI"]].value))
defaults = {}
if "defaults" in sheetNames:
currentAction = "loading worksheet: defaults"
for i, row in enumerate(oimWb["defaults"]):
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row))
fileCol = header["file"]
else:
defaults[row[fileCol].value] = dict((header[j], col.value) for j, col in enumerate(row) if j != fileCol)
facts = []
for sheetName in sheetNames:
if sheetName == "facts" or "-facts" in sheetName:
currentAction = "loading worksheet: {}".format(sheetName)
tableDefaults = defaults.get(sheetName, {})
for i, row in enumerate(oimWb[sheetName]):
if i == 0:
header = [col.value for col in row]
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col.value is not None:
if header[j]: # skip cols with no header
if header[j].endswith("Value"):
fact["value"] = str(col.value)
else:
fact[header[j]] = str(col.value)
facts.append(fact)
footnotes = []
if "footnotes" in sheetNames:
currentAction = "loading worksheet: footnotes"
for i, row in enumerate(oimWb["footnotes"]):
if i == 0:
header = dict((col.value,i) for i,col in enumerate(row) if col.value)
else:
footnotes.append(dict((header[j], col.value) for j, col in enumerate(row) if col.value))
currentAction = "identifying default dimensions"
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
currentAction = "validating OIM"
prefixes = {}
prefixedUris = {}
for _prefix, _uri in prefixesList:
if not _prefix:
modelXbrl.error("oime:emptyPrefix",
_("The empty string must not be used as a prefix, uri %(uri)s"),
modelObject=modelXbrl, uri=_uri)
elif not NCNamePattern.match(_prefix):
modelXbrl.error("oime:prefixPattern",
_("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"),
modelObject=modelXbrl, prefix=_prefix, uri=_uri)
elif _prefix in prefixes:
modelXbrl.error("oime:duplicatedPrefix",
_("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"),
modelObject=modelXbrl, prefix=_prefix, uri1=prefixes[_prefix], uri2=_uri)
elif _uri in prefixedUris:
modelXbrl.error("oime:duplicatedUri",
_("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"),
modelObject=modelXbrl, uri=_uri, prefix1=prefixedUris[_uri], prefix2=_prefix)
else:
prefixes[_prefix] = _uri
prefixedUris[_uri] = _prefix
oimPrefix = None
for _nsOim in nsOim:
if _nsOim in prefixedUris:
oimPrefix = prefixedUris[_nsOim]
if not oimPrefix:
raise OIMException("oime:noOimPrefix",
_("The oim namespace must have a declared prefix"))
oimConcept = "{}:concept".format(oimPrefix)
oimEntity = "{}:entity".format(oimPrefix)
oimPeriod = "{}:period".format(oimPrefix)
oimPeriodStart = "{}:periodStart".format(oimPrefix)
oimPeriodEnd = "{}:periodEnd".format(oimPrefix)
oimUnit = "{}:unit".format(oimPrefix)
oimPrefix = "{}:".format(oimPrefix)
# create the instance document
currentAction = "creating instance document"
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(
modelXbrl,
Type.INSTANCE,
instanceFileName,
schemaRefs=[dtsRef["href"] for dtsRef in dtsReferences if dtsRef["type"] == "schema"],
isEntry=True,
initialComment="extracted from OIM {}".format(mappedUri),
documentEncoding="utf-8")
cntxTbl = {}
unitTbl = {}
currentAction = "creating facts"
for fact in facts:
conceptQn = qname(fact[oimConcept], prefixes)
concept = modelXbrl.qnameConcepts.get(conceptQn)
entityAsQn = qname(fact[oimEntity], prefixes)
if oimPeriod in fact:
periodStart = fact[oimPeriod]["start"]
periodEnd = fact[oimPeriod]["end"]
else:
periodStart = fact[oimPeriodStart]
periodEnd = fact[oimPeriodEnd]
cntxKey = ( # hashable context key
("periodType", concept.periodType),
("entity", entityAsQn),
("periodStart", periodStart),
("periodEnd", periodEnd)) + tuple(sorted(
(dimName, dimVal)
for dimName, dimVal in fact.items()
if ":" in dimName and not dimName.startswith(oimPrefix)))
if cntxKey in cntxTbl:
_cntx = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
qnameDims = {}
for dimName, dimVal in fact.items():
if ":" in dimName and not dimName.startswith(oimPrefix) and dimVal:
dimQname = qname(dimName, prefixes)
dimConcept = modelXbrl.qnameConcepts.get(dimQname)
if ":" in dimVal and dimVal.partition(':')[0] in prefixes:
mem = qname(dimVal, prefixes) # explicit dim
elif dimConcept.isTypedDimension:
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
mem = addChild(modelXbrl.modelDocument, dimConcept.typedDomainElement.qname, text=dimVal, appendChild=False)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "segment")
_cntx = modelXbrl.createContext(
entityAsQn.namespaceURI,
entityAsQn.localName,
concept.periodType,
None if concept.periodType == "instant" else dateTime(periodStart, type=DATETIME),
dateTime(periodEnd, type=DATETIME),
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId)
cntxTbl[cntxKey] = _cntx
if oimUnit in fact:
unitKey = fact[oimUnit]
if unitKey in unitTbl:
_unit = unitTbl[unitKey]
else:
_unit = None
# validate unit
unitKeySub = PrefixedQName.sub(UnitPrefixedQNameSubstitutionChar, unitKey)
if not UnitPattern.match(unitKeySub):
modelXbrl.error("oime:unitStringRepresentation",
_("Unit string representation is lexically invalid, %(unit)s"),
modelObject=modelXbrl, unit=unitKey)
else:
_mul, _sep, _div = unitKey.partition('/')
if _mul.startswith('('):
_mul = _mul[1:-1]
_muls = [u for u in _mul.split('*') if u]
if _div.startswith('('):
_div = _div[1:-1]
_divs = [u for u in _div.split('*') if u]
if _muls != sorted(_muls) or _divs != sorted(_divs):
modelXbrl.error("oime:unitStringRepresentation",
_("Unit string representation measures are not in alphabetical order, %(unit)s"),
modelObject=modelXbrl, unit=unitKey)
try:
mulQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"),
unit=u))
for u in _muls]
divQns = [qname(u, prefixes, prefixException=OIMException("oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"),
unit=u))
for u in _divs]
unitId = 'u-{:02}'.format(len(unitTbl) + 1)
for _measures in mulQns, divQns:
for _measure in _measures:
addQnameValue(modelXbrl.modelDocument, _measure)
_unit = modelXbrl.createUnit(mulQns, divQns, id=unitId)
except OIMException as ex:
modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
unitTbl[unitKey] = _unit
else:
_unit = None
attrs = {"contextRef": _cntx.id}
if fact.get("value") is None:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact["value"]
if fact.get("id"):
attrs["id"] = fact["id"]
if concept.isNumeric:
if _unit is None:
continue # skip creating fact because unit was invalid
attrs["unitRef"] = _unit.id
if "accuracy" in fact:
attrs["decimals"] = fact["accuracy"]
# is value a QName?
if concept.baseXbrliType == "QName":
addQnameValue(modelXbrl.modelDocument, qname(text.strip(), prefixes))
f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
currentAction = "creating footnotes"
footnoteLinks = {} # ELR elements
factLocs = {} # index by (linkrole, factId)
footnoteNbr = 0
locNbr = 0
for factOrFootnote in footnotes:
if "factId" in factOrFootnote:
factId = factOrFootnote["factId"]
factFootnotes = (factOrFootnote,) # CSV or XL
elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
factId = factOrFootnote["id"]
factFootnotes = factOrFootnote["footnotes"]
else:
factFootnotes = ()
for footnote in factFootnotes:
linkrole = footnote.get("group")
arcrole = footnote.get("footnoteType")
if not factId or not linkrole or not arcrole or not (
footnote.get("factRef") or footnote.get("footnote")):
# invalid footnote
continue
if linkrole not in footnoteLinks:
footnoteLinks[linkrole] = addChild(modelXbrl.modelDocument.xmlRootElement,
XbrlConst.qnLinkFootnoteLink,
attributes={"{http://www.w3.org/1999/xlink}type": "extended",
"{http://www.w3.org/1999/xlink}role": linkrole})
footnoteLink = footnoteLinks[linkrole]
if (linkrole, factId) not in factLocs:
locNbr += 1
locLabel = "l_{:02}".format(locNbr)
factLocs[(linkrole, factId)] = locLabel
addChild(footnoteLink, XbrlConst.qnLinkLoc,
attributes={XLINKTYPE: "locator",
XLINKHREF: "#" + factId,
XLINKLABEL: locLabel})
locLabel = factLocs[(linkrole, factId)]
if footnote.get("footnote"):
footnoteNbr += 1
footnoteLabel = "f_{:02}".format(footnoteNbr)
attrs = {XLINKTYPE: "resource",
XLINKLABEL: footnoteLabel}
if footnote.get("language"):
attrs[XMLLANG] = footnote["language"]
# note, for HTML will need to build an element structure
addChild(footnoteLink, XbrlConst.qnLinkFootnote, attributes=attrs, text=footnote["footnote"])
elif footnote.get("factRef"):
factRef = footnote.get("factRef")
if (linkrole, factRef) not in factLocs:
locNbr += 1
locLabel = "f_{:02}".format(footnoteNbr)
factLoc[(linkrole, factRef)] = locLabel
addChild(footnoteLink, XbrlConst.qnLinkLoc,
attributes={XLINKTYPE: "locator",
XLINKHREF: "#" + factRef,
XLINKLABEL: locLabel})
footnoteLabel = factLoc[(linkrole, factId)]
footnoteArc = addChild(footnoteLink,
XbrlConst.qnLinkFootnoteArc,
attributes={XLINKTYPE: "arc",
XLINKARCROLE: arcrole,
XLINKFROM: locLabel,
XLINKTO: footnoteLabel})
currentAction = "done loading facts and footnotes"
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
except Exception as ex:
if isinstance(ex, OIMException):
modelXbrl.error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
else:
modelXbrl.error("arelleOIMloader:error",
"Error while %(action)s, error %(error)s\ntraceback %(traceback)s",
modelObject=modelXbrl, action=currentAction, error=ex,
traceback=traceback.format_tb(sys.exc_info()[2]))
if priorCWD:
os.chdir(priorCWD) # restore prior current working directory startingErrorCount = len(modelXbrl.errors)
if startingErrorCount < len(modelXbrl.errors):
# had errors, don't allow ModelDocument.load to continue
return OIMException("arelleOIMloader:unableToLoad", "Unable to load due to reported errors")
return getattr(modelXbrl, "modelDocument", None) # none if returning from exception
def loadFromOIM(cntlr, modelXbrl, oimFile, mappedUri):
from openpyxl import load_workbook
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
try:
currentAction = "initializing"
startingErrorCount = len(modelXbrl.errors)
startedAt = time.time()
if os.path.isabs(oimFile):
# allow relative filenames to loading directory
priorCWD = os.getcwd()
os.chdir(os.path.dirname(oimFile))
else:
priorCWD = None
currentAction = "determining file type"
isJSON = oimFile.endswith(
".json") and not oimFile.endswith("-metadata.json")
isCSV = oimFile.endswith(".csv") or oimFile.endswith("-metadata.json")
isXL = oimFile.endswith(".xlsx") or oimFile.endswith(".xls")
isCSVorXL = isCSV or isXL
instanceFileName = os.path.splitext(oimFile)[0] + ".xbrl"
if isJSON:
currentAction = "loading and parsing JSON OIM file"
def loadDict(keyValuePairs):
_dict = OrderedDict(
) # preserve fact order in resulting instance
for key, value in keyValuePairs:
if isinstance(value, dict):
if DUPJSONKEY in value:
for _errKey, _errValue, _otherValue in value[
DUPJSONKEY]:
if key == "prefixes":
modelXbrl.error(
"oime:duplicatedPrefix",
_("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"
),
modelObject=modelXbrl,
prefix=_errKey,
uri1=_errValue,
uri2=_otherValue)
del value[DUPJSONKEY]
if key in _dict:
if DUPJSONKEY not in _dict:
_dict[DUPJSONKEY] = []
_dict[DUPJSONKEY].append((key, value, _dict[key]))
else:
_dict[key] = value
return _dict
with io.open(oimFile, 'rt', encoding='utf-8') as f:
oimObject = json.load(f, object_pairs_hook=loadDict)
missing = [
t for t in ("dtsReferences", "prefixes", "facts")
if t not in oimObject
]
if missing:
raise OIMException(
"oime:missingJSONElements",
_("Required element(s) are missing from JSON input: %(missing)s"
),
missing=", ".join(missing))
currentAction = "identifying JSON objects"
dtsReferences = oimObject["dtsReferences"]
prefixesList = oimObject["prefixes"].items()
facts = oimObject["facts"]
footnotes = oimObject["facts"] # shares this object
elif isCSV:
currentAction = "identifying CSV input tables"
if sys.version[0] >= '3':
csvOpenMode = 'w'
csvOpenNewline = ''
else:
csvOpenMode = 'wb' # for 2.7
csvOpenNewline = None
oimFileBase = None
if "-facts" in oimFile:
oimFileBase = oimFile.partition("-facts")[0]
else:
for suffix in ("-dtsReferences.csv", "-defaults.csv",
"-prefixes.csv", "-footnotes.csv",
"-metadata.json"):
if oimFile.endswith(suffix):
oimFileBase = oimFile[:-len(suffix)]
break
if oimFileBase is None:
raise OIMException(
"oime:missingCSVTables",
_("Unable to identify CSV tables file name pattern"))
if (not os.path.exists(oimFileBase + "-dtsReferences.csv")
or not os.path.exists(oimFileBase + "-prefixes.csv")):
raise OIMException(
"oime:missingCSVTables",
_("Unable to identify CSV tables for dtsReferences or prefixes"
))
instanceFileName = oimFileBase + ".xbrl"
currentAction = "loading CSV dtsReferences table"
dtsReferences = []
with io.open(oimFileBase + "-dtsReferences.csv",
'rt',
encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
dtsReferences.append(
dict(
(header[j], col) for j, col in enumerate(row)))
currentAction = "loading CSV prefixes table"
prefixesList = []
with io.open(oimFileBase + "-prefixes.csv",
'rt',
encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = dict((col, i) for i, col in enumerate(row))
else:
prefixesList.append(
(row[header["prefix"]], row[header["URI"]]))
defaults = {}
if os.path.exists(oimFileBase + "-defaults.csv"):
currentAction = "loading CSV defaults table"
with io.open(oimFileBase + "-defaults.csv",
'rt',
encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
fileCol = row.index("file")
else:
defaults[row[fileCol]] = dict(
(header[j], col) for j, col in enumerate(row)
if j != fileCol)
currentAction = "loading CSV facts tables"
facts = []
_dir = os.path.dirname(oimFileBase)
factsFileBasename = os.path.basename(oimFileBase) + "-facts"
for filename in os.listdir(_dir):
filepath = os.path.join(_dir, filename)
if filename.startswith(factsFileBasename):
currentAction = "loading CSV facts table {}".format(
filename)
tableDefaults = defaults.get(filename, {})
with io.open(filepath, 'rt', encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col is not None:
if header[
j]: # skip cols with no header
if header[j].endswith("Value"):
if col: # ignore empty columns (= null CSV value)
fact["value"] = col
else:
fact[header[j]] = col
facts.append(fact)
footnotes = []
if os.path.exists(oimFileBase + "-footnotes.csv"):
currentAction = "loading CSV footnotes table"
with io.open(oimFileBase + "-footnotes.csv",
'rt',
encoding='utf-8-sig') as f:
csvReader = csv.reader(f)
for i, row in enumerate(csvReader):
if i == 0:
header = row
else:
footnotes.append(
dict((header[j], col)
for j, col in enumerate(row) if col))
elif isXL:
currentAction = "identifying workbook input worksheets"
oimWb = load_workbook(oimFile, read_only=True, data_only=True)
sheetNames = oimWb.get_sheet_names()
if (not any(sheetName == "prefixes" for sheetName in sheetNames)
or not any(sheetName == "dtsReferences"
for sheetName in sheetNames)
or not any("facts" in sheetName
for sheetName in sheetNames)):
raise OIMException(
"oime:missingWorkbookWorksheets",
_("Unable to identify worksheet tabs for dtsReferences, prefixes or facts"
))
currentAction = "loading worksheet: dtsReferences"
dtsReferences = []
for i, row in enumerate(oimWb["dtsReferences"]):
if i == 0:
header = [col.value for col in row]
else:
dtsReferences.append(
dict((header[j], col.value)
for j, col in enumerate(row)))
currentAction = "loading worksheet: prefixes"
prefixesList = []
for i, row in enumerate(oimWb["prefixes"]):
if i == 0:
header = dict((col.value, i) for i, col in enumerate(row))
else:
prefixesList.append((row[header["prefix"]].value,
row[header["URI"]].value))
defaults = {}
if "defaults" in sheetNames:
currentAction = "loading worksheet: defaults"
for i, row in enumerate(oimWb["defaults"]):
if i == 0:
header = dict(
(col.value, i) for i, col in enumerate(row))
fileCol = header["file"]
else:
defaults[row[fileCol].value] = dict(
(header[j], col.value) for j, col in enumerate(row)
if j != fileCol)
facts = []
for sheetName in sheetNames:
if sheetName == "facts" or "-facts" in sheetName:
currentAction = "loading worksheet: {}".format(sheetName)
tableDefaults = defaults.get(sheetName, {})
for i, row in enumerate(oimWb[sheetName]):
if i == 0:
header = [col.value for col in row]
else:
fact = {}
fact.update(tableDefaults)
for j, col in enumerate(row):
if col.value is not None:
if header[j]: # skip cols with no header
if header[j].endswith("Value"):
fact["value"] = str(col.value)
else:
fact[header[j]] = str(col.value)
facts.append(fact)
footnotes = []
if "footnotes" in sheetNames:
currentAction = "loading worksheet: footnotes"
for i, row in enumerate(oimWb["footnotes"]):
if i == 0:
header = dict((col.value, i)
for i, col in enumerate(row)
if col.value)
else:
footnotes.append(
dict((header[j], col.value)
for j, col in enumerate(row) if col.value))
currentAction = "identifying default dimensions"
ValidateXbrlDimensions.loadDimensionDefaults(
modelXbrl) # needs dimension defaults
currentAction = "validating OIM"
prefixes = {}
prefixedUris = {}
for _prefix, _uri in prefixesList:
if not _prefix:
modelXbrl.error(
"oime:emptyPrefix",
_("The empty string must not be used as a prefix, uri %(uri)s"
),
modelObject=modelXbrl,
uri=_uri)
elif not NCNamePattern.match(_prefix):
modelXbrl.error(
"oime:prefixPattern",
_("The prefix %(prefix)s must match the NCName lexical pattern, uri %(uri)s"
),
modelObject=modelXbrl,
prefix=_prefix,
uri=_uri)
elif _prefix in prefixes:
modelXbrl.error(
"oime:duplicatedPrefix",
_("The prefix %(prefix)s is used on uri %(uri1)s and uri %(uri2)s"
),
modelObject=modelXbrl,
prefix=_prefix,
uri1=prefixes[_prefix],
uri2=_uri)
elif _uri in prefixedUris:
modelXbrl.error(
"oime:duplicatedUri",
_("The uri %(uri)s is used on prefix %(prefix1)s and prefix %(prefix2)s"
),
modelObject=modelXbrl,
uri=_uri,
prefix1=prefixedUris[_uri],
prefix2=_prefix)
else:
prefixes[_prefix] = _uri
prefixedUris[_uri] = _prefix
oimPrefix = None
for _nsOim in nsOim:
if _nsOim in prefixedUris:
oimPrefix = prefixedUris[_nsOim]
if not oimPrefix:
raise OIMException(
"oime:noOimPrefix",
_("The oim namespace must have a declared prefix"))
oimConcept = "{}:concept".format(oimPrefix)
oimEntity = "{}:entity".format(oimPrefix)
oimPeriod = "{}:period".format(oimPrefix)
oimPeriodStart = "{}:periodStart".format(oimPrefix)
oimPeriodEnd = "{}:periodEnd".format(oimPrefix)
oimUnit = "{}:unit".format(oimPrefix)
oimPrefix = "{}:".format(oimPrefix)
# create the instance document
currentAction = "creating instance document"
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(
modelXbrl,
Type.INSTANCE,
instanceFileName,
schemaRefs=[
dtsRef["href"] for dtsRef in dtsReferences
if dtsRef["type"] == "schema"
],
isEntry=True,
initialComment="extracted from OIM {}".format(mappedUri),
documentEncoding="utf-8")
cntxTbl = {}
unitTbl = {}
currentAction = "creating facts"
for fact in facts:
conceptQn = qname(fact[oimConcept], prefixes)
concept = modelXbrl.qnameConcepts.get(conceptQn)
entityAsQn = qname(fact[oimEntity], prefixes)
if oimPeriod in fact:
periodStart = fact[oimPeriod]["start"]
periodEnd = fact[oimPeriod]["end"]
else:
periodStart = fact[oimPeriodStart]
periodEnd = fact[oimPeriodEnd]
cntxKey = ( # hashable context key
("periodType", concept.periodType), ("entity", entityAsQn),
("periodStart", periodStart),
("periodEnd", periodEnd)) + tuple(
sorted(
(dimName, dimVal)
for dimName, dimVal in fact.items() if ":" in dimName
and not dimName.startswith(oimPrefix)))
if cntxKey in cntxTbl:
_cntx = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
qnameDims = {}
for dimName, dimVal in fact.items():
if ":" in dimName and not dimName.startswith(
oimPrefix) and dimVal:
dimQname = qname(dimName, prefixes)
dimConcept = modelXbrl.qnameConcepts.get(dimQname)
if ":" in dimVal and dimVal.partition(
':')[0] in prefixes:
mem = qname(dimVal, prefixes) # explicit dim
elif dimConcept.isTypedDimension:
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
mem = addChild(modelXbrl.modelDocument,
dimConcept.typedDomainElement.qname,
text=dimVal,
appendChild=False)
qnameDims[dimQname] = DimValuePrototype(
modelXbrl, None, dimQname, mem, "segment")
_cntx = modelXbrl.createContext(
entityAsQn.namespaceURI,
entityAsQn.localName,
concept.periodType,
None if concept.periodType == "instant" else dateTime(
periodStart, type=DATETIME),
dateTime(periodEnd, type=DATETIME),
None, # no dimensional validity checking (like formula does)
qnameDims,
[],
[],
id=cntxId)
cntxTbl[cntxKey] = _cntx
if oimUnit in fact:
unitKey = fact[oimUnit]
if unitKey in unitTbl:
_unit = unitTbl[unitKey]
else:
_unit = None
# validate unit
unitKeySub = PrefixedQName.sub(
UnitPrefixedQNameSubstitutionChar, unitKey)
if not UnitPattern.match(unitKeySub):
modelXbrl.error(
"oime:unitStringRepresentation",
_("Unit string representation is lexically invalid, %(unit)s"
),
modelObject=modelXbrl,
unit=unitKey)
else:
_mul, _sep, _div = unitKey.partition('/')
if _mul.startswith('('):
_mul = _mul[1:-1]
_muls = [u for u in _mul.split('*') if u]
if _div.startswith('('):
_div = _div[1:-1]
_divs = [u for u in _div.split('*') if u]
if _muls != sorted(_muls) or _divs != sorted(_divs):
modelXbrl.error(
"oime:unitStringRepresentation",
_("Unit string representation measures are not in alphabetical order, %(unit)s"
),
modelObject=modelXbrl,
unit=unitKey)
try:
mulQns = [
qname(
u,
prefixes,
prefixException=OIMException(
"oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"
),
unit=u)) for u in _muls
]
divQns = [
qname(
u,
prefixes,
prefixException=OIMException(
"oime:unitPrefix",
_("Unit prefix is not declared: %(unit)s"
),
unit=u)) for u in _divs
]
unitId = 'u-{:02}'.format(len(unitTbl) + 1)
for _measures in mulQns, divQns:
for _measure in _measures:
addQnameValue(modelXbrl.modelDocument,
_measure)
_unit = modelXbrl.createUnit(mulQns,
divQns,
id=unitId)
except OIMException as ex:
modelXbrl.error(ex.code,
ex.message,
modelObject=modelXbrl,
**ex.msgArgs)
unitTbl[unitKey] = _unit
else:
_unit = None
attrs = {"contextRef": _cntx.id}
if fact.get("value") is None:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
else:
text = fact["value"]
if fact.get("id"):
attrs["id"] = fact["id"]
if concept.isNumeric:
if _unit is None:
continue # skip creating fact because unit was invalid
attrs["unitRef"] = _unit.id
if "accuracy" in fact:
attrs["decimals"] = fact["accuracy"]
# is value a QName?
if concept.baseXbrliType == "QName":
addQnameValue(modelXbrl.modelDocument,
qname(text.strip(), prefixes))
f = modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
currentAction = "creating footnotes"
footnoteLinks = {} # ELR elements
factLocs = {} # index by (linkrole, factId)
footnoteNbr = 0
locNbr = 0
for factOrFootnote in footnotes:
if "factId" in factOrFootnote:
factId = factOrFootnote["factId"]
factFootnotes = (factOrFootnote, ) # CSV or XL
elif "id" in factOrFootnote and "footnotes" in factOrFootnote:
factId = factOrFootnote["id"]
factFootnotes = factOrFootnote["footnotes"]
else:
factFootnotes = ()
for footnote in factFootnotes:
linkrole = footnote.get("group")
arcrole = footnote.get("footnoteType")
if not factId or not linkrole or not arcrole or not (
footnote.get("factRef") or footnote.get("footnote")):
# invalid footnote
continue
if linkrole not in footnoteLinks:
footnoteLinks[linkrole] = addChild(
modelXbrl.modelDocument.xmlRootElement,
XbrlConst.qnLinkFootnoteLink,
attributes={
"{http://www.w3.org/1999/xlink}type": "extended",
"{http://www.w3.org/1999/xlink}role": linkrole
})
footnoteLink = footnoteLinks[linkrole]
if (linkrole, factId) not in factLocs:
locNbr += 1
locLabel = "l_{:02}".format(locNbr)
factLocs[(linkrole, factId)] = locLabel
addChild(footnoteLink,
XbrlConst.qnLinkLoc,
attributes={
XLINKTYPE: "locator",
XLINKHREF: "#" + factId,
XLINKLABEL: locLabel
})
locLabel = factLocs[(linkrole, factId)]
if footnote.get("footnote"):
footnoteNbr += 1
footnoteLabel = "f_{:02}".format(footnoteNbr)
attrs = {XLINKTYPE: "resource", XLINKLABEL: footnoteLabel}
if footnote.get("language"):
attrs[XMLLANG] = footnote["language"]
# note, for HTML will need to build an element structure
addChild(footnoteLink,
XbrlConst.qnLinkFootnote,
attributes=attrs,
text=footnote["footnote"])
elif footnote.get("factRef"):
factRef = footnote.get("factRef")
if (linkrole, factRef) not in factLocs:
locNbr += 1
locLabel = "f_{:02}".format(footnoteNbr)
factLoc[(linkrole, factRef)] = locLabel
addChild(footnoteLink,
XbrlConst.qnLinkLoc,
attributes={
XLINKTYPE: "locator",
XLINKHREF: "#" + factRef,
XLINKLABEL: locLabel
})
footnoteLabel = factLoc[(linkrole, factId)]
footnoteArc = addChild(footnoteLink,
XbrlConst.qnLinkFootnoteArc,
attributes={
XLINKTYPE: "arc",
XLINKARCROLE: arcrole,
XLINKFROM: locLabel,
XLINKTO: footnoteLabel
})
currentAction = "done loading facts and footnotes"
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
except Exception as ex:
if isinstance(ex, OIMException):
modelXbrl.error(ex.code,
ex.message,
modelObject=modelXbrl,
**ex.msgArgs)
else:
modelXbrl.error(
"arelleOIMloader:error",
"Error while %(action)s, error %(error)s\ntraceback %(traceback)s",
modelObject=modelXbrl,
action=currentAction,
error=ex,
traceback=traceback.format_tb(sys.exc_info()[2]))
if priorCWD:
os.chdir(
priorCWD
) # restore prior current working directory startingErrorCount = len(modelXbrl.errors)
if startingErrorCount < len(modelXbrl.errors):
# had errors, don't allow ModelDocument.load to continue
return OIMException("arelleOIMloader:unableToLoad",
"Unable to load due to reported errors")
return getattr(modelXbrl, "modelDocument",
None) # none if returning from exception
