def factCheck(val, fact):
concept = fact.concept
context = fact.context
if concept is None or context is None:
return # not checkable
try:
if fact.isNumeric and not fact.isNil and fact.xValue is not None:
# 2.4.1 decimal disagreement
if fact.decimals and fact.decimals != "INF":
vf = float(fact.value)
if _ISFINITE(vf):
dec = _INT(fact.decimals)
vround = round(vf, dec)
if vf != vround:
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.4.1",
_("Decimal disagreement %(fact)s in context %(contextID)s unit %(unitID)s value %(value)s has insignificant value %(insignificantValue)s"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID,
value=fact.effectiveValue, insignificantValue=Locale.format(val.modelXbrl.locale, "%.*f",
(dec + 2 if dec > 0 else 0, vf - vround),
True))
# 2.5.1 fractions disallowed on a disclosure
if fact.isFraction:
if any(val.linroleDefinitionIsDisclosure.match(roleType.definition)
for rel in val.modelXbrl.relationshipSet(XbrlConst.parentChild).toModelObject(concept)
for roleType in val.modelXbrl.roleTypes.get(rel.linkrole,())):
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.5.1",
_("Disclosure %(fact)s in context %(contextID)s value %(value)s is a fraction"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, value=fact.value)
# deprecated concept
if concept.qname.namespaceURI == ugtNamespace:
if concept.name in val.usgaapDeprecations:
val.deprecatedFactConcepts[concept].append(fact)
elif concept.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedFactConcepts[concept].append(fact)
if fact.isItem and fact.context is not None:
for dimConcept, modelDim in fact.context.segDimValues.items():
if dimConcept.qname.namespaceURI == ugtNamespace:
if dimConcept.name in val.usgaapDeprecations:
val.deprecatedDimensions[dimConcept].append(fact)
elif dimConcept.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedDimensions[dimConcept].append(fact)
if modelDim.isExplicit:
member = modelDim.member
if member is not None:
if member.qname.namespaceURI == ugtNamespace:
if member.name in val.usgaapDeprecations:
val.deprecatedMembers[member].append(fact)
elif member.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedMembers[member].append(fact)
except Exception as err:
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.testingException",
_("%(fact)s in context %(contextID)s unit %(unitID)s value %(value)s cannot be tested due to: %(err)s"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID,
value=fact.effectiveValue, err=err)
def effectiveValue(self):
concept = self.concept
if concept is None or concept.isTuple:
return None
if self.isNil:
return "(nil)"
if concept.isNumeric:
val = self.value
try:
num = float(val)
dec = self.decimals
if dec is None or dec == "INF":
dec = len(val.partition(".")[2])
else:
dec = int(dec) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
return Locale.format(self.modelXbrl.locale, "%.*f", (dec, num), True)
except ValueError:
return "(error)"
return self.value
def effectiveValue(self):
concept = self.concept
if concept is None or concept.isTuple:
return None
if self.isNil:
return "(nil)"
if concept.isNumeric:
val = self.value
try:
num = float(val)
dec = self.decimals
if dec is None or dec == "INF":
dec = len(val.partition(".")[2])
else:
dec = int(
dec
) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
return Locale.format(self.modelXbrl.locale, "%.*f", (dec, num),
True)
except ValueError:
return "(error)"
return self.value
def effectiveValue(self):
"""(str) -- Effective value for views, (nil) if isNil, None if no value,
locale-formatted string of decimal value (if decimals specified) , otherwise string value"""
concept = self.concept
if concept is None or concept.isTuple:
return None
if self.isNil:
return "(nil)"
try:
if concept.isNumeric:
val = self.value
try:
num = float(val)
dec = self.decimals
if dec is None or dec == "INF":
dec = len(val.partition(".")[2])
else:
dec = int(dec) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
return Locale.format(self.modelXbrl.locale, "%.*f", (dec, num), True)
except ValueError:
return "(error)"
return self.value
except Exception as ex:
return str(ex) # could be transform value of inline fact
def loadXbrlFromDB(self, loadDBsaveToFile):
# load from database
modelXbrl = self.modelXbrl
# find instance in DB
instanceURI = os.path.basename(loadDBsaveToFile)
results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant"
" FROM dInstance WHERE FileName = '{}'"
.format(instanceURI))
instanceId = moduleId = None
for instanceId, moduleId, entScheme, entId, datePerEnd in results:
break
# find module in DB
results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId))
xbrlSchemaRef = None
for result in results:
xbrlSchemaRef = result[0]
break
if not instanceId or not xbrlSchemaRef:
raise XPDBException("sqlDB:MissingDTS",
_("The instance and module were not found for %(instanceURI)"),
instanceURI = instanceURI)
if modelXbrl.skipDTS:
# find prefixes and namespaces in DB
results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]")
dpmPrefixedNamespaces = dict((prefix, namespace)
for owner, prefix, namespace in results)
# create the instance document and resulting filing
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(modelXbrl,
Type.INSTANCE,
loadDBsaveToFile,
schemaRefs=[xbrlSchemaRef],
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement,
'xbrl-streamable-instance',
'version="1.0" contextBuffer="1"')
# add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue)
# filing indicator code IDs
# get filing indicators
results = self.execute("SELECT mToT.TemplateOrTableCode "
" FROM dFilingIndicator dFI, mTemplateOrTable mToT "
" WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID"
.format(instanceId))
filingIndicatorCodes = [code[0] for code in results]
if filingIndicatorCodes:
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
{}, [], [],
id='c')
filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators)
for filingIndicatorCode in filingIndicatorCodes:
modelXbrl.createFact(qnFindFilingIndicator,
parent=filingIndicatorsTuple,
attributes={"contextRef": "c"},
text=filingIndicatorCode)
# facts in this instance
factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards,"
" Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue "
"FROM dFact WHERE InstanceID = {} "
"ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL "
" THEN DataPointSignature"
" ELSE DataPointSignatureWithValuesForWildcards"
" END, instr(DataPointSignature,'|') + 1)"
.format(instanceId))
# results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal
# get typed dimension values
prefixedNamespaces = modelXbrl.prefixedNamespaces
prefixedNamespaces["iso4217"] = XbrlConst.iso4217
if modelXbrl.skipDTS:
prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed
cntxTbl = {} # index by d
unitTbl = {}
def typedDimElt(s):
# add xmlns into s for known qnames
tag, angleBrkt, rest = s[1:].partition('>')
text, angleBrkt, rest = rest.partition("<")
qn = qname(tag, prefixedNamespaces)
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
# contexts and facts
for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl:
metric, sep, dims = (dpSigTypedDims or dpSig).partition('|')
conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces)
concept = modelXbrl.qnameConcepts.get(conceptQn)
isNumeric = isBool = isDateTime = isQName = isText = False
if concept is not None:
if concept.isNumeric:
isNumeric = True
else:
baseXbrliType = concept.baseXbrliType
if baseXbrliType == "booleanItemType":
isBool = True
elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
isDateTime = True
elif baseXbrliType == "QNameItemType":
isQName = True
else:
c = conceptQn.localName[0]
if c == 'm':
isNumeric = True
elif c == 'd':
isDateTime = True
elif c == 'b':
isBool = True
elif c == 'e':
isQName = True
isText = not (isNumeric or isBool or isDateTime or isQName)
if isinstance(datePerEnd, _STR_BASE):
datePerEnd = datetimeValue(datePerEnd, addOneDay=True)
cntxKey = (dims, entId, datePerEnd)
if cntxKey in cntxTbl:
cntxId = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
cntxTbl[cntxKey] = cntxId
qnameDims = {}
if dims:
for dim in dims.split('|'):
dQn, sep, dVal = dim[:-1].partition('(')
dimQname = qname(dQn, prefixedNamespaces)
if dVal.startswith('<'):
mem = typedDimElt(dVal) # typed dim
else:
mem = qname(dVal, prefixedNamespaces) # explicit dim (even if treat-as-typed)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario")
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId)
if unit:
if unit in unitTbl:
unitId = unitTbl[unit]
else:
unitQn = qname(unit, prefixedNamespaces)
unitId = 'u{}'.format(unitQn.localName)
unitTbl[unit] = unitId
modelXbrl.createUnit([unitQn], [], id=unitId)
else:
unitId = None
attrs = {"contextRef": cntxId}
if unitId:
attrs["unitRef"] = unitId
if dec is not None:
if isinstance(dec, float): # must be an integer
dec = int(dec)
elif isinstance(dec, _STR_BASE) and '.' in dec:
dec = dec.partition('.')[0] # drop .0 from any SQLite string
attrs["decimals"] = str(dec) # somehow it is float from the database
if False: # fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
elif numVal is not None:
num = roundValue(numVal, None, dec) # round using reported decimals
if dec is None or dec == "INF": # show using decimals or reported format
dec = len(numVal.partition(".")[2])
else: # max decimals at 28
dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num))
elif dateVal is not None:
text = dateVal
elif boolVal is not None:
text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false'
else:
if isQName: # declare namespace
addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces))
text = textVal
modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
# add footnotes if any
# save to file
modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
return modelXbrl.modelDocument
def factCheck(val, fact):
concept = fact.concept
context = fact.context
if concept is None or context is None or not val.validateLoggingSemantic:
return # not checkable
try:
if fact.isNumeric:
# 2.3.3 additional unit tests beyond UTR spec
unit = fact.unit
if unit is not None and concept.type is not None and val.validateUTR:
typeName = concept.type.name
if typeName == "perUnitItemType" and any(m.namespaceURI == XbrlConst.iso4217 or
m in (XbrlConst.qnXbrliPure, XbrlConst.qnXbrliShares)
for m in unit.measures[1]):
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.3.3.perUnitItemType",
_("PureItemType fact %(fact)s in context %(contextID)s unit %(unitID)s value %(value)s has disallowed unit denominator %(denominator)s"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID,
value=fact.effectiveValue, denominator=", ".join((str(m) for m in unit.measures[1])))
if not fact.isNil and getattr(fact, "xValue", None) is not None:
# 2.4.1 decimal disagreement
if fact.decimals and fact.decimals != "INF":
vf = float(fact.value)
if _ISFINITE(vf):
dec = _INT(fact.decimals)
vround = round(vf, dec)
if vf != vround:
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.4.1",
_("Decimal disagreement %(fact)s in context %(contextID)s unit %(unitID)s value %(value)s has insignificant value %(insignificantValue)s"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID,
value=fact.effectiveValue, insignificantValue=Locale.format(val.modelXbrl.locale, "%.*f",
(dec + 2 if dec > 0 else 0, vf - vround),
True))
# 2.5.1 fractions disallowed on a disclosure
if fact.isFraction:
if any(val.linroleDefinitionIsDisclosure.match(roleType.definition)
for rel in val.modelXbrl.relationshipSet(XbrlConst.parentChild).toModelObject(concept)
for roleType in val.modelXbrl.roleTypes.get(rel.linkrole,())):
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.5.1",
_("Disclosure %(fact)s in context %(contextID)s value %(value)s is a fraction"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, value=fact.value)
# deprecated concept
if concept.qname.namespaceURI == val.ugtNamespace:
if concept.name in val.usgaapDeprecations:
val.deprecatedFactConcepts[concept].append(fact)
elif concept.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedFactConcepts[concept].append(fact)
if fact.isItem and fact.context is not None:
for dimConcept, modelDim in fact.context.segDimValues.items():
if dimConcept.qname.namespaceURI == val.ugtNamespace:
if dimConcept.name in val.usgaapDeprecations:
val.deprecatedDimensions[dimConcept].append(fact)
elif dimConcept.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedDimensions[dimConcept].append(fact)
if modelDim.isExplicit:
member = modelDim.member
if member is not None:
if member.qname.namespaceURI == val.ugtNamespace:
if member.name in val.usgaapDeprecations:
val.deprecatedMembers[member].append(fact)
elif member.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedMembers[member].append(fact)
except Exception as err:
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.testingException",
_("%(fact)s in context %(contextID)s unit %(unitID)s value %(value)s cannot be tested due to: %(err)s"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID,
value=fact.effectiveValue, err=err)
def factCheck(val, fact):
concept = fact.concept
context = fact.context
if concept is None or context is None or not val.validateLoggingSemantic:
return # not checkable
try:
if fact.isNumeric:
# 2.3.3 additional unit tests beyond UTR spec
unit = fact.unit
if unit is not None and concept.type is not None and val.validateUTR:
typeName = concept.type.name
if typeName == "perUnitItemType" and any(m.namespaceURI == XbrlConst.iso4217 or
m in (XbrlConst.qnXbrliPure, XbrlConst.qnXbrliShares)
for m in unit.measures[1]):
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.3.3.perUnitItemType",
_("PureItemType fact %(fact)s in context %(contextID)s unit %(unitID)s value %(value)s has disallowed unit denominator %(denominator)s"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID,
value=fact.effectiveValue, denominator=", ".join((str(m) for m in unit.measures[1])))
if not fact.isNil and getattr(fact, "xValue", None) is not None:
# 2.4.1 decimal disagreement
if fact.decimals and fact.decimals != "INF":
vf = float(fact.value)
if _ISFINITE(vf):
dec = _INT(fact.decimals)
vround = round(vf, dec)
if vf != vround:
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.4.1",
_("Decimal disagreement %(fact)s in context %(contextID)s unit %(unitID)s value %(value)s has insignificant value %(insignificantValue)s"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID,
value=fact.effectiveValue, insignificantValue=Locale.format(val.modelXbrl.locale, "%.*f",
(dec + 2 if dec > 0 else 0, vf - vround),
True))
# 2.5.1 fractions disallowed on a disclosure
if fact.isFraction:
if any(val.linroleDefinitionIsDisclosure.match(roleType.definition)
for rel in val.modelXbrl.relationshipSet(XbrlConst.parentChild).toModelObject(concept)
for roleType in val.modelXbrl.roleTypes.get(rel.linkrole,())):
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.5.1",
_("Disclosure %(fact)s in context %(contextID)s value %(value)s is a fraction"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, value=fact.value)
# deprecated concept
if concept.qname.namespaceURI == val.ugtNamespace:
if concept.name in val.usgaapDeprecations:
val.deprecatedFactConcepts[concept].append(fact)
elif concept.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedFactConcepts[concept].append(fact)
if fact.isItem and fact.context is not None:
for dimConcept, modelDim in fact.context.segDimValues.items():
if dimConcept.qname.namespaceURI == val.ugtNamespace:
if dimConcept.name in val.usgaapDeprecations:
val.deprecatedDimensions[dimConcept].append(fact)
elif dimConcept.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedDimensions[dimConcept].append(fact)
if modelDim.isExplicit:
member = modelDim.member
if member is not None:
if member.qname.namespaceURI == val.ugtNamespace:
if member.name in val.usgaapDeprecations:
val.deprecatedMembers[member].append(fact)
elif member.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedMembers[member].append(fact)
except Exception as err:
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.testingException",
_("%(fact)s in context %(contextID)s unit %(unitID)s value %(value)s cannot be tested due to: %(err)s"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID,
value=fact.effectiveValue, err=err)
def loadXbrlFromDB(self, loadDBsaveToFile):
# load from database
modelXbrl = self.modelXbrl
# find instance in DB
instanceURI = os.path.basename(loadDBsaveToFile)
results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant"
" FROM dInstance WHERE FileName = '{}'"
.format(instanceURI))
instanceId = moduleId = None
for instanceId, moduleId, entScheme, entId, datePerEnd in results:
break
# find module in DB
results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId))
xbrlSchemaRef = None
for result in results:
xbrlSchemaRef = result[0]
break
if not instanceId or not xbrlSchemaRef:
raise XPDBException("sqlDB:MissingDTS",
_("The instance and module were not found for %(instanceURI)"),
instanceURI = instanceURI)
if modelXbrl.skipDTS:
# find prefixes and namespaces in DB
results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]")
dpmPrefixedNamespaces = dict((prefix, namespace)
for owner, prefix, namespace in results)
# create the instance document and resulting filing
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(modelXbrl,
Type.INSTANCE,
loadDBsaveToFile,
schemaRefs=[xbrlSchemaRef],
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement,
'xbrl-streamable-instance',
'version="1.0" contextBuffer="1"')
# add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue)
# filing indicator code IDs
# get filing indicators
results = self.execute("SELECT mToT.TemplateOrTableCode "
" FROM dFilingIndicator dFI, mTemplateOrTable mToT "
" WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID"
.format(instanceId))
filingIndicatorCodes = [code[0] for code in results]
if filingIndicatorCodes:
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
{}, [], [],
id='c')
filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators)
for filingIndicatorCode in filingIndicatorCodes:
modelXbrl.createFact(qnFindFilingIndicator,
parent=filingIndicatorsTuple,
attributes={"contextRef": "c"},
text=filingIndicatorCode)
# facts in this instance
factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards,"
" Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue "
"FROM dFact WHERE InstanceID = {} "
"ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL "
" THEN DataPointSignature"
" ELSE DataPointSignatureWithValuesForWildcards"
" END, instr(DataPointSignature,'|') + 1)"
.format(instanceId))
# results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal
# get typed dimension values
prefixedNamespaces = modelXbrl.prefixedNamespaces
prefixedNamespaces["iso4217"] = XbrlConst.iso4217
if modelXbrl.skipDTS:
prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed
cntxTbl = {} # index by d
unitTbl = {}
def typedDimElt(s):
# add xmlns into s for known qnames
tag, angleBrkt, rest = s[1:].partition('>')
text, angleBrkt, rest = rest.partition("<")
qn = qname(tag, prefixedNamespaces)
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
# contexts and facts
for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl:
metric, sep, dims = (dpSigTypedDims or dpSig).partition('|')
conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces)
concept = modelXbrl.qnameConcepts.get(conceptQn)
isNumeric = isBool = isDateTime = isQName = isText = False
if concept is not None:
if concept.isNumeric:
isNumeric = True
else:
baseXbrliType = concept.baseXbrliType
if baseXbrliType == "booleanItemType":
isBool = True
elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
isDateTime = True
elif baseXbrliType == "QNameItemType":
isQName = True
else:
c = conceptQn.localName[0]
if c == 'm':
isNumeric = True
elif c == 'd':
isDateTime = True
elif c == 'b':
isBool = True
elif c == 'e':
isQName = True
isText = not (isNumeric or isBool or isDateTime or isQName)
if isinstance(datePerEnd, _STR_BASE):
datePerEnd = datetimeValue(datePerEnd, addOneDay=True)
cntxKey = (dims, entId, datePerEnd)
if cntxKey in cntxTbl:
cntxId = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
cntxTbl[cntxKey] = cntxId
qnameDims = {}
if dims:
for dim in dims.split('|'):
dQn, sep, dVal = dim[:-1].partition('(')
dimQname = qname(dQn, prefixedNamespaces)
if dVal.startswith('<'): # typed dim
mem = typedDimElt(dVal)
else:
mem = qname(dVal, prefixedNamespaces)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario")
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId)
if unit:
if unit in unitTbl:
unitId = unitTbl[unit]
else:
unitQn = qname(unit, prefixedNamespaces)
unitId = 'u{}'.format(unitQn.localName)
unitTbl[unit] = unitId
modelXbrl.createUnit([unitQn], [], id=unitId)
else:
unitId = None
attrs = {"contextRef": cntxId}
if unitId:
attrs["unitRef"] = unitId
if dec is not None:
if isinstance(dec, float): # must be an integer
dec = int(dec)
elif isinstance(dec, _STR_BASE) and '.' in dec:
dec = dec.partition('.')[0] # drop .0 from any SQLite string
attrs["decimals"] = str(dec) # somehow it is float from the database
if False: # fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
elif numVal is not None:
num = roundValue(numVal, None, dec) # round using reported decimals
if dec is None or dec == "INF": # show using decimals or reported format
dec = len(numVal.partition(".")[2])
else: # max decimals at 28
dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num))
elif dateVal is not None:
text = dateVal
elif boolVal is not None:
text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false'
else:
if isQName: # declare namespace
addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces))
text = textVal
modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
# add footnotes if any
# save to file
modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
return modelXbrl.modelDocument
def validate(self):
if not self.modelXbrl.contexts and not self.modelXbrl.facts:
return # skip if no contexts or facts
self.modelXbrl.info("info","Validating calculations inferring %(inferMode)s",
inferMode=_("decimals") if self.inferDecimals else _("precision"))
# identify equal contexts
self.modelXbrl.profileActivity()
uniqueContextHashes = {}
for context in self.modelXbrl.contexts.values():
h = context.contextDimAwareHash
if h in uniqueContextHashes:
if context.isEqualTo(uniqueContextHashes[h]):
self.mapContext[context] = uniqueContextHashes[h]
else:
uniqueContextHashes[h] = context
del uniqueContextHashes
self.modelXbrl.profileActivity("... identify equal contexts", minTimeToShow=1.0)
# identify equal contexts
uniqueUnitHashes = {}
for unit in self.modelXbrl.units.values():
h = unit.hash
if h in uniqueUnitHashes:
if unit.isEqualTo(uniqueUnitHashes[h]):
self.mapUnit[unit] = uniqueUnitHashes[h]
else:
uniqueUnitHashes[h] = unit
self.modelXbrl.profileActivity("... identify equal units", minTimeToShow=1.0)
# identify concepts participating in essence-alias relationships
# identify calcluation & essence-alias base sets (by key)
for baseSetKey in self.modelXbrl.baseSets.keys():
arcrole, ELR, linkqname, arcqname = baseSetKey
if ELR and linkqname and arcqname:
if arcrole in (XbrlConst.essenceAlias, XbrlConst.requiresElement):
conceptsSet = {XbrlConst.essenceAlias:self.conceptsInEssencesAlias,
XbrlConst.requiresElement:self.conceptsInRequiresElement}[arcrole]
for modelRel in self.modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname).modelRelationships:
for concept in (modelRel.fromModelObject, modelRel.toModelObject):
conceptsSet.add(concept)
self.modelXbrl.profileActivity("... identify requires-element and esseance-aliased concepts", minTimeToShow=1.0)
self.bindFacts(self.modelXbrl.facts,[self.modelXbrl.modelDocument.xmlRootElement])
self.modelXbrl.profileActivity("... bind facts", minTimeToShow=1.0)
# identify calcluation & essence-alias base sets (by key)
for baseSetKey in self.modelXbrl.baseSets.keys():
arcrole, ELR, linkqname, arcqname = baseSetKey
if ELR and linkqname and arcqname:
if arcrole in (XbrlConst.summationItem, XbrlConst.essenceAlias, XbrlConst.requiresElement):
relsSet = self.modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname)
if arcrole == XbrlConst.summationItem:
fromRelationships = relsSet.fromModelObjects()
for sumConcept, modelRels in fromRelationships.items():
sumBindingKeys = self.sumConceptBindKeys[sumConcept]
dupBindingKeys = set()
boundSumKeys = set()
# determine boundSums
for modelRel in modelRels:
itemBindingKeys = self.itemConceptBindKeys[modelRel.toModelObject]
boundSumKeys |= sumBindingKeys & itemBindingKeys
# add up rounded items
boundSums = defaultdict(decimal.Decimal) # sum of facts meeting factKey
boundSummationItems = defaultdict(list) # corresponding fact refs for messages
for modelRel in modelRels:
weight = modelRel.weightDecimal
itemConcept = modelRel.toModelObject
for itemBindKey in boundSumKeys:
ancestor, contextHash, unit = itemBindKey
factKey = (itemConcept, ancestor, contextHash, unit)
if factKey in self.itemFacts:
for fact in self.itemFacts[factKey]:
if fact in self.duplicatedFacts:
dupBindingKeys.add(itemBindKey)
else:
boundSums[itemBindKey] += roundFact(fact, self.inferDecimals) * weight
boundSummationItems[itemBindKey].append(fact)
for sumBindKey in boundSumKeys:
ancestor, contextHash, unit = sumBindKey
factKey = (sumConcept, ancestor, contextHash, unit)
if factKey in self.sumFacts:
sumFacts = self.sumFacts[factKey]
for fact in sumFacts:
if fact in self.duplicatedFacts:
dupBindingKeys.add(sumBindKey)
elif sumBindKey not in dupBindingKeys:
roundedSum = roundFact(fact, self.inferDecimals)
roundedItemsSum = roundFact(fact, self.inferDecimals, vDecimal=boundSums[sumBindKey])
if roundedItemsSum != roundFact(fact, self.inferDecimals):
d = inferredDecimals(fact)
if isnan(d) or isinf(d): d = 4
self.modelXbrl.error("xbrl.5.2.5.2:calcInconsistency",
_("Calculation inconsistent from %(concept)s in link role %(linkrole)s reported sum %(reportedSum)s computed sum %(computedSum)s context %(contextID)s unit %(unitID)s"),
modelObject=[fact] + boundSummationItems[sumBindKey],
concept=sumConcept.qname, linkrole=ELR,
reportedSum=Locale.format(self.modelXbrl.locale, "%.*f", (d, roundedSum), True),
computedSum=Locale.format(self.modelXbrl.locale, "%.*f", (d, roundedItemsSum), True),
contextID=context.id, unitID=unit.id)
boundSummationItems.clear() # dereference facts in list
elif arcrole == XbrlConst.essenceAlias:
for modelRel in relsSet.modelRelationships:
essenceConcept = modelRel.fromModelObject
aliasConcept = modelRel.toModelObject
essenceBindingKeys = self.esAlConceptBindKeys[essenceConcept]
aliasBindingKeys = self.esAlConceptBindKeys[aliasConcept]
for esAlBindKey in essenceBindingKeys & aliasBindingKeys:
ancestor, contextHash = esAlBindKey
essenceFactsKey = (essenceConcept, ancestor, contextHash)
aliasFactsKey = (aliasConcept, ancestor, contextHash)
if essenceFactsKey in self.esAlFacts and aliasFactsKey in self.esAlFacts:
for eF in self.esAlFacts[essenceFactsKey]:
for aF in self.esAlFacts[aliasFactsKey]:
essenceUnit = self.mapUnit.get(eF.unit,eF.unit)
aliasUnit = self.mapUnit.get(aF.unit,aF.unit)
if essenceUnit != aliasUnit:
self.modelXbrl.error("xbrl.5.2.6.2.2:essenceAliasUnitsInconsistency",
_("Essence-Alias inconsistent units from %(essenceConcept)s to %(aliasConcept)s in link role %(linkrole)s context %(contextID)s"),
modelObject=(modelRel, eF, aF),
essenceConcept=essenceConcept.qname, aliasConcept=aliasConcept.qname,
linkrole=ELR, contextID=context.id)
if not XbrlUtil.vEqual(eF, aF):
self.modelXbrl.error("xbrl.5.2.6.2.2:essenceAliasUnitsInconsistency",
_("Essence-Alias inconsistent value from %(essenceConcept)s to %(aliasConcept)s in link role %(linkrole)s context %(contextID)s"),
modelObject=(modelRel, eF, aF),
essenceConcept=essenceConcept.qname, aliasConcept=aliasConcept.qname,
linkrole=ELR, contextID=context.id)
elif arcrole == XbrlConst.requiresElement:
for modelRel in relsSet.modelRelationships:
sourceConcept = modelRel.fromModelObject
requiredConcept = modelRel.toModelObject
if sourceConcept in self.requiresElementFacts and \
not requiredConcept in self.requiresElementFacts:
self.modelXbrl.error("xbrl.5.2.6.2.4:requiresElementInconsistency",
_("Requires-Element %(requiringConcept)s missing required fact for %(requiredConcept)s in link role %(linkrole)s"),
modelObject=sourceConcept,
requiringConcept=sourceConcept.qname, requiredConcept=requiredConcept.qname,
linkrole=ELR)
self.modelXbrl.profileActivity("... find inconsistencies", minTimeToShow=1.0)
self.modelXbrl.profileActivity() # reset
