def _values_unequal(val1, val2, dec_scale, margin_scale=2):
"""
Checks the values for equality based on their scaling.
Returns False if the values are equal, otherwise True.
:param val1: first value to round
:type val1: :class:'~arelle.ModelXbrl.ModelXbrl'
:param val2: second value to round
:type val2: :class:'~arelle.ModelXbrl.ModelXbrl'
:param dec_scale: precision on rounded value
:type dec_scale: :class:'~decimal.Decimal'
:param margin_scale: margin of scale for the margin of error
:type margin_scale: float
:return: True if the values are not equal
:rtype: bool
"""
if not _min_dec_valid(dec_scale):
return False
round_val1 = roundValue(val1, decimals=dec_scale)
round_val2 = roundValue(val2, decimals=dec_scale)
margin_of_error = (Decimal(margin_scale) *
(Decimal(10)**Decimal(-dec_scale)))
return (round_val1 < round_val2 - margin_of_error
or round_val1 > round_val2 + margin_of_error)
def _values_unequal(val1, val2, dec_scale, margin_scale=2):
"""
Checks the values for equality based on their scaling.
Returns False if the values are equal, otherwise True.
:param val1: first value to round
:type val1: :class:'~arelle.ModelXbrl.ModelXbrl'
:param val2: second value to round
:type val2: :class:'~arelle.ModelXbrl.ModelXbrl'
:param dec_scale: precision on rounded value
:type dec_scale: :class:'~decimal.Decimal'
:param margin_scale: margin of scale for the margin of error
:type margin_scale: float
:return: True if the values are not equal
:rtype: bool
"""
if not _min_dec_valid(dec_scale):
return False
round_val1 = roundValue(val1, decimals=dec_scale)
round_val2 = roundValue(val2, decimals=dec_scale)
margin_of_error = (
Decimal(margin_scale) * (Decimal(10) ** Decimal(-dec_scale))
)
return (
round_val1 < round_val2 - margin_of_error or
round_val1 > round_val2 + margin_of_error
)
def _assets_eq_liability_equity(modelXbrl):
assets_concept = modelXbrl.nameConcepts[_ASSETS_CONCEPT][
0] if modelXbrl.nameConcepts[_ASSETS_CONCEPT] else None
liability_equity_concept = modelXbrl.nameConcepts[_LIABILITIES_CONCEPT][
0] if modelXbrl.nameConcepts[_LIABILITIES_CONCEPT] else None
if assets_concept is not None and liability_equity_concept is not None:
assets_facts = modelXbrl.factsByQname[assets_concept.qname]
liability_equity_facts = modelXbrl.factsByQname[
liability_equity_concept.qname]
fact_dict = {}
fact_dict[_ASSETS_CONCEPT] = assets_facts
fact_dict[_LIABILITIES_CONCEPT] = liability_equity_facts
fact_groups = facts.prepare_facts_for_calculation(fact_dict)
for fact_group in fact_groups:
fact_assets = fact_group[_ASSETS_CONCEPT]
fact_liabilities = fact_group[_LIABILITIES_CONCEPT]
if fact_assets.context is not None and fact_assets.context.instantDatetime is not None:
dec_assets = inferredDecimals(fact_assets)
dec_liabilities = inferredDecimals(fact_liabilities)
min_dec = min(dec_assets, dec_liabilities)
if roundValue(fact_assets.xValue,
decimals=min_dec) != roundValue(
fact_liabilities.xValue, decimals=min_dec):
yield fact_assets, fact_liabilities
def isVEqualTo(self, other, deemP0Equal=False):
"""(bool) -- v-equality of two facts
Note that facts may be in different instances
"""
if self.isTuple or other.isTuple:
return False
if self.isNil:
return other.isNil
if other.isNil:
return False
if not self.context.isEqualTo(other.context):
return False
if self.concept.isNumeric:
if other.concept.isNumeric:
if not self.unit.isEqualTo(other.unit):
return False
if self.modelXbrl.modelManager.validateInferDecimals:
d = min((inferredDecimals(self), inferredDecimals(other))); p = None
if isnan(d) and deemP0Equal:
return True
else:
d = None; p = min((inferredPrecision(self), inferredPrecision(other)))
if p == 0 and deemP0Equal:
return True
return roundValue(self.value,precision=p,decimals=d) == roundValue(other.value,precision=p,decimals=d)
else:
return False
selfValue = self.value
otherValue = other.value
if isinstance(selfValue,str) and isinstance(otherValue,str):
return selfValue.strip() == otherValue.strip()
else:
return selfValue == otherValue
def isVEqualTo(self, other):
if self.isTuple or other.isTuple:
return False
if self.isNil:
return other.isNil
if other.isNil:
return False
if not self.context.isEqualTo(other.context):
return False
if self.concept.isNumeric:
if other.concept.isNumeric:
if not self.unit.isEqualTo(other.unit):
return False
if self.modelXbrl.modelManager.validateInferDecimals:
d = min((inferredDecimals(self), inferredDecimals(other))); p = None
else:
d = None; p = min((inferredPrecision(self), inferredPrecision(other)))
return roundValue(float(self.value),precision=p,decimals=d) == roundValue(float(other.value),precision=p,decimals=d)
else:
return False
selfValue = self.value
otherValue = other.value
if isinstance(selfValue,str) and isinstance(otherValue,str):
return selfValue.strip() == otherValue.strip()
else:
return selfValue == otherValue
def _values_unequal(val1, val2, dec_scale, margin_scale=2):
"""
Checks the values for equality based on their scaling.
Returns False if the values are equal, otherwise True.
"""
round_val1 = roundValue(val1, decimals=dec_scale)
round_val2 = roundValue(val2, decimals=dec_scale)
margin_of_error = Decimal(margin_scale) * (Decimal(10) ** Decimal(-dec_scale))
return round_val1 < round_val2 - margin_of_error or round_val1 > round_val2 + margin_of_error
def parsed_value(fact: ModelFact):
if fact is None:
return None
concept = fact.concept # type: ModelConcept
if concept is None or concept.isTuple or fact.isNil:
return None
if concept.isFraction:
num, den = map(fractions.Fraction, fact.fractionValue)
return num / den
val = fact.value.strip()
if concept.isInteger:
return int(val)
elif concept.isNumeric:
dec = fact.decimals
if dec is None or dec == "INF": # show using decimals or reported format
dec = len(val.partition(".")[2])
else: # max decimals at 28
dec = max(
min(int(dec), 28), -28
) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
num = roundValue(val, fact.precision,
dec) # round using reported decimals
return num
elif concept.baseXbrliType == 'dateItemType':
return dateTime(val)
elif concept.baseXbrliType == 'booleanItemType':
return val.lower() in ('1', 'true')
elif concept.isTextBlock:
return ' '.join(val.split())
return val
def insertFactSet(modelFacts, tupleFactId):
table = self.getTable('fact', 'fact_id',
('accession_id', 'tuple_fact_id', 'context_id', 'unit_id', 'element_id', 'effective_value', 'fact_value',
'xml_id', 'precision_value', 'decimals_value',
'is_precision_infinity', 'is_decimals_infinity', ),
('accession_id', 'xml_id'),
tuple((accsId,
tupleFactId,
self.cntxId.get((accsId,fact.contextID)),
self.unitId.get((accsId,fact.unitID)),
self.conceptElementId(fact.concept),
roundValue(fact.value, fact.precision, fact.decimals) if fact.isNumeric and not fact.isNil else None,
fact.value,
elementFragmentIdentifier(fact),
fact.xAttributes['precision'].xValue if ('precision' in fact.xAttributes and isinstance(fact.xAttributes['precision'].xValue,int)) else None,
fact.xAttributes['decimals'].xValue if ('decimals' in fact.xAttributes and isinstance(fact.xAttributes['decimals'].xValue,int)) else None,
'precision' in fact.xAttributes and fact.xAttributes['precision'].xValue == 'INF',
'decimals' in fact.xAttributes and fact.xAttributes['decimals'].xValue == 'INF',
)
for fact in modelFacts))
factId = dict((xmlId, id)
for id, _accsId, xmlId in table)
for fact in modelFacts:
if fact.isTuple:
insertFactSet(fact.modelTupleFacts,
factId[elementFragmentIdentifier(fact)])
def _assets_eq_liability_equity(modelXbrl):
assets_concept = modelXbrl.nameConcepts[_ASSETS_CONCEPT][0] if modelXbrl.nameConcepts[_ASSETS_CONCEPT] else None
liability_equity_concept = modelXbrl.nameConcepts[_LIABILITIES_CONCEPT][0] if modelXbrl.nameConcepts[_LIABILITIES_CONCEPT] else None
if assets_concept is not None and liability_equity_concept is not None:
assets_facts = modelXbrl.factsByQname[assets_concept.qname]
liability_equity_facts = modelXbrl.factsByQname[liability_equity_concept.qname]
fact_dict = {}
fact_dict[_ASSETS_CONCEPT] = assets_facts
fact_dict[_LIABILITIES_CONCEPT] = liability_equity_facts
fact_groups = facts.prepare_facts_for_calculation(fact_dict)
for fact_group in fact_groups:
fact_assets = fact_group[_ASSETS_CONCEPT]
fact_liabilities = fact_group[_LIABILITIES_CONCEPT]
if fact_assets.context is not None and fact_assets.context.instantDatetime is not None:
dec_assets = inferredDecimals(fact_assets)
dec_liabilities = inferredDecimals(fact_liabilities)
min_dec = min(dec_assets, dec_liabilities)
if roundValue(fact_assets.xValue, decimals=min_dec) != roundValue(fact_liabilities.xValue, decimals=min_dec):
yield fact_assets, fact_liabilities
def isVEqualTo(self,
other,
deemP0Equal=False): # facts may be in different instances
if self.isTuple or other.isTuple:
return False
if self.isNil:
return other.isNil
if other.isNil:
return False
if not self.context.isEqualTo(other.context):
return False
if self.concept.isNumeric:
if other.concept.isNumeric:
if not self.unit.isEqualTo(other.unit):
return False
if self.modelXbrl.modelManager.validateInferDecimals:
d = min((inferredDecimals(self), inferredDecimals(other)))
p = None
else:
d = None
p = min(
(inferredPrecision(self), inferredPrecision(other)))
if p == 0 and deemP0Equal:
return True
return roundValue(self.value, precision=p,
decimals=d) == roundValue(other.value,
precision=p,
decimals=d)
else:
return False
selfValue = self.value
otherValue = other.value
if isinstance(selfValue, str) and isinstance(otherValue, str):
return selfValue.strip() == otherValue.strip()
else:
return selfValue == otherValue
def insertFactSet(modelFacts, parentDatapointId):
table = self.getTable('data_point', 'datapoint_id',
('report_id', 'document_id', 'xml_id', 'source_line',
'parent_datapoint_id', # tuple
'aspect_id',
'context_xml_id', 'entity_id', 'period_id', 'aspect_value_selections_id', 'unit_id',
'is_nil', 'precision_value', 'decimals_value', 'effective_value', 'value'),
('document_id', 'xml_id'),
tuple((reportId,
documentId,
elementFragmentIdentifier(fact),
fact.sourceline,
parentDatapointId, # parent ID
self.aspectQnameId.get(fact.qname),
fact.contextID,
self.entityId.get((reportId, cntx.entityIdentifier[0], cntx.entityIdentifier[1]))
if cntx is not None else None,
self.periodId.get((reportId,
cntx.startDatetime if cntx.isStartEndPeriod else None,
cntx.endDatetime if cntx.isStartEndPeriod else None,
cntx.isInstantPeriod,
cntx.isForeverPeriod)) if cntx is not None else None,
cntxAspectValueSelectionSetId.get(cntx) if cntx is not None else None,
self.unitId.get((reportId,fact.unit.md5hash)) if fact.unit is not None else None,
fact.isNil,
fact.precision,
fact.decimals,
roundValue(fact.value, fact.precision, fact.decimals) if fact.isNumeric and not fact.isNil else None,
fact.value
)
for fact in modelFacts
for cntx in (fact.context,)
for documentId in (self.documentIds[fact.modelDocument],)))
xmlIdDataPointId = dict(((docId, xmlId), datapointId)
for datapointId, docId, xmlId in table)
self.factDataPointId.update(xmlIdDataPointId)
for fact in modelFacts:
if fact.isTuple:
insertFactSet(fact.modelTupleFacts,
xmlIdDataPointId[(self.documentIds[fact.modelDocument],
elementFragmentIdentifier(fact))])
def _roundPrecision(node, sphinxContext, args):
args = numericArgs(node, sphinxContext, args, 2)
return roundValue(args[0], precision=args[1])
def _roundDecimals(node, sphinxContext, args):
args = numericArgs(node, sphinxContext, args, 2)
return roundValue(args[0], decmials=args[1])
def _roundItem(node, sphinxContext, args):
fact = factArg(node, sphinxContext, args, 0)
return roundValue(fact.xValue, decimals=fact.decimals, precision=fact.precision)
def loadEntityInformation(dts, rssItem):
entityInformation = {}
# identify tables
disclosureSystem = dts.modelManager.disclosureSystem
if disclosureSystem.EFM:
reloadCache = False
if rssItem is not None:
accession = rssItem.url.split('/')[-2]
fileUrl = os.path.dirname(rssItem.url) + '/' + accession[0:10] + '-' + accession[10:12] + '-' + accession[12:] + ".hdr.sgml"
reloadCache = getattr(rssItem.modelXbrl, "reloadCache", False)
elif dts.uri.startswith("http://www.sec.gov/Archives/edgar/data") and dts.uri.endswith(".xml"):
accession = dts.uri.split('/')[-2]
dirPart = os.path.dirname(dts.uri)
if accession.endswith("-xbrl.zip"): # might be an instance document inside a xbrl.zip file
accession = dts.uri.split('/')[-3]
dirPart = os.path.dirname(dirPart)
fileUrl = dirPart + '/' + accession[0:10] + '-' + accession[10:12] + '-' + accession[12:] + ".hdr.sgml"
else:
fileUrl = ''
if fileUrl:
# try to load and use it
normalizedUrl = dts.modelManager.cntlr.webCache.normalizeUrl(fileUrl)
hdrSgml = ''
try:
filePath = dts.modelManager.cntlr.webCache.getfilename(normalizedUrl, reload=reloadCache)
if filePath:
with open(filePath) as fh:
hdrSgml = fh.read()
except (IOError, EnvironmentError) as err:
dts.info("xpDB:headerSgmlDocumentLoadingError",
_("Loading XBRL DB: header SGML document %(file)s loading error: %(error)s"),
modelObject=dts, file=normalizedUrl, error=err)
hdrSgml = ''
record = ''
formerCompanyNumber = 0
for match in re.finditer(r"[<]([^>]+)[>]([^<\n\r]*)", hdrSgml, re.MULTILINE):
tag = match.group(1).lower()
v = match.group(2).replace("<","<").replace(">",">").replace("&","&")
if tag in ('business-address','mail-address'):
record = tag + '.'
elif tag == 'former-company':
formerCompanyNumber += 1
record = "{}-{}.".format(tag, formerCompanyNumber)
elif tag.startswith('/'):
record = ''
elif v:
if tag.endswith("-datetime"):
try:
v = datetime.datetime(_INT(v[0:4]),_INT(v[4:6]),_INT(v[6:8]),_INT(v[8:10]),_INT(v[10:12]),_INT(v[12:14]))
except ValueError:
pass
elif tag.endswith("-date") or tag.startswith("date-of-"):
try:
v = datetime.date(_INT(v[0:4]),_INT(v[4:6]),_INT(v[6:8]))
except ValueError:
pass
elif tag.endswith("-year-end") and len(v) == 4:
v = "{0}-{1}".format(v[0:2],v[2:4])
elif tag in ('assigned-sic',):
try:
v = _INT(v)
except ValueError:
v = None
entityInformation[record + tag] = v
# primary document if no rssItem
if rssItem is None:
# try to sgml txt file
normalizedUrl = normalizedUrl.replace(".hdr.sgml", ".txt")
httpDir = normalizedUrl.rpartition('/')[0]
txtSgml = ''
try:
filePath = dts.modelManager.cntlr.webCache.getfilename(normalizedUrl, reload=reloadCache)
if filePath:
with open(filePath, encoding='utf-8') as fh:
txtSgml = fh.read()
# remove from cache, very large file
os.remove(filePath)
except (IOError, EnvironmentError) as err:
dts.info("xpDB:txtSgmlDocumentLoadingError",
_("Loading XBRL DB: txt SGML document %(file)s loading error: %(error)s"),
modelObject=dts, file=normalizedUrl, error=err)
txtSgml = ''
documentType = documentSequence = None
itemsFound = 0
for match in re.finditer(r"[<]([^>]+)[>]([^<\n\r]*)", txtSgml, re.MULTILINE):
tag = match.group(1).lower()
v = match.group(2).replace("<","<").replace(">",">").replace("&","&")
if tag == 'sequence':
documentSequence = v
elif tag == 'type':
documentType = v
elif tag == 'filename':
if documentType.endswith('.INS') and 'instance-url' not in entityInformation:
entityInformation['instance-url'] = httpDir + '/' + v
documentType = documentSequence = None
itemsFound += 1
if documentSequence == '1':
entityInformation['primary-document-url'] = httpDir + '/' + v
documentType = documentSequence = None
itemsFound += 1
if itemsFound >= 2:
break
del txtSgml # dereference big string
# instance information
for factName, entityField in (("EntityFilerCategory", "filer-category"),
("EntityPublicFloat", "public-float"),
("TradingSymbol", "trading-symbol"),
("DocumentFisalYearFocus", "fiscal-year-focus"),
("DocumentFisalPeriodFocus", "fiscal-period-focus")):
try:
concept = dts.nameConcepts[factName][0] # get qname irrespective of taxonomy year
facts = dts.factsByQname[concept.qname]
for fact in facts:
if not fact.context.qnameDims: #default context
if factName in ("EntityPublicFloat",):
entityInformation[entityField] = roundValue(fact.value, fact.precision, fact.decimals) if fact.isNumeric and not fact.isNil else None
else:
entityInformation[entityField] = fact.value.strip() # may have white space
break
except IndexError:
pass
return entityInformation
def loadXbrlFromDB(self, loadDBsaveToFile):
# load from database
modelXbrl = self.modelXbrl
# find instance in DB
instanceURI = os.path.basename(loadDBsaveToFile)
results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant"
" FROM dInstance WHERE FileName = '{}'"
.format(instanceURI))
instanceId = moduleId = None
for instanceId, moduleId, entScheme, entId, datePerEnd in results:
break
# find module in DB
results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId))
xbrlSchemaRef = None
for result in results:
xbrlSchemaRef = result[0]
break
if not instanceId or not xbrlSchemaRef:
raise XPDBException("sqlDB:MissingDTS",
_("The instance and module were not found for %(instanceURI)"),
instanceURI = instanceURI)
if modelXbrl.skipDTS:
# find prefixes and namespaces in DB
results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]")
dpmPrefixedNamespaces = dict((prefix, namespace)
for owner, prefix, namespace in results)
# create the instance document and resulting filing
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(modelXbrl,
Type.INSTANCE,
loadDBsaveToFile,
schemaRefs=[xbrlSchemaRef],
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement,
'xbrl-streamable-instance',
'version="1.0" contextBuffer="1"')
# add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue)
# filing indicator code IDs
# get filing indicators
results = self.execute("SELECT mToT.TemplateOrTableCode "
" FROM dFilingIndicator dFI, mTemplateOrTable mToT "
" WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID"
.format(instanceId))
filingIndicatorCodes = [code[0] for code in results]
if filingIndicatorCodes:
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
{}, [], [],
id='c')
filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators)
for filingIndicatorCode in filingIndicatorCodes:
modelXbrl.createFact(qnFindFilingIndicator,
parent=filingIndicatorsTuple,
attributes={"contextRef": "c"},
text=filingIndicatorCode)
# facts in this instance
factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards,"
" Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue "
"FROM dFact WHERE InstanceID = {} "
"ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL "
" THEN DataPointSignature"
" ELSE DataPointSignatureWithValuesForWildcards"
" END, instr(DataPointSignature,'|') + 1)"
.format(instanceId))
# results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal
# get typed dimension values
prefixedNamespaces = modelXbrl.prefixedNamespaces
prefixedNamespaces["iso4217"] = XbrlConst.iso4217
if modelXbrl.skipDTS:
prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed
cntxTbl = {} # index by d
unitTbl = {}
def typedDimElt(s):
# add xmlns into s for known qnames
tag, angleBrkt, rest = s[1:].partition('>')
text, angleBrkt, rest = rest.partition("<")
qn = qname(tag, prefixedNamespaces)
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
# contexts and facts
for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl:
metric, sep, dims = (dpSigTypedDims or dpSig).partition('|')
conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces)
concept = modelXbrl.qnameConcepts.get(conceptQn)
isNumeric = isBool = isDateTime = isQName = isText = False
if concept is not None:
if concept.isNumeric:
isNumeric = True
else:
baseXbrliType = concept.baseXbrliType
if baseXbrliType == "booleanItemType":
isBool = True
elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
isDateTime = True
elif baseXbrliType == "QNameItemType":
isQName = True
else:
c = conceptQn.localName[0]
if c == 'm':
isNumeric = True
elif c == 'd':
isDateTime = True
elif c == 'b':
isBool = True
elif c == 'e':
isQName = True
isText = not (isNumeric or isBool or isDateTime or isQName)
if isinstance(datePerEnd, _STR_BASE):
datePerEnd = datetimeValue(datePerEnd, addOneDay=True)
cntxKey = (dims, entId, datePerEnd)
if cntxKey in cntxTbl:
cntxId = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
cntxTbl[cntxKey] = cntxId
qnameDims = {}
if dims:
for dim in dims.split('|'):
dQn, sep, dVal = dim[:-1].partition('(')
dimQname = qname(dQn, prefixedNamespaces)
if dVal.startswith('<'):
mem = typedDimElt(dVal) # typed dim
else:
mem = qname(dVal, prefixedNamespaces) # explicit dim (even if treat-as-typed)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario")
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId)
if unit:
if unit in unitTbl:
unitId = unitTbl[unit]
else:
unitQn = qname(unit, prefixedNamespaces)
unitId = 'u{}'.format(unitQn.localName)
unitTbl[unit] = unitId
modelXbrl.createUnit([unitQn], [], id=unitId)
else:
unitId = None
attrs = {"contextRef": cntxId}
if unitId:
attrs["unitRef"] = unitId
if dec is not None:
if isinstance(dec, float): # must be an integer
dec = int(dec)
elif isinstance(dec, _STR_BASE) and '.' in dec:
dec = dec.partition('.')[0] # drop .0 from any SQLite string
attrs["decimals"] = str(dec) # somehow it is float from the database
if False: # fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
elif numVal is not None:
num = roundValue(numVal, None, dec) # round using reported decimals
if dec is None or dec == "INF": # show using decimals or reported format
dec = len(numVal.partition(".")[2])
else: # max decimals at 28
dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num))
elif dateVal is not None:
text = dateVal
elif boolVal is not None:
text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false'
else:
if isQName: # declare namespace
addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces))
text = textVal
modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
# add footnotes if any
# save to file
modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
return modelXbrl.modelDocument
def evaluate(xpCtx, varSet, derivedFact):
# there may be multiple consis assertions parenting any formula
for consisAsserRel in xpCtx.modelXbrl.relationshipSet(
XbrlConst.consistencyAssertionFormula).toModelObject(varSet):
consisAsser = consisAsserRel.fromModelObject
hasProportionalAcceptanceRadius = consisAsser.hasProportionalAcceptanceRadius
hasAbsoluteAcceptanceRadius = consisAsser.hasAbsoluteAcceptanceRadius
if derivedFact is None:
continue
isNumeric = derivedFact.isNumeric
if isNumeric and not derivedFact.isNil:
derivedFactInferredPrecision = inferredPrecision(derivedFact)
if derivedFactInferredPrecision == 0 and not hasProportionalAcceptanceRadius and not hasAbsoluteAcceptanceRadius:
if xpCtx.formulaOptions.traceVariableSetExpressionResult:
xpCtx.modelXbrl.error(
_("Consistency assertion {0} formula {1} fact {2} has zero precision and no radius is defined, skipping consistency assertion"
).format(consisAsser.id, varSet.xlinkLabel,
derivedFact), "info", "formula:trace")
continue
# check xbrl validity of new fact
# find source facts which match derived fact
aspectMatchedInputFacts = []
isStrict = consisAsser.isStrict
for inputFact in xpCtx.modelXbrl.facts:
if (not inputFact.isNil and inputFact.qname == derivedFact.qname
and inputFact.context.isEqualTo(
derivedFact.context,
dimensionalAspectModel=(varSet.aspectModel
== "dimensional")) and
(not isNumeric or inputFact.unit.isEqualTo(derivedFact.unit))):
aspectMatchedInputFacts.append(inputFact)
if len(aspectMatchedInputFacts) == 0:
if isStrict:
if derivedFact.isNil:
isSatisfied = True
else:
isSatisfied = False
else:
if xpCtx.formulaOptions.traceVariableSetExpressionResult:
xpCtx.modelXbrl.error(
_("Consistency assertion {0} Formula {1} no input facts matched to {2}, skipping consistency assertion"
).format(consisAsser.id, varSet.xlinkLabel,
derivedFact), "info", "formula:trace")
continue
elif derivedFact.isNil:
isSatisfied = False
else:
isSatisfied = True
paramQnamesAdded = []
for paramRel in consisAsser.orderedVariableRelationships:
paramQname = paramRel.variableQname
paramVar = paramRel.toModelObject
paramValue = xpCtx.inScopeVars.get(paramVar.qname)
paramAlreadyInVars = paramQname in xpCtx.inScopeVars
if not paramAlreadyInVars:
paramQnamesAdded.append(paramQname)
xpCtx.inScopeVars[paramQname] = paramValue
for fact in aspectMatchedInputFacts:
if isSatisfied != True:
break
if fact.isNil:
if not derivedFact.isNil:
isSatisfied = False
elif isNumeric:
factInferredPrecision = inferredPrecision(fact)
if factInferredPrecision == 0 and not hasProportionalAcceptanceRadius and not hasAbsoluteAcceptanceRadius:
if xpCtx.formulaOptions.traceVariableSetExpressionResult:
xpCtx.modelXbrl.error(
_("Consistency assertion {0} Formula {1} input fact matched to {2} has zero precision and no radius, skipping consistency assertion"
).format(consisAsser.id, varSet.xlinkLabel,
derivedFact), "info", "formula:trace")
isSatisfied = None
break
if hasProportionalAcceptanceRadius or hasAbsoluteAcceptanceRadius:
acceptance = consisAsser.evalRadius(
xpCtx, derivedFact.vEqValue)
if acceptance is not None:
if hasProportionalAcceptanceRadius:
acceptance *= derivedFact.vEqValue
isSatisfied = fabs(derivedFact.vEqValue -
fact.vEqValue) <= fabs(acceptance)
else:
isSatisfied = None # no radius
else:
p = min(derivedFactInferredPrecision,
factInferredPrecision)
if (p == 0 or roundValue(derivedFact.vEqValue, precision=p)
!= roundValue(fact.vEqValue, precision=p)):
isSatisfied = False
else:
if not xEqual(fact.concept,
fact.element,
derivedFact.element,
equalMode=S_EQUAL2):
isSatisfied = False
for paramQname in paramQnamesAdded:
xpCtx.inScopeVars.pop(paramQname)
if isSatisfied is None:
continue # no evaluation
if xpCtx.formulaOptions.traceVariableSetExpressionResult:
xpCtx.modelXbrl.error(
_("Consistency Assertion {0} result {1}").format(
consisAsser.id, isSatisfied), "info", "formula:trace")
message = consisAsser.message(isSatisfied)
if message:
xpCtx.modelXbrl.error(message.evaluate(xpCtx), "info",
"message:" + consisAsser.id)
if isSatisfied: consisAsser.countSatisfied += 1
else: consisAsser.countNotSatisfied += 1
def evaluate(xpCtx, varSet, derivedFact):
# there may be multiple consis assertions parenting any formula
for consisAsserRel in xpCtx.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).toModelObject(varSet):
consisAsser = consisAsserRel.fromModelObject
hasProportionalAcceptanceRadius = consisAsser.hasProportionalAcceptanceRadius
hasAbsoluteAcceptanceRadius = consisAsser.hasAbsoluteAcceptanceRadius
if derivedFact is None:
continue
isNumeric = derivedFact.isNumeric
if isNumeric and not derivedFact.isNil:
derivedFactInferredPrecision = inferredPrecision(derivedFact)
if derivedFactInferredPrecision == 0 and not hasProportionalAcceptanceRadius and not hasAbsoluteAcceptanceRadius:
if xpCtx.formulaOptions.traceVariableSetExpressionResult:
xpCtx.modelXbrl.error( _("Consistency assertion {0} formula {1} fact {2} has zero precision and no radius is defined, skipping consistency assertion").format(
consisAsser.id, varSet.xlinkLabel, derivedFact),
"info", "formula:trace")
continue
# check xbrl validity of new fact
# find source facts which match derived fact
aspectMatchedInputFacts = []
isStrict = consisAsser.isStrict
for inputFact in xpCtx.modelXbrl.facts:
if (not inputFact.isNil and
inputFact.qname == derivedFact.qname and
inputFact.context.isEqualTo(derivedFact.context,
dimensionalAspectModel=(varSet.aspectModel == "dimensional")) and
(not isNumeric or inputFact.unit.isEqualTo(derivedFact.unit))):
aspectMatchedInputFacts.append( inputFact )
if len(aspectMatchedInputFacts) == 0:
if isStrict:
if derivedFact.isNil:
isSatisfied = True
else:
isSatisfied = False
else:
if xpCtx.formulaOptions.traceVariableSetExpressionResult:
xpCtx.modelXbrl.error( _("Consistency assertion {0} Formula {1} no input facts matched to {2}, skipping consistency assertion").format(
consisAsser.id, varSet.xlinkLabel, derivedFact),
"info", "formula:trace")
continue
elif derivedFact.isNil:
isSatisfied = False
else:
isSatisfied = True
paramQnamesAdded = []
for paramRel in consisAsser.orderedVariableRelationships:
paramQname = paramRel.variableQname
paramVar = paramRel.toModelObject
paramValue = xpCtx.inScopeVars.get(paramVar.qname)
paramAlreadyInVars = paramQname in xpCtx.inScopeVars
if not paramAlreadyInVars:
paramQnamesAdded.append(paramQname)
xpCtx.inScopeVars[paramQname] = paramValue
for fact in aspectMatchedInputFacts:
if isSatisfied != True:
break
if fact.isNil:
if not derivedFact.isNil:
isSatisfied = False
elif isNumeric:
factInferredPrecision = inferredPrecision(fact)
if factInferredPrecision == 0 and not hasProportionalAcceptanceRadius and not hasAbsoluteAcceptanceRadius:
if xpCtx.formulaOptions.traceVariableSetExpressionResult:
xpCtx.modelXbrl.error( _("Consistency assertion {0} Formula {1} input fact matched to {2} has zero precision and no radius, skipping consistency assertion").format(
consisAsser.id, varSet.xlinkLabel, derivedFact),
"info", "formula:trace")
isSatisfied = None
break
if hasProportionalAcceptanceRadius or hasAbsoluteAcceptanceRadius:
acceptance = consisAsser.evalRadius(xpCtx, derivedFact.vEqValue)
if acceptance is not None:
if hasProportionalAcceptanceRadius:
acceptance *= derivedFact.vEqValue
isSatisfied = fabs(derivedFact.vEqValue - fact.vEqValue) <= fabs(acceptance)
else:
isSatisfied = None # no radius
else:
p = min(derivedFactInferredPrecision, factInferredPrecision)
if (p == 0 or
roundValue(derivedFact.vEqValue, precision=p) != roundValue(fact.vEqValue, precision=p)):
isSatisfied = False
else:
if not xEqual(fact.concept, fact.element, derivedFact.element, equalMode=S_EQUAL2):
isSatisfied = False
for paramQname in paramQnamesAdded:
xpCtx.inScopeVars.pop(paramQname)
if isSatisfied is None:
continue # no evaluation
if xpCtx.formulaOptions.traceVariableSetExpressionResult:
xpCtx.modelXbrl.error( _("Consistency Assertion {0} result {1}").format( consisAsser.id, isSatisfied),
"info", "formula:trace")
message = consisAsser.message(isSatisfied)
if message:
xpCtx.modelXbrl.error(message.evaluate(xpCtx), "info", "message:" + consisAsser.id)
if isSatisfied: consisAsser.countSatisfied += 1
else: consisAsser.countNotSatisfied += 1
def loadXbrlFromDB(self, loadDBsaveToFile):
# load from database
modelXbrl = self.modelXbrl
# find instance in DB
instanceURI = os.path.basename(loadDBsaveToFile)
results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant"
" FROM dInstance WHERE FileName = '{}'"
.format(instanceURI))
instanceId = moduleId = None
for instanceId, moduleId, entScheme, entId, datePerEnd in results:
break
# find module in DB
results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId))
xbrlSchemaRef = None
for result in results:
xbrlSchemaRef = result[0]
break
if not instanceId or not xbrlSchemaRef:
raise XPDBException("sqlDB:MissingDTS",
_("The instance and module were not found for %(instanceURI)"),
instanceURI = instanceURI)
if modelXbrl.skipDTS:
# find prefixes and namespaces in DB
results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]")
dpmPrefixedNamespaces = dict((prefix, namespace)
for owner, prefix, namespace in results)
# create the instance document and resulting filing
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(modelXbrl,
Type.INSTANCE,
loadDBsaveToFile,
schemaRefs=[xbrlSchemaRef],
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement,
'xbrl-streamable-instance',
'version="1.0" contextBuffer="1"')
# add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue)
# filing indicator code IDs
# get filing indicators
results = self.execute("SELECT mToT.TemplateOrTableCode "
" FROM dFilingIndicator dFI, mTemplateOrTable mToT "
" WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID"
.format(instanceId))
filingIndicatorCodes = [code[0] for code in results]
if filingIndicatorCodes:
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
{}, [], [],
id='c')
filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators)
for filingIndicatorCode in filingIndicatorCodes:
modelXbrl.createFact(qnFindFilingIndicator,
parent=filingIndicatorsTuple,
attributes={"contextRef": "c"},
text=filingIndicatorCode)
# facts in this instance
factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards,"
" Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue "
"FROM dFact WHERE InstanceID = {} "
"ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL "
" THEN DataPointSignature"
" ELSE DataPointSignatureWithValuesForWildcards"
" END, instr(DataPointSignature,'|') + 1)"
.format(instanceId))
# results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal
# get typed dimension values
prefixedNamespaces = modelXbrl.prefixedNamespaces
prefixedNamespaces["iso4217"] = XbrlConst.iso4217
if modelXbrl.skipDTS:
prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed
cntxTbl = {} # index by d
unitTbl = {}
def typedDimElt(s):
# add xmlns into s for known qnames
tag, angleBrkt, rest = s[1:].partition('>')
text, angleBrkt, rest = rest.partition("<")
qn = qname(tag, prefixedNamespaces)
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
# contexts and facts
for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl:
metric, sep, dims = (dpSigTypedDims or dpSig).partition('|')
conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces)
concept = modelXbrl.qnameConcepts.get(conceptQn)
isNumeric = isBool = isDateTime = isQName = isText = False
if concept is not None:
if concept.isNumeric:
isNumeric = True
else:
baseXbrliType = concept.baseXbrliType
if baseXbrliType == "booleanItemType":
isBool = True
elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
isDateTime = True
elif baseXbrliType == "QNameItemType":
isQName = True
else:
c = conceptQn.localName[0]
if c == 'm':
isNumeric = True
elif c == 'd':
isDateTime = True
elif c == 'b':
isBool = True
elif c == 'e':
isQName = True
isText = not (isNumeric or isBool or isDateTime or isQName)
if isinstance(datePerEnd, _STR_BASE):
datePerEnd = datetimeValue(datePerEnd, addOneDay=True)
cntxKey = (dims, entId, datePerEnd)
if cntxKey in cntxTbl:
cntxId = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
cntxTbl[cntxKey] = cntxId
qnameDims = {}
if dims:
for dim in dims.split('|'):
dQn, sep, dVal = dim[:-1].partition('(')
dimQname = qname(dQn, prefixedNamespaces)
if dVal.startswith('<'): # typed dim
mem = typedDimElt(dVal)
else:
mem = qname(dVal, prefixedNamespaces)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario")
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId)
if unit:
if unit in unitTbl:
unitId = unitTbl[unit]
else:
unitQn = qname(unit, prefixedNamespaces)
unitId = 'u{}'.format(unitQn.localName)
unitTbl[unit] = unitId
modelXbrl.createUnit([unitQn], [], id=unitId)
else:
unitId = None
attrs = {"contextRef": cntxId}
if unitId:
attrs["unitRef"] = unitId
if dec is not None:
if isinstance(dec, float): # must be an integer
dec = int(dec)
elif isinstance(dec, _STR_BASE) and '.' in dec:
dec = dec.partition('.')[0] # drop .0 from any SQLite string
attrs["decimals"] = str(dec) # somehow it is float from the database
if False: # fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
elif numVal is not None:
num = roundValue(numVal, None, dec) # round using reported decimals
if dec is None or dec == "INF": # show using decimals or reported format
dec = len(numVal.partition(".")[2])
else: # max decimals at 28
dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num))
elif dateVal is not None:
text = dateVal
elif boolVal is not None:
text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false'
else:
if isQName: # declare namespace
addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces))
text = textVal
modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
# add footnotes if any
# save to file
modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
return modelXbrl.modelDocument
def loadEntityInformation(dts, entrypoint, rssItem):
entityInformation = {}
# identify tables
disclosureSystem = dts.modelManager.disclosureSystem
if disclosureSystem.validationType == "EFM":
reloadCache = False
if rssItem is not None:
accession = rssItem.url.split('/')[-2]
fileUrl = os.path.dirname(rssItem.url) + '/' + accession[0:10] + '-' + accession[10:12] + '-' + accession[12:] + ".hdr.sgml"
reloadCache = getattr(rssItem.modelXbrl, "reloadCache", False)
elif dts.uri.startswith("http://www.sec.gov/Archives/edgar/data") and dts.uri.endswith(".xml"):
accession = dts.uri.split('/')[-2]
dirPart = os.path.dirname(dts.uri)
if accession.endswith("-xbrl.zip"): # might be an instance document inside a xbrl.zip file
accession = dts.uri.split('/')[-3]
dirPart = os.path.dirname(dirPart)
fileUrl = dirPart + '/' + accession[0:10] + '-' + accession[10:12] + '-' + accession[12:] + ".hdr.sgml"
else:
fileUrl = ''
if fileUrl:
# try to load and use it
normalizedUrl = dts.modelManager.cntlr.webCache.normalizeUrl(fileUrl)
hdrSgml = ''
try:
filePath = dts.modelManager.cntlr.webCache.getfilename(normalizedUrl, reload=reloadCache)
if filePath:
with open(filePath) as fh:
hdrSgml = fh.read()
except (IOError, EnvironmentError) as err:
dts.info("xpDB:headerSgmlDocumentLoadingError",
_("Loading XBRL DB: header SGML document %(file)s loading error: %(error)s"),
modelObject=dts, file=normalizedUrl, error=err)
hdrSgml = ''
record = ''
formerCompanyNumber = 0
for match in re.finditer(r"[<]([^>]+)[>]([^<\n\r]*)", hdrSgml, re.MULTILINE):
tag = match.group(1).lower()
v = match.group(2).replace("<","<").replace(">",">").replace("&","&")
if tag in ('business-address','mail-address'):
record = tag + '.'
elif tag == 'former-company':
formerCompanyNumber += 1
record = "{}-{}.".format(tag, formerCompanyNumber)
elif tag.startswith('/'):
record = ''
elif v:
if tag.endswith("-datetime"):
try:
v = datetime.datetime(_INT(v[0:4]),_INT(v[4:6]),_INT(v[6:8]),_INT(v[8:10]),_INT(v[10:12]),_INT(v[12:14]))
except ValueError:
pass
elif tag.endswith("-date") or tag.startswith("date-of-"):
try:
v = datetime.date(_INT(v[0:4]),_INT(v[4:6]),_INT(v[6:8]))
except ValueError:
pass
elif tag.endswith("-year-end") and len(v) == 4:
v = "{0}-{1}".format(v[0:2],v[2:4])
elif tag in ('assigned-sic',):
try:
v = _INT(v)
except ValueError:
v = None
entityInformation[record + tag] = v
# primary document if no rssItem
if rssItem is None:
# try to sgml txt file
normalizedUrl = normalizedUrl.replace(".hdr.sgml", ".txt")
httpDir = normalizedUrl.rpartition('/')[0]
txtSgml = ''
try:
filePath = dts.modelManager.cntlr.webCache.getfilename(normalizedUrl, reload=reloadCache)
if filePath:
with open(filePath, encoding='utf-8') as fh:
txtSgml = fh.read()
# remove from cache, very large file
os.remove(filePath)
except (IOError, EnvironmentError) as err:
dts.info("xpDB:txtSgmlDocumentLoadingError",
_("Loading XBRL DB: txt SGML document %(file)s loading error: %(error)s"),
modelObject=dts, file=normalizedUrl, error=err)
txtSgml = ''
documentType = documentSequence = None
itemsFound = 0
for match in re.finditer(r"[<]([^>]+)[>]([^<\n\r]*)", txtSgml, re.MULTILINE):
tag = match.group(1).lower()
v = match.group(2).replace("<","<").replace(">",">").replace("&","&")
if tag == 'sequence':
documentSequence = v
elif tag == 'type':
documentType = v
elif tag == 'filename':
if documentType.endswith('.INS') and 'instance-url' not in entityInformation:
entityInformation['instance-url'] = httpDir + '/' + v
documentType = documentSequence = None
itemsFound += 1
if documentSequence == '1':
entityInformation['primary-document-url'] = httpDir + '/' + v
documentType = documentSequence = None
itemsFound += 1
if itemsFound >= 2:
break
del txtSgml # dereference big string
# instance information
for factName, entityField in (("EntityFilerCategory", "filer-category"),
("EntityPublicFloat", "public-float"),
("TradingSymbol", "trading-symbol"),
("DocumentFisalYearFocus", "fiscal-year-focus"),
("DocumentFisalPeriodFocus", "fiscal-period-focus"),
("DocumentType", "document-type")):
try:
concept = dts.nameConcepts[factName][0] # get qname irrespective of taxonomy year
facts = dts.factsByQname[concept.qname]
for fact in facts:
if not fact.context.qnameDims: #default context
if factName in ("EntityPublicFloat",):
entityInformation[entityField] = roundValue(fact.value, fact.precision, fact.decimals) if fact.isNumeric and not fact.isNil else None
else:
entityInformation[entityField] = fact.value.strip() # may have white space
break
except IndexError:
pass
return entityInformation
def insertFacts(self):
accsId = self.accessionId
self.showStatus("insert facts")
# units
table = self.getTable('unit', 'unit_id',
('accession_id', 'unit_xml_id'),
('accession_id', 'unit_xml_id'),
tuple((accsId,
unitId)
for unitId in self.modelXbrl.units.keys()))
self.unitId = dict(((_accsId, xmlId), id)
for id, _accsId, xmlId in table)
# measures
table = self.getTable('unit_measure', 'unit_measure_id',
('unit_id', 'qname_id', 'location_id'),
('qname_id', 'location_id'),
tuple((self.unitId[(accsId,unit.id)],
self.qnameId[measure],
1 if (not unit.measures[1]) else (i + 1))
for unit in self.modelXbrl.units.values()
for i in range(2)
for measure in unit.measures[i]))
#table = self.getTable('enumeration_measure_location', 'enumeration_measure_location_id',
# ('description',),
# ('description',),
# (('measure',), ('numerator',), ('denominator',)))
# context
table = self.getTable('context', 'context_id',
('accession_id', 'period_start', 'period_end', 'period_instant', 'specifies_dimensions', 'context_xml_id', 'entity_scheme', 'entity_identifier'),
('accession_id', 'context_xml_id'),
tuple((accsId,
cntx.startDatetime if cntx.isStartEndPeriod else None,
cntx.endDatetime if cntx.isStartEndPeriod else None,
cntx.instantDatetime if cntx.isInstantPeriod else None,
bool(cntx.qnameDims),
cntx.id,
cntx.entityIdentifier[0],
cntx.entityIdentifier[1])
for cntx in self.modelXbrl.contexts.values()))
self.cntxId = dict(((_accsId, xmlId), id)
for id, _accsId, xmlId in table)
# context_dimension
values = []
for cntx in self.modelXbrl.contexts.values():
for dim in cntx.qnameDims.values():
values.append((self.cntxId[(accsId,cntx.id)],
self.qnameId[dim.dimensionQname],
self.qnameId.get(dim.memberQname), # may be None
self.qnameId.get(dim.typedMember.qname) if dim.isTyped else None,
False, # not default
dim.contextElement == "segment",
dim.typedMember.innerText if dim.isTyped else None))
for dimQname, memQname in self.modelXbrl.qnameDimensionDefaults.items():
if dimQname not in cntx.qnameDims:
values.append((self.cntxId[(accsId,cntx.id)],
self.qnameId[dimQname],
self.qnameId[memQname],
None,
True, # is default
True, # ambiguous and irrelevant for the XDT model
None))
table = self.getTable('context_dimension', 'context_dimension_id',
('context_id', 'dimension_qname_id', 'member_qname_id', 'typed_qname_id', 'is_default', 'is_segment', 'typed_text_content'),
('dimension_qname_id',),
values)
# facts
table = self.getTable('fact', 'fact_id',
('accession_id', 'context_id', 'unit_id', 'element_id', 'effective_value', 'fact_value',
'xml_id', 'precision_value', 'decimals_value',
'is_precision_infinity', 'is_decimals_infinity', ),
('accession_id', 'context_id', 'unit_id', 'element_id', 'fact_value'),
tuple((accsId,
self.cntxId.get((accsId,fact.contextID)),
self.unitId.get((accsId,fact.unitID)),
self.elementId.get(self.qnameId.get(fact.qname)),
roundValue(fact.value, fact.precision, fact.decimals) if fact.isNumeric else None,
fact.value,
fact.id,
fact.xAttributes['precision'].xValue if ('precision' in fact.xAttributes and isinstance(fact.xAttributes['precision'].xValue,int)) else None,
fact.xAttributes['decimals'].xValue if ('decimals' in fact.xAttributes and isinstance(fact.xAttributes['decimals'].xValue,int)) else None,
'precision' in fact.xAttributes and fact.xAttributes['precision'].xValue == 'INF',
'decimals' in fact.xAttributes and fact.xAttributes['decimals'].xValue == 'INF',
)
for fact in self.modelXbrl.facts))
def _roundItem(node, sphinxContext, args):
fact = factArg(node, sphinxContext, args, 0)
return roundValue(fact.xValue,
decimals=fact.decimals,
precision=fact.precision)
def _roundDecimals(node, sphinxContext, args):
args = numericArgs(node, sphinxContext, args, 2)
return roundValue(args[0], decmials=args[1])
def _roundPrecision(node, sphinxContext, args):
args = numericArgs(node, sphinxContext, args, 2)
return roundValue(args[0], precision=args[1])
def insertFacts(self):
accsId = self.accessionId
self.showStatus("insert facts")
# units
table = self.getTable(
'unit', 'unit_id', ('accession_id', 'unit_xml_id'),
('accession_id', 'unit_xml_id'),
tuple((accsId, unitId) for unitId in self.modelXbrl.units.keys()))
self.unitId = dict(
((_accsId, xmlId), id) for id, _accsId, xmlId in table)
# measures
table = self.getTable(
'unit_measure', 'unit_measure_id',
('unit_id', 'qname_id', 'location_id'),
('qname_id', 'location_id'),
tuple((self.unitId[(accsId, unit.id)], self.qnameId[measure],
1 if (not unit.measures[1]) else (i + 1))
for unit in self.modelXbrl.units.values() for i in range(2)
for measure in unit.measures[i]))
#table = self.getTable('enumeration_measure_location', 'enumeration_measure_location_id',
# ('description',),
# ('description',),
# (('measure',), ('numerator',), ('denominator',)))
# context
table = self.getTable(
'context', 'context_id',
('accession_id', 'period_start', 'period_end', 'period_instant',
'specifies_dimensions', 'context_xml_id', 'entity_scheme',
'entity_identifier'), ('accession_id', 'context_xml_id'),
tuple(
(accsId, cntx.startDatetime if cntx.isStartEndPeriod else None,
cntx.endDatetime if cntx.isStartEndPeriod else None,
cntx.instantDatetime if cntx.isInstantPeriod else None,
bool(cntx.qnameDims), cntx.id, cntx.entityIdentifier[0],
cntx.entityIdentifier[1])
for cntx in self.modelXbrl.contexts.values()))
self.cntxId = dict(
((_accsId, xmlId), id) for id, _accsId, xmlId in table)
# context_dimension
values = []
for cntx in self.modelXbrl.contexts.values():
for dim in cntx.qnameDims.values():
values.append((
self.cntxId[(accsId, cntx.id)],
self.qnameId[dim.dimensionQname],
self.qnameId.get(dim.memberQname), # may be None
self.qnameId.get(dim.typedMember.qname)
if dim.isTyped else None,
False, # not default
dim.contextElement == "segment",
dim.typedMember.innerText if dim.isTyped else None))
for dimQname, memQname in self.modelXbrl.qnameDimensionDefaults.items(
):
if dimQname not in cntx.qnameDims:
values.append((
self.cntxId[(accsId, cntx.id)],
self.qnameId[dimQname],
self.qnameId[memQname],
None,
True, # is default
True, # ambiguous and irrelevant for the XDT model
None))
table = self.getTable(
'context_dimension', 'context_dimension_id',
('context_id', 'dimension_qname_id', 'member_qname_id',
'typed_qname_id', 'is_default', 'is_segment',
'typed_text_content'), ('dimension_qname_id', ), values)
# facts
table = self.getTable(
'fact', 'fact_id', (
'accession_id',
'context_id',
'unit_id',
'element_id',
'effective_value',
'fact_value',
'xml_id',
'precision_value',
'decimals_value',
'is_precision_infinity',
'is_decimals_infinity',
), ('accession_id', 'context_id', 'unit_id', 'element_id',
'fact_value'),
tuple((
accsId,
self.cntxId.get((accsId, fact.contextID)),
self.unitId.get((accsId, fact.unitID)),
self.elementId.get(self.qnameId.get(fact.qname)),
roundValue(fact.value, fact.precision, fact.decimals) if fact.
isNumeric else None,
fact.value,
fact.id,
fact.xAttributes['precision'].xValue if (
'precision' in fact.xAttributes and isinstance(
fact.xAttributes['precision'].xValue, int)) else None,
fact.xAttributes['decimals'].xValue if (
'decimals' in fact.xAttributes and isinstance(
fact.xAttributes['decimals'].xValue, int)) else None,
'precision' in fact.xAttributes
and fact.xAttributes['precision'].xValue == 'INF',
'decimals' in fact.xAttributes
and fact.xAttributes['decimals'].xValue == 'INF',
) for fact in self.modelXbrl.facts))
def evaluate(xpCtx, varSet, derivedFact):
# there may be multiple consis assertions parenting any formula
for consisAsserRel in xpCtx.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).toModelObject(varSet):
consisAsser = consisAsserRel.fromModelObject
hasProportionalAcceptanceRadius = consisAsser.hasProportionalAcceptanceRadius
hasAbsoluteAcceptanceRadius = consisAsser.hasAbsoluteAcceptanceRadius
if derivedFact is None:
continue
isNumeric = derivedFact.isNumeric
if isNumeric and not derivedFact.isNil:
derivedFactInferredPrecision = inferredPrecision(derivedFact)
if derivedFactInferredPrecision == 0 and not hasProportionalAcceptanceRadius and not hasAbsoluteAcceptanceRadius:
if xpCtx.formulaOptions.traceVariableSetExpressionResult:
xpCtx.modelXbrl.info(u"formula:trace",
_(u"Consistency assertion %(id)s formula %(xlinkLabel)s fact %(derivedFact)s has zero precision and no radius is defined, skipping consistency assertion"),
modelObject=consisAsser, id=consisAsser.id, xlinkLabel=varSet.xlinkLabel, derivedFact=derivedFact)
continue
# check xbrl validity of new fact
# find source facts which match derived fact
aspectMatchedInputFacts = []
isStrict = consisAsser.isStrict
for inputFact in xpCtx.modelXbrl.facts:
if (not inputFact.isNil and
inputFact.qname == derivedFact.qname and
inputFact.context.isEqualTo(derivedFact.context,
dimensionalAspectModel=(varSet.aspectModel == u"dimensional")) and
(not isNumeric or inputFact.unit.isEqualTo(derivedFact.unit))):
aspectMatchedInputFacts.append( inputFact )
if len(aspectMatchedInputFacts) == 0:
if isStrict:
if derivedFact.isNil:
isSatisfied = True
else:
isSatisfied = False
else:
if xpCtx.formulaOptions.traceVariableSetExpressionResult:
xpCtx.modelXbrl.info(u"formula:trace",
_(u"Consistency assertion %(id)s formula %(xlinkLabel)s no input facts matched to %(derivedFact)s, skipping consistency assertion"),
modelObject=consisAsser, id=consisAsser.id, xlinkLabel=varSet.xlinkLabel, derivedFact=derivedFact)
continue
elif derivedFact.isNil:
isSatisfied = False
else:
isSatisfied = True
paramQnamesAdded = []
for paramRel in consisAsser.orderedVariableRelationships:
paramQname = paramRel.variableQname
paramVar = paramRel.toModelObject
paramValue = xpCtx.inScopeVars.get(paramVar.parameterQname)
paramAlreadyInVars = paramQname in xpCtx.inScopeVars
if not paramAlreadyInVars:
paramQnamesAdded.append(paramQname)
xpCtx.inScopeVars[paramQname] = paramValue
acceptance = None
for fact in aspectMatchedInputFacts:
if isSatisfied != True:
break
if fact.isNil:
if not derivedFact.isNil:
isSatisfied = False
elif isNumeric:
factInferredPrecision = inferredPrecision(fact)
if factInferredPrecision == 0 and not hasProportionalAcceptanceRadius and not hasAbsoluteAcceptanceRadius:
if xpCtx.formulaOptions.traceVariableSetExpressionResult:
xpCtx.modelXbrl.info(u"formula:trace",
_(u"Consistency assertion %(id)s formula %(xlinkLabel)s input fact matched to %(derivedFact)s has zero precision and no radius, skipping consistency assertion"),
modelObject=consisAsser, id=consisAsser.id, xlinkLabel=varSet.xlinkLabel, derivedFact=derivedFact)
isSatisfied = None
break
if hasProportionalAcceptanceRadius or hasAbsoluteAcceptanceRadius:
acceptance = consisAsser.evalRadius(xpCtx, derivedFact.vEqValue)
if acceptance is not None:
if hasProportionalAcceptanceRadius:
acceptance *= derivedFact.vEqValue
isSatisfied = fabs(derivedFact.vEqValue - fact.vEqValue) <= fabs(acceptance)
else:
isSatisfied = None # no radius
else:
p = min(derivedFactInferredPrecision, factInferredPrecision)
if (p == 0 or
roundValue(derivedFact.value, precision=p) != roundValue(fact.value, precision=p)):
isSatisfied = False
else:
if not xEqual(fact, derivedFact, equalMode=S_EQUAL2):
isSatisfied = False
if isSatisfied is not None: # None means no evaluation
if xpCtx.formulaOptions.traceVariableSetExpressionResult:
xpCtx.modelXbrl.info(u"formula:trace",
_(u"Consistency assertion %(id)s result %(result)s"),
modelObject=consisAsser, id=consisAsser.id, result=isSatisfied)
message = consisAsser.message(isSatisfied)
if message is not None:
xpCtx.inScopeVars[XbrlConst.qnCaAspectMatchedFacts] = aspectMatchedInputFacts
xpCtx.inScopeVars[XbrlConst.qnCaAcceptanceRadius] = acceptance
xpCtx.inScopeVars[XbrlConst.qnCaAbsoluteAcceptanceRadiusExpression] = consisAsser.get(u"absoluteAcceptanceRadius")
xpCtx.inScopeVars[XbrlConst.qnCaProportionalAcceptanceRadiusExpression] = consisAsser.get(u"proportionalAcceptanceRadius")
xpCtx.modelXbrl.info(u"message:" + consisAsser.id, message.evaluate(xpCtx),
modelObject=message,
messageCodes=(u"message:{variableSetID|xlinkLabel}"))
xpCtx.inScopeVars.pop(XbrlConst.qnCaAspectMatchedFacts)
xpCtx.inScopeVars.pop(XbrlConst.qnCaAcceptanceRadius)
xpCtx.inScopeVars.pop(XbrlConst.qnCaAbsoluteAcceptanceRadiusExpression)
xpCtx.inScopeVars.pop(XbrlConst.qnCaProportionalAcceptanceRadiusExpression)
if isSatisfied: consisAsser.countSatisfied += 1
else: consisAsser.countNotSatisfied += 1
for paramQname in paramQnamesAdded:
xpCtx.inScopeVars.pop(paramQname)
