def evaluateTableIndex(modelXbrl):
disclosureSystem = modelXbrl.modelManager.disclosureSystem
if disclosureSystem.EFM:
COVER = "1Cover"
STMTS = "2Financial Statements"
NOTES = "3Notes to Financial Statements"
POLICIES = "4Accounting Policies"
TABLES = "5Notes Tables"
DETAILS = "6Notes Details"
UNCATEG = "7Uncategorized"
roleDefinitionPattern = re.compile(r"([0-9]+) - (Statement|Disclosure|Schedule|Document) - (.+)")
# build EFM rendering-compatible index
definitionElrs = dict((roleType.definition, roleType)
for roleURI in modelXbrl.relationshipSet(XbrlConst.parentChild).linkRoleUris
for roleType in modelXbrl.roleTypes.get(roleURI,()))
isRR = any(ns.startswith("http://xbrl.sec.gov/rr/") for ns in modelXbrl.namespaceDocs.keys())
tableGroup = None
firstTableLinkroleURI = None
firstDocumentLinkroleURI = None
sortedRoleTypes = sorted(definitionElrs.items(), key=lambda item: item[0])
for roleDefinition, roleType in sortedRoleTypes:
roleType._tableChildren = []
match = roleDefinitionPattern.match(roleDefinition) if roleDefinition else None
if not match:
roleType._tableIndex = (UNCATEG, "", roleType.roleURI)
continue
seq, tblType, tblName = match.groups()
if isRR:
tableGroup = COVER
elif not tableGroup:
tableGroup = ("Paren" in tblName and COVER or tblType == "Statement" and STMTS or
"(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or
"(Detail" in tblName and DETAILS or COVER)
elif tableGroup == COVER:
tableGroup = (tblType == "Statement" and STMTS or "Paren" in tblName and COVER or
"(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or
"(Detail" in tblName and DETAILS or NOTES)
elif tableGroup == STMTS:
tableGroup = ((tblType == "Statement" or "Paren" in tblName) and STMTS or
"(Polic" in tblName and NOTES or "(Table" in tblName and TABLES or
"(Detail" in tblName and DETAILS or NOTES)
elif tableGroup == NOTES:
tableGroup = ("(Polic" in tblName and POLICIES or "(Table" in tblName and TABLES or
"(Detail" in tblName and DETAILS or tblType == "Disclosure" and NOTES or UNCATEG)
elif tableGroup == POLICIES:
tableGroup = ("(Table" in tblName and TABLES or "(Detail" in tblName and DETAILS or
("Paren" in tblName or "(Polic" in tblName) and POLICIES or UNCATEG)
elif tableGroup == TABLES:
tableGroup = ("(Detail" in tblName and DETAILS or
("Paren" in tblName or "(Table" in tblName) and TABLES or UNCATEG)
elif tableGroup == DETAILS:
tableGroup = (("Paren" in tblName or "(Detail" in tblName) and DETAILS or UNCATEG)
else:
tableGroup = UNCATEG
if firstTableLinkroleURI is None and tableGroup == COVER:
firstTableLinkroleURI = roleType.roleURI
if tblType == "Document" and not firstDocumentLinkroleURI:
firstDocumentLinkroleURI = roleType.roleURI
roleType._tableIndex = (tableGroup, seq, tblName)
# flow allocate facts to roles (SEC presentation groups)
if not modelXbrl.qnameDimensionDefaults: # may not have run validatino yet
from arelle import ValidateXbrlDimensions
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
reportedFacts = set() # facts which were shown in a higher-numbered ELR table
factsByQname = modelXbrl.factsByQname
reportingPeriods = set()
nextEnd = None
deiFact = {}
for conceptName in ("DocumentPeriodEndDate", "DocumentType", "CurrentFiscalPeriodEndDate"):
for concept in modelXbrl.nameConcepts[conceptName]:
for fact in factsByQname[concept.qname]:
deiFact[conceptName] = fact
if fact.context is not None:
reportingPeriods.add((None, fact.context.endDatetime)) # for instant
reportingPeriods.add((fact.context.startDatetime, fact.context.endDatetime)) # for startEnd
nextEnd = fact.context.startDatetime
duration = (fact.context.endDatetime - fact.context.startDatetime).days + 1
break
if "DocumentType" in deiFact:
fact = deiFact["DocumentType"]
if "-Q" in fact.xValue:
# need quarterly and yr to date durations
endDatetime = fact.context.endDatetime
# if within 2 days of end of month use last day of month
endDatetimeMonth = endDatetime.month
if (endDatetime + timedelta(2)).month != endDatetimeMonth:
# near end of month
endOfMonth = True
while endDatetime.month == endDatetimeMonth:
endDatetime += timedelta(1) # go forward to next month
else:
endOfMonth = False
startYr = endDatetime.year
startMo = endDatetime.month - 3
if startMo <= 0:
startMo += 12
startYr -= 1
startDatetime = datetime(startYr, startMo, endDatetime.day, endDatetime.hour, endDatetime.minute, endDatetime.second)
if endOfMonth:
startDatetime -= timedelta(1)
endDatetime -= timedelta(1)
reportingPeriods.add((startDatetime, endDatetime))
duration = 91
# find preceding compatible default context periods
while (nextEnd is not None):
thisEnd = nextEnd
prevMaxStart = thisEnd - timedelta(duration * .9)
prevMinStart = thisEnd - timedelta(duration * 1.1)
nextEnd = None
for cntx in modelXbrl.contexts.values():
if (cntx.isStartEndPeriod and not cntx.qnameDims and thisEnd == cntx.endDatetime and
prevMinStart <= cntx.startDatetime <= prevMaxStart):
reportingPeriods.add((None, cntx.endDatetime))
reportingPeriods.add((cntx.startDatetime, cntx.endDatetime))
nextEnd = cntx.startDatetime
break
elif (cntx.isInstantPeriod and not cntx.qnameDims and thisEnd == cntx.endDatetime):
reportingPeriods.add((None, cntx.endDatetime))
stmtReportingPeriods = set(reportingPeriods)
sortedRoleTypes.reverse() # now in descending order
for i, roleTypes in enumerate(sortedRoleTypes):
roleDefinition, roleType = roleTypes
# find defined non-default axes in pre hierarchy for table
tableFacts = set()
tableGroup, tableSeq, tableName = roleType._tableIndex
roleURIdims, priItemQNames = EFMlinkRoleURIstructure(modelXbrl, roleType.roleURI)
for priItemQName in priItemQNames:
for fact in factsByQname[priItemQName]:
cntx = fact.context
# non-explicit dims must be default
if (cntx is not None and
all(dimQn in modelXbrl.qnameDimensionDefaults
for dimQn in (roleURIdims.keys() - cntx.qnameDims.keys())) and
all(mdlDim.memberQname in roleURIdims[dimQn]
for dimQn, mdlDim in cntx.qnameDims.items()
if dimQn in roleURIdims)):
# the flow-up part, drop
cntxStartDatetime = cntx.startDatetime
cntxEndDatetime = cntx.endDatetime
if (tableGroup != STMTS or
(cntxStartDatetime, cntxEndDatetime) in stmtReportingPeriods and
(fact not in reportedFacts or
all(dimQn not in cntx.qnameDims # unspecified dims are all defaulted if reported elsewhere
for dimQn in (cntx.qnameDims.keys() - roleURIdims.keys())))):
tableFacts.add(fact)
reportedFacts.add(fact)
roleType._tableFacts = tableFacts
# find parent if any
closestParentType = None
closestParentMatchLength = 0
for _parentRoleDefinition, parentRoleType in sortedRoleTypes[i+1:]:
matchLen = parentNameMatchLen(tableName, parentRoleType)
if matchLen > closestParentMatchLength:
closestParentMatchLength = matchLen
closestParentType = parentRoleType
if closestParentType is not None:
closestParentType._tableChildren.insert(0, roleType)
# remove lesser-matched children if there was a parent match
unmatchedChildRoles = set()
longestChildMatchLen = 0
numChildren = 0
for childRoleType in roleType._tableChildren:
matchLen = parentNameMatchLen(tableName, childRoleType)
if matchLen < closestParentMatchLength:
unmatchedChildRoles.add(childRoleType)
elif matchLen > longestChildMatchLen:
longestChildMatchLen = matchLen
numChildren += 1
if numChildren > 1:
# remove children that don't have the full match pattern length to parent
for childRoleType in roleType._tableChildren:
if (childRoleType not in unmatchedChildRoles and
parentNameMatchLen(tableName, childRoleType) < longestChildMatchLen):
unmatchedChildRoles.add(childRoleType)
for unmatchedChildRole in unmatchedChildRoles:
roleType._tableChildren.remove(unmatchedChildRole)
for childRoleType in roleType._tableChildren:
childRoleType._tableParent = roleType
unmatchedChildRoles = None # dereference
global UGT_TOPICS
if UGT_TOPICS is None:
try:
from arelle import FileSource
fh = FileSource.openFileStream(modelXbrl.modelManager.cntlr,
os.path.join(modelXbrl.modelManager.cntlr.configDir, "ugt-topics.zip/ugt-topics.json"),
'r', 'utf-8')
UGT_TOPICS = json.load(fh)
fh.close()
for topic in UGT_TOPICS:
topic[6] = set(topic[6]) # change concept abstracts list into concept abstracts set
topic[7] = set(topic[7]) # change concept text blocks list into concept text blocks set
topic[8] = set(topic[8]) # change concept names list into concept names set
except Exception as ex:
UGT_TOPICS = None
if UGT_TOPICS is not None:
def roleUgtConcepts(roleType):
roleConcepts = set()
for rel in modelXbrl.relationshipSet(XbrlConst.parentChild, roleType.roleURI).modelRelationships:
if rel.toModelObject is not None:
roleConcepts.add(rel.toModelObject.name)
if rel.fromModelObject is not None:
roleConcepts.add(rel.fromModelObject.name)
if hasattr(roleType, "_tableChildren"):
for _tableChild in roleType._tableChildren:
roleConcepts |= roleUgtConcepts(_tableChild)
return roleConcepts
topicMatches = {} # topicNum: (best score, roleType)
for roleDefinition, roleType in sortedRoleTypes:
roleTopicType = 'S' if roleDefinition.startswith('S') else 'D'
if getattr(roleType, "_tableParent", None) is None:
# rooted tables in reverse order
concepts = roleUgtConcepts(roleType)
for i, ugtTopic in enumerate(UGT_TOPICS):
if ugtTopic[0] == roleTopicType:
countAbstracts = len(concepts & ugtTopic[6])
countTextBlocks = len(concepts & ugtTopic[7])
countLineItems = len(concepts & ugtTopic[8])
if countAbstracts or countTextBlocks or countLineItems:
_score = (10 * countAbstracts +
1000 * countTextBlocks +
countLineItems / len(concepts))
if i not in topicMatches or _score > topicMatches[i][0]:
topicMatches[i] = (_score, roleType)
for topicNum, scoredRoleType in topicMatches.items():
_score, roleType = scoredRoleType
if _score > getattr(roleType, "_tableTopicScore", 0):
ugtTopic = UGT_TOPICS[topicNum]
roleType._tableTopicScore = _score
roleType._tableTopicType = ugtTopic[0]
roleType._tableTopicName = ugtTopic[3]
roleType._tableTopicCode = ugtTopic[4]
# print ("Match score {:.2f} topic {} preGrp {}".format(_score, ugtTopic[3], roleType.definition))
return firstTableLinkroleURI or firstDocumentLinkroleURI # did build _tableIndex attributes
return None
def evaluateTableIndex(modelXbrl):
disclosureSystem = modelXbrl.modelManager.disclosureSystem
if disclosureSystem.EFM:
COVER = "1Cover"
STMTS = "2Financial Statements"
NOTES = "3Notes to Financial Statements"
POLICIES = "4Accounting Policies"
TABLES = "5Notes Tables"
DETAILS = "6Notes Details"
UNCATEG = "7Uncategorized"
roleDefinitionPattern = re.compile(
r"([0-9]+) - (Statement|Disclosure|Schedule|Document) - (.+)")
# build EFM rendering-compatible index
definitionElrs = dict(
(roleType.definition, roleType)
for roleURI in modelXbrl.relationshipSet(
XbrlConst.parentChild).linkRoleUris
for roleType in modelXbrl.roleTypes.get(roleURI, ()))
isRR = any(
ns.startswith("http://xbrl.sec.gov/rr/")
for ns in modelXbrl.namespaceDocs.keys())
tableGroup = None
firstTableLinkroleURI = None
firstDocumentLinkroleURI = None
sortedRoleTypes = sorted(definitionElrs.items(),
key=lambda item: item[0])
for roleDefinition, roleType in sortedRoleTypes:
roleType._tableChildren = []
match = roleDefinitionPattern.match(
roleDefinition) if roleDefinition else None
if not match:
roleType._tableIndex = (UNCATEG, "", roleType.roleURI)
continue
seq, tblType, tblName = match.groups()
if isRR:
tableGroup = COVER
elif not tableGroup:
tableGroup = ("Paren" in tblName and COVER
or tblType == "Statement" and STMTS
or "(Polic" in tblName and NOTES
or "(Table" in tblName and TABLES
or "(Detail" in tblName and DETAILS or COVER)
elif tableGroup == COVER:
tableGroup = (tblType == "Statement" and STMTS
or "Paren" in tblName and COVER
or "(Polic" in tblName and NOTES
or "(Table" in tblName and TABLES
or "(Detail" in tblName and DETAILS or NOTES)
elif tableGroup == STMTS:
tableGroup = ((tblType == "Statement" or "Paren" in tblName)
and STMTS or "(Polic" in tblName and NOTES
or "(Table" in tblName and TABLES
or "(Detail" in tblName and DETAILS or NOTES)
elif tableGroup == NOTES:
tableGroup = ("(Polic" in tblName and POLICIES
or "(Table" in tblName and TABLES
or "(Detail" in tblName and DETAILS
or tblType == "Disclosure" and NOTES or UNCATEG)
elif tableGroup == POLICIES:
tableGroup = ("(Table" in tblName and TABLES
or "(Detail" in tblName and DETAILS
or ("Paren" in tblName or "(Polic" in tblName)
and POLICIES or UNCATEG)
elif tableGroup == TABLES:
tableGroup = ("(Detail" in tblName and DETAILS
or ("Paren" in tblName or "(Table" in tblName)
and TABLES or UNCATEG)
elif tableGroup == DETAILS:
tableGroup = (("Paren" in tblName or "(Detail" in tblName)
and DETAILS or UNCATEG)
else:
tableGroup = UNCATEG
if firstTableLinkroleURI is None and tableGroup == COVER:
firstTableLinkroleURI = roleType.roleURI
if tblType == "Document" and not firstDocumentLinkroleURI:
firstDocumentLinkroleURI = roleType.roleURI
roleType._tableIndex = (tableGroup, seq, tblName)
# flow allocate facts to roles (SEC presentation groups)
if not modelXbrl.qnameDimensionDefaults: # may not have run validatino yet
from arelle import ValidateXbrlDimensions
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
reportedFacts = set(
) # facts which were shown in a higher-numbered ELR table
reportingPeriods = set()
nextEnd = None
deiFact = {}
for conceptName in ("DocumentPeriodEndDate", "DocumentType",
"CurrentFiscalPeriodEndDate"):
for concept in modelXbrl.nameConcepts[conceptName]:
for fact in modelXbrl.factsByQname(concept.qname):
deiFact[conceptName] = fact
if fact.context is not None:
reportingPeriods.add(
(None, fact.context.endDatetime)) # for instant
reportingPeriods.add(
(fact.context.startDatetime,
fact.context.endDatetime)) # for startEnd
nextEnd = fact.context.startDatetime
duration = (fact.context.endDatetime -
fact.context.startDatetime).days + 1
break
if "DocumentType" in deiFact:
fact = deiFact["DocumentType"]
if "-Q" in fact.xValue:
# need quarterly and yr to date durations
endDatetime = fact.context.endDatetime
# if within 2 days of end of month use last day of month
endDatetimeMonth = endDatetime.month
if (endDatetime + timedelta(2)).month != endDatetimeMonth:
# near end of month
endOfMonth = True
while endDatetime.month == endDatetimeMonth:
endDatetime += timedelta(1) # go forward to next month
else:
endOfMonth = False
startYr = endDatetime.year
startMo = endDatetime.month - 3
if startMo <= 0:
startMo += 12
startYr -= 1
startDatetime = datetime(startYr, startMo, endDatetime.day,
endDatetime.hour, endDatetime.minute,
endDatetime.second)
if endOfMonth:
startDatetime -= timedelta(1)
endDatetime -= timedelta(1)
reportingPeriods.add((startDatetime, endDatetime))
duration = 91
# find preceding compatible default context periods
while (nextEnd is not None):
thisEnd = nextEnd
prevMaxStart = thisEnd - timedelta(duration * .9)
prevMinStart = thisEnd - timedelta(duration * 1.1)
nextEnd = None
for cntx in modelXbrl.contexts.values():
if cntx is not None:
if (cntx.isStartEndPeriod and not cntx.qnameDims
and thisEnd == cntx.endDatetime and prevMinStart <=
cntx.startDatetime <= prevMaxStart):
reportingPeriods.add((None, cntx.endDatetime))
reportingPeriods.add(
(cntx.startDatetime, cntx.endDatetime))
nextEnd = cntx.startDatetime
break
elif (cntx.isInstantPeriod and not cntx.qnameDims
and thisEnd == cntx.endDatetime):
reportingPeriods.add((None, cntx.endDatetime))
stmtReportingPeriods = set(reportingPeriods)
sortedRoleTypes.reverse() # now in descending order
for i, roleTypes in enumerate(sortedRoleTypes):
roleDefinition, roleType = roleTypes
# find defined non-default axes in pre hierarchy for table
tableFacts = set()
tableGroup, tableSeq, tableName = roleType._tableIndex
roleURIdims, priItemQNames = EFMlinkRoleURIstructure(
modelXbrl, roleType.roleURI)
for priItemQName in priItemQNames:
for fact in modelXbrl.factsByQname(priItemQName):
cntx = fact.context
# non-explicit dims must be default
if (cntx is not None
and all(dimQn in modelXbrl.qnameDimensionDefaults
for dimQn in (roleURIdims.keys() -
cntx.qnameDims.keys())) and
all(mdlDim.memberQname in roleURIdims[dimQn]
for dimQn, mdlDim in cntx.qnameDims.items()
if dimQn in roleURIdims)):
# the flow-up part, drop
cntxStartDatetime = cntx.startDatetime
cntxEndDatetime = cntx.endDatetime
if (tableGroup != STMTS
or (cntxStartDatetime, cntxEndDatetime)
in stmtReportingPeriods and
(fact not in reportedFacts or all(
dimQn not in cntx.
qnameDims # unspecified dims are all defaulted if reported elsewhere
for dimQn in (cntx.qnameDims.keys() -
roleURIdims.keys())))):
tableFacts.add(fact)
reportedFacts.add(fact)
roleType._tableFacts = tableFacts
# find parent if any
closestParentType = None
closestParentMatchLength = 0
for _parentRoleDefinition, parentRoleType in sortedRoleTypes[i +
1:]:
matchLen = parentNameMatchLen(tableName, parentRoleType)
if matchLen > closestParentMatchLength:
closestParentMatchLength = matchLen
closestParentType = parentRoleType
if closestParentType is not None:
closestParentType._tableChildren.insert(0, roleType)
# remove lesser-matched children if there was a parent match
unmatchedChildRoles = set()
longestChildMatchLen = 0
numChildren = 0
for childRoleType in roleType._tableChildren:
matchLen = parentNameMatchLen(tableName, childRoleType)
if matchLen < closestParentMatchLength:
unmatchedChildRoles.add(childRoleType)
elif matchLen > longestChildMatchLen:
longestChildMatchLen = matchLen
numChildren += 1
if numChildren > 1:
# remove children that don't have the full match pattern length to parent
for childRoleType in roleType._tableChildren:
if (childRoleType not in unmatchedChildRoles
and parentNameMatchLen(tableName, childRoleType) <
longestChildMatchLen):
unmatchedChildRoles.add(childRoleType)
for unmatchedChildRole in unmatchedChildRoles:
roleType._tableChildren.remove(unmatchedChildRole)
for childRoleType in roleType._tableChildren:
childRoleType._tableParent = roleType
unmatchedChildRoles = None # dereference
global UGT_TOPICS
if UGT_TOPICS is None:
try:
from arelle import FileSource
fh = FileSource.openFileStream(
modelXbrl.modelManager.cntlr,
os.path.join(modelXbrl.modelManager.cntlr.configDir,
"ugt-topics.zip/ugt-topics.json"), 'r',
'utf-8')
UGT_TOPICS = json.load(fh)
fh.close()
for topic in UGT_TOPICS:
topic[6] = set(
topic[6]
) # change concept abstracts list into concept abstracts set
topic[7] = set(
topic[7]
) # change concept text blocks list into concept text blocks set
topic[8] = set(
topic[8]
) # change concept names list into concept names set
except Exception as ex:
UGT_TOPICS = None
if UGT_TOPICS is not None:
def roleUgtConcepts(roleType):
roleConcepts = set()
for rel in modelXbrl.relationshipSet(
XbrlConst.parentChild,
roleType.roleURI).modelRelationships:
if rel.toModelObject is not None:
roleConcepts.add(rel.toModelObject.name)
if rel.fromModelObject is not None:
roleConcepts.add(rel.fromModelObject.name)
if hasattr(roleType, "_tableChildren"):
for _tableChild in roleType._tableChildren:
roleConcepts |= roleUgtConcepts(_tableChild)
return roleConcepts
topicMatches = {} # topicNum: (best score, roleType)
for roleDefinition, roleType in sortedRoleTypes:
roleTopicType = 'S' if roleDefinition.startswith('S') else 'D'
if getattr(roleType, "_tableParent", None) is None:
# rooted tables in reverse order
concepts = roleUgtConcepts(roleType)
for i, ugtTopic in enumerate(UGT_TOPICS):
if ugtTopic[0] == roleTopicType:
countAbstracts = len(concepts & ugtTopic[6])
countTextBlocks = len(concepts & ugtTopic[7])
countLineItems = len(concepts & ugtTopic[8])
if countAbstracts or countTextBlocks or countLineItems:
_score = (10 * countAbstracts +
1000 * countTextBlocks +
countLineItems / len(concepts))
if i not in topicMatches or _score > topicMatches[
i][0]:
topicMatches[i] = (_score, roleType)
for topicNum, scoredRoleType in topicMatches.items():
_score, roleType = scoredRoleType
if _score > getattr(roleType, "_tableTopicScore", 0):
ugtTopic = UGT_TOPICS[topicNum]
roleType._tableTopicScore = _score
roleType._tableTopicType = ugtTopic[0]
roleType._tableTopicName = ugtTopic[3]
roleType._tableTopicCode = ugtTopic[4]
# print ("Match score {:.2f} topic {} preGrp {}".format(_score, ugtTopic[3], roleType.definition))
return firstTableLinkroleURI or firstDocumentLinkroleURI # did build _tableIndex attributes
return None
