def loadNonNegativeFacts(modelXbrl):
signwarnings = loadDqc0015signwarningRules(modelXbrl)
concepts = set()
excludedMembers = set()
excludedMemberStrings = set()
excludedAxesMembers = defaultdict(set)
for modelDocument in modelXbrl.urlDocs.values():
ns = modelDocument.targetNamespace # set up non neg lookup by full NS
for abbrNs in (abbreviatedNamespace(ns), abbreviatedWildNamespace(ns)):
nsMatch = False
for exName, exSet, isQName in (("conceptNames", concepts, True),
("excludedMemberNames", excludedMembers, True),
("excludedMemberStrings", excludedMemberStrings, False)):
for localName in signwarnings[exName].get(abbrNs, ()):
exSet.add(qname(ns, localName) if isQName else localName)
nsMatch = True
for localDimName, localMemNames in signwarnings["excludedAxesMembers"].get(abbrNs, EMPTY_DICT).items():
for localMemName in localMemNames:
excludedAxesMembers[qname(ns, localDimName)].add(qname(ns, localMemName) if localMemName != "*" else "*")
nsMatch = True
if nsMatch:
break # use explicit year rules if available, else generic year rules
return attrdict(concepts=concepts,
excludedAxesMembers=excludedAxesMembers,
excludedMembers=excludedMembers,
excludedMemberNamesPattern=re.compile("|".join(excludedMemberStrings), re.IGNORECASE)
if excludedMemberStrings else None)
def save(self, outPath, outBasenameSuffix="", outzipFilePrefix=""):
"""
Save the iXBRL viewer
"""
if isinstance(outPath, io.BytesIO): # zip output stream
# zipfile may be cumulatively added to by inline extraction, EdgarRenderer etc
_outPrefix = outzipFilePrefix + ("/" if outzipFilePrefix
and outzipFilePrefix[-1] not in
("/", "\\") else "")
with zipfile.ZipFile(outPath, "a", zipfile.ZIP_DEFLATED,
True) as zout:
for f in self.files:
self.dts.info("viewer:info",
"Saving in output zip %s" % f.filename)
fout = attrdict(write=lambda s: zout.writestr(
_outPrefix + f.filename, s))
writer = XHTMLSerializer()
writer.serialize(f.xmlDocument, fout)
zout.write(
os.path.join(os.path.dirname(__file__), "viewer", "dist",
"ixbrlviewer.js"),
_outPrefix + "ixbrlviewer.js")
elif os.path.isdir(outPath):
# If output is a directory, write each file in the doc set to that
# directory using its existing filename
for f in self.files:
filename = os.path.join(
outPath,
"{0[0]}{1}{0[1]}".format(os.path.splitext(f.filename),
outBasenameSuffix))
self.dts.info("viewer:info", "Writing %s" % filename)
with open(filename, "wb") as fout:
writer = XHTMLSerializer()
writer.serialize(f.xmlDocument, fout)
else:
if len(self.files) > 1:
self.dts.error(
"viewer:error",
"More than one file in input, but output is not a directory"
)
elif outPath.endswith(os.sep):
# Looks like a directory, but isn't one
self.dts.error("viewer:error",
"Directory %s does not exist" % outPath)
elif not os.path.isdir(os.path.dirname(os.path.abspath(outPath))):
# Directory part of filename doesn't exist
self.dts.error(
"viewer:error", "Directory %s does not exist" %
os.path.dirname(os.path.abspath(outPath)))
else:
self.dts.info("viewer:info", "Writing %s" % outPath)
with open(
"{0[0]}{1}{0[1]}".format(os.path.splitext(outPath),
outBasenameSuffix),
"wb") as fout:
writer = XHTMLSerializer()
writer.serialize(self.files[0].xmlDocument, fout)
def selectPlugin(parent, pluginChoices):
filesource = attrdict(
isRss=False, url="Plug-ins", selection=""
) # emulates a filesource object for the selection return
dialog = DialogOpenArchive(parent, PLUGIN, filesource, pluginChoices, _("File"), _("Select Plug-in Module"))
if dialog and dialog.accepted:
return filesource.selection
return None
def selectPackage(parent, packageChoices):
filesource = attrdict(
isRss=False, url="Packages",
selection="") # emulates a filesource object for the selection return
dialog = DialogOpenArchive(parent, PACKAGE, filesource, packageChoices,
_("Name"), _("Select Package"))
if dialog and dialog.accepted:
return filesource.selection
return None
def selectPlugin(parent, pluginChoices):
filesource = attrdict(
isRss=False, url="Plug-ins",
selection="") # emulates a filesource object for the selection return
dialog = DialogOpenArchive(parent, PLUGIN, filesource, pluginChoices,
_("File"), _("Select Plug-in Module"))
if dialog and dialog.accepted:
return filesource.selection
return None
def selectPackage(parent, packageChoices):
filesource = attrdict(isRss=False, url="Packages", selection="") # emulates a filesource object for the selection return
dialog = DialogOpenArchive(parent,
PACKAGE,
filesource,
packageChoices,
_("Name"),
_("Select Package"))
if dialog and dialog.accepted:
return filesource.selection
return None
def loadCustomAxesReplacements(modelXbrl): # returns match expression, standard patterns
_file = openFileStream(modelXbrl.modelManager.cntlr, resourcesFilePath(modelXbrl.modelManager, "axiswarnings.json"), 'rt', encoding='utf-8')
axiswarnings = json.load(_file) # {localName: date, ...}
_file.close()
standardAxes = {}
matchPattern = []
for i, (standardAxis, customAxisPattern) in enumerate(axiswarnings.items()):
if standardAxis not in ("#", "copyright", "description"):
patternName = "_{}".format(i)
standardAxes[patternName] = standardAxis
matchPattern.append("(?P<{}>^{}$)".format(patternName, customAxisPattern))
return attrdict(standardAxes=standardAxes,
customNamePatterns=re.compile("|".join(matchPattern)))
def loadCustomAxesReplacements(modelXbrl): # returns match expression, standard patterns
_file = openFileStream(modelXbrl.modelManager.cntlr, resourcesFilePath(modelXbrl.modelManager, "axiswarnings.json"), 'rt', encoding='utf-8')
axiswarnings = json.load(_file) # {localName: date, ...}
_file.close()
standardAxes = {}
matchPattern = []
for i, (standardAxis, customAxisPattern) in enumerate(axiswarnings.items()):
if standardAxis not in ("#", "copyright", "description"):
patternName = "_{}".format(i)
standardAxes[patternName] = standardAxis
matchPattern.append("(?P<{}>^{}$)".format(patternName, customAxisPattern))
return attrdict(standardAxes=standardAxes,
customNamePatterns=re.compile("|".join(matchPattern)))
def loadNonNegativeFacts(modelXbrl, dqcRules, ugtRels):
# for us-gaap newer than 2020 use DQCRT non-negative facts.
if dqcRules and ugtRels: # not used before 2020
if usgaapYear(
modelXbrl) == "2020" and "dqcrt-2021-usgaap-2020" not in (
modelXbrl.modelManager.disclosureSystem.options or ""):
dqcRules.clear() # remove dqc rules
return ugtRels[
"DQC.US.0015"] # use 20.1 2020 nonNegFacts test and warning
return None # use all available DQCRT tests
# for us-gaap < dqcyear use EFM non-negative warning insead of DQC rule
_file = openFileStream(modelXbrl.modelManager.cntlr,
resourcesFilePath(modelXbrl.modelManager,
"signwarnings.json"),
'rt',
encoding='utf-8')
signwarnings = json.load(_file) # {localName: date, ...}
_file.close()
concepts = set()
excludedMembers = set()
excludedMemberStrings = set()
excludedAxesMembers = defaultdict(set)
for modelDocument in modelXbrl.urlDocs.values():
ns = modelDocument.targetNamespace # set up non neg lookup by full NS
for abbrNs in (abbreviatedNamespace(ns),
abbreviatedNamespace(ns, WILD)):
nsMatch = False
for exName, exSet, isQName in (("conceptNames", concepts,
True), ("excludedMemberNames",
excludedMembers, True),
("excludedMemberStrings",
excludedMemberStrings, False)):
for localName in signwarnings[exName].get(abbrNs, ()):
exSet.add(qname(ns, localName) if isQName else localName)
nsMatch = True
for localDimName, localMemNames in signwarnings[
"excludedAxesMembers"].get(abbrNs, EMPTY_DICT).items():
for localMemName in localMemNames:
excludedAxesMembers[qname(ns, localDimName)].add(
qname(ns, localMemName) if localMemName != "*" else "*"
)
nsMatch = True
if nsMatch:
break # use explicit year rules if available, else generic year rules
return attrdict(concepts=concepts,
excludedAxesMembers=excludedAxesMembers,
excludedMembers=excludedMembers,
excludedMemberNamesPattern=re.compile(
"|".join(excludedMemberStrings), re.IGNORECASE)
if excludedMemberStrings else None)
def loadNonNegativeFacts(modelXbrl):
_file = openFileStream(modelXbrl.modelManager.cntlr, resourcesFilePath(modelXbrl.modelManager, "signwarnings.json"), 'rt', encoding='utf-8')
signwarnings = json.load(_file) # {localName: date, ...}
_file.close()
concepts = set()
excludedAxesMembers = defaultdict(set)
for modelDocument in modelXbrl.urlDocs.values():
ns = modelDocument.targetNamespace # set up non neg lookup by full NS
for abbrNs in (abbreviatedNamespace(ns), abbreviatedWildNamespace(ns)):
for localName in signwarnings["conceptNames"].get(abbrNs, ()):
concepts.add(qname(ns, localName))
for localDimName, localMemNames in signwarnings["excludedAxesMembers"].get(abbrNs, EMPTY_DICT).items():
for localMemName in localMemNames:
excludedAxesMembers[qname(ns, localDimName)].add(qname(ns, localMemName) if localMemName != "*" else "*")
return attrdict(concepts=concepts, excludedAxesMembers=excludedAxesMembers)
def loadNonNegativeFacts(modelXbrl):
_file = openFileStream(modelXbrl.modelManager.cntlr, resourcesFilePath(modelXbrl.modelManager, "signwarnings.json"), 'rt', encoding='utf-8')
signwarnings = json.load(_file) # {localName: date, ...}
_file.close()
concepts = set()
excludedAxesMembers = defaultdict(set)
for modelDocument in modelXbrl.urlDocs.values():
ns = modelDocument.targetNamespace # set up non neg lookup by full NS
for abbrNs in (abbreviatedNamespace(ns), abbreviatedWildNamespace(ns)):
for localName in signwarnings["conceptNames"].get(abbrNs, ()):
concepts.add(qname(ns, localName))
for localDimName, localMemNames in signwarnings["excludedAxesMembers"].get(abbrNs, EMPTY_DICT).items():
for localMemName in localMemNames:
excludedAxesMembers[qname(ns, localDimName)].add(qname(ns, localMemName) if localMemName != "*" else None)
return attrdict(concepts=concepts, excludedAxesMembers=excludedAxesMembers)
def loadNonNegativeFacts(modelXbrl):
_file = openFileStream(modelXbrl.modelManager.cntlr,
resourcesFilePath(modelXbrl.modelManager,
"signwarnings.json"),
'rt',
encoding='utf-8')
signwarnings = json.load(_file) # {localName: date, ...}
_file.close()
concepts = set()
excludedMembers = set()
excludedMemberStrings = set()
excludedAxesMembers = defaultdict(set)
for modelDocument in modelXbrl.urlDocs.values():
ns = modelDocument.targetNamespace # set up non neg lookup by full NS
for abbrNs in (abbreviatedNamespace(ns),
abbreviatedNamespace(ns, WILD)):
nsMatch = False
for exName, exSet, isQName in (("conceptNames", concepts,
True), ("excludedMemberNames",
excludedMembers, True),
("excludedMemberStrings",
excludedMemberStrings, False)):
for localName in signwarnings[exName].get(abbrNs, ()):
exSet.add(qname(ns, localName) if isQName else localName)
nsMatch = True
for localDimName, localMemNames in signwarnings[
"excludedAxesMembers"].get(abbrNs, EMPTY_DICT).items():
for localMemName in localMemNames:
excludedAxesMembers[qname(ns, localDimName)].add(
qname(ns, localMemName) if localMemName != "*" else "*"
)
nsMatch = True
if nsMatch:
break # use explicit year rules if available, else generic year rules
return attrdict(concepts=concepts,
excludedAxesMembers=excludedAxesMembers,
excludedMembers=excludedMembers,
excludedMemberNamesPattern=re.compile(
"|".join(excludedMemberStrings), re.IGNORECASE)
if excludedMemberStrings else None)
def testcaseVariationArchiveIxds(val, filesource, entrypointFiles):
commandLineFilingStart(val.modelXbrl.modelManager.cntlr,
attrdict(skipExpectedInstanceComparison=True),
filesource, entrypointFiles)
def loadUgtRelQnames(modelXbrl, dqcRules):
if not dqcRules:
return {} # not a us-gaap filing
abbrNs = ""
for modelDocument in modelXbrl.urlDocs.values():
abbrNs = abbreviatedNamespace(modelDocument.targetNamespace)
if abbrNs and abbrNs.startswith("us-gaap/"):
break
if not abbrNs: # no gaap/ifrs taxonomy for this filing
return {}
_ugtRelsFileName = resourcesFilePath(
modelXbrl.modelManager,
"us-gaap-rels-{}.json".format(abbrNs.rpartition("/")[2]))
if not os.path.exists(_ugtRelsFileName):
buildUgtFullRelsFiles(modelXbrl, dqcRules)
if not os.path.exists(_ugtRelsFileName):
return {}
_file = openFileStream(modelXbrl.modelManager.cntlr,
_ugtRelsFileName,
'rt',
encoding='utf-8')
ugtRels = json.load(_file) # {localName: date, ...}
_file.close()
def qn(nsPrefix, localName):
return qname(nsPrefix + ":" + localName, modelXbrl.prefixedNamespaces)
ugtCalcsByQnames = defaultdict(
dict
) # store as concept indices to avoid using memory for repetitive strings
for wgt, fromNSes in ugtRels["calcs"].items():
calcWgtObj = ugtCalcsByQnames.setdefault(
float(wgt), {}) # json weight object needs to be float
for fromNs, fromObjs in fromNSes.items():
for fromName, toNSes in fromObjs.items():
fromConcept = modelXbrl.qnameConcepts.get(qn(fromNs, fromName))
if fromConcept is not None:
calcFromObj = calcWgtObj.setdefault(
fromConcept.qname, set())
for toNs, toNames in toNSes.items():
for toName in toNames:
toConcept = modelXbrl.qnameConcepts.get(
qn(toNs, toName))
if toConcept is not None:
calcFromObj.add(toConcept.qname)
ugtAxesByQnames = defaultdict(
set
) # store as concept indices to avoid using memory for repetitive strings
for axisName, memNames in ugtRels["axes"].items():
for axisConcept in modelXbrl.nameConcepts.get(axisName, ()):
axisObj = ugtAxesByQnames[axisConcept.name]
for memName in memNames:
for memConcept in modelXbrl.nameConcepts.get(memName, ()):
axisObj.add(memConcept.qname)
ugt = {"calcs": ugtCalcsByQnames, "axes": ugtAxesByQnames}
# dqc0015
if "DQC.US.0015" in ugtRels:
dqc0015 = ugtRels["DQC.US.0015"]
concepts = set()
excludedMembers = set()
excludedMemberStrings = set()
excludedAxesMembers = defaultdict(set)
conceptRuleIDs = {}
for exName, exSet, isQName in (("conceptNames", concepts, True),
("excludedMemberNames", excludedMembers,
True), ("excludedMemberStrings",
excludedMemberStrings, False)):
for ns, names in dqc0015[exName].items():
for localName in names:
exSet.add(qn(ns, localName) if isQName else localName)
for localDimNs, localDimMems in dqc0015["excludedAxesMembers"].items():
for localDimName, localMemObjs in localDimMems.items():
for localMemNs, localMemNames in localMemObjs.items():
if localMemNs == "*":
excludedAxesMembers[qn(localDimNs,
localDimName)].add("*")
else:
for localMemName in localMemNames:
excludedAxesMembers[qn(
localDimNs, localDimName)].add(
qn(localMemNs, localMemName
) if localMemName != "*" else "*")
#if abbrNs < "us-gaap/2021": # no rel ids in us-gaap/2020
# _ugtRelsFileName = resourcesFilePath(modelXbrl.modelManager, "us-gaap-rels-2021.json")
# _file = openFileStream(modelXbrl.modelManager.cntlr, _ugtRelsFileName, 'rt', encoding='utf-8')
# ugtRels = json.load(_file) # {localName: date, ...}
# _file.close()
for conceptNs, conceptNameIDs in ugtRels["DQC.US.0015"][
"conceptRuleIDs"].items():
for conceptName, conceptID in conceptNameIDs.items():
conceptRuleIDs[qn(conceptNs, conceptName)] = conceptID
ugt["DQC.US.0015"] = attrdict(
concepts=concepts,
excludedAxesMembers=excludedAxesMembers,
excludedMembers=excludedMembers,
excludedMemberNamesPattern=re.compile(
"|".join(excludedMemberStrings), re.IGNORECASE)
if excludedMemberStrings else None,
conceptRuleIDs=conceptRuleIDs)
return ugt
"us-gaap/2021": [
"http://xbrl.fasb.org/us-gaap/2021/entire/us-gaap-entryPoint-std-2021-01-31.xsd",
# "http://xbrl.fasb.org/us-gaap/2021/dqcrules/dqcrules-2021-01-31.xsd"
"https://xbrl.fasb.org/us-gaap/2022/dqcrules/dqcrules-entire-2022.xsd"
],
"us-gaap/2022": [
"https://xbrl.fasb.org/us-gaap/2022/entire/us-gaap-entryPoint-std-2022.xsd",
"https://xbrl.fasb.org/us-gaap/2022/dqcrules/dqcrules-entire-2022.xsd"
]
}
linkbaseValidations = {
"cef":
attrdict(efmPre="6.12.10",
efmCal="6.14.06",
efmDef="6.16.10",
elrPre=re.compile("http://xbrl.sec.gov/cef/role/N2"),
elrDefInNs=re.compile("http://xbrl.sec.gov/cef/role/N2"),
elrDefExNs=re.compile(
"http://xbrl.sec.gov/cef/role/(Security|Risk)Only"),
preSources=("AllSecuritiesMember", "AllRisksMember")),
"vip":
attrdict(efmPre="6.12.11",
efmCal="6.14.07",
efmDef="6.16.11",
elrPre=re.compile("http://xbrl.sec.gov/vip/role/N[346]"),
elrDefInNs=re.compile("http://xbrl.sec.gov/vip/role/.*Only"),
elrDefExNs=re.compile("http://xbrl.sec.gov/vip/role/.*Only"),
preSources=())
}
