def backgroundProfileFormula(cntlr, profileReportFile, maxRunTime, excludeCompileTime):
from arelle import Locale, XPathParser, ValidateXbrlDimensions, ValidateFormula
# build grammar before profiling (if this is the first pass, so it doesn't count in profile statistics)
XPathParser.initializeParser(cntlr.modelManager)
# load dimension defaults
ValidateXbrlDimensions.loadDimensionDefaults(cntlr.modelManager)
import cProfile, pstats, sys, time
# a minimal validation class for formula validator parameters that are needed
class Validate:
def __init__(self, modelXbrl, maxRunTime):
self.modelXbrl = modelXbrl
self.parameters = None
self.validateSBRNL = False
self.maxFormulaRunTime = maxRunTime
def close(self):
self.__dict__.clear()
val = Validate(cntlr.modelManager.modelXbrl, maxRunTime)
formulaOptions = val.modelXbrl.modelManager.formulaOptions
if excludeCompileTime:
startedAt = time.time()
cntlr.addToLog(_("pre-compiling formulas before profiling"))
val.validateFormulaCompileOnly = True
ValidateFormula.validate(val)
del val.validateFormulaCompileOnly
cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale,
_("formula pre-compiling completed in %.2f secs"),
time.time() - startedAt))
cntlr.addToLog(_("executing formulas for profiling"))
else:
cntlr.addToLog(_("compiling and executing formulas for profiling"))
startedAt = time.time()
statsFile = profileReportFile + ".bin"
cProfile.runctx("ValidateFormula.validate(val)", globals(), locals(), statsFile)
cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale,
_("formula profiling completed in %.2f secs"),
time.time() - startedAt))
# dereference val
val.close()
# specify a file for log
priorStdOut = sys.stdout
sys.stdout = open(profileReportFile, "w")
statObj = pstats.Stats(statsFile)
statObj.strip_dirs()
statObj.sort_stats("time")
statObj.print_stats()
statObj.print_callees()
statObj.print_callers()
sys.stdout.flush()
sys.stdout.close()
del statObj
sys.stdout = priorStdOut
os.remove(statsFile)
def backgroundProfileFormula(cntlr, profileReportFile, maxRunTime, excludeCompileTime):
from arelle import Locale, XPathParser, ValidateXbrlDimensions, ValidateFormula
# build grammar before profiling (if this is the first pass, so it doesn't count in profile statistics)
XPathParser.initializeParser(cntlr.modelManager)
# load dimension defaults
ValidateXbrlDimensions.loadDimensionDefaults(cntlr.modelManager)
import cProfile, pstats, sys, time
# a minimal validation class for formula validator parameters that are needed
class Validate:
def __init__(self, modelXbrl, maxRunTime):
self.modelXbrl = modelXbrl
self.parameters = None
self.validateSBRNL = False
self.maxFormulaRunTime = maxRunTime
def close(self):
self.__dict__.clear()
val = Validate(cntlr.modelManager.modelXbrl, maxRunTime)
formulaOptions = val.modelXbrl.modelManager.formulaOptions
if excludeCompileTime:
startedAt = time.time()
cntlr.addToLog(_("pre-compiling formulas before profiling"))
val.validateFormulaCompileOnly = True
ValidateFormula.validate(val)
del val.validateFormulaCompileOnly
cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale,
_("formula pre-compiling completed in %.2f secs"),
time.time() - startedAt))
cntlr.addToLog(_("executing formulas for profiling"))
else:
cntlr.addToLog(_("compiling and executing formulas for profiling"))
startedAt = time.time()
statsFile = profileReportFile + ".bin"
cProfile.runctx("ValidateFormula.validate(val)", globals(), locals(), statsFile)
cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale,
_("formula profiling completed in %.2f secs"),
time.time() - startedAt))
# dereference val
val.close()
# specify a file for log
priorStdOut = sys.stdout
sys.stdout = open(profileReportFile, "w")
statObj = pstats.Stats(statsFile)
statObj.strip_dirs()
statObj.sort_stats("time")
statObj.print_stats()
statObj.print_callees()
statObj.print_callers()
sys.stdout.flush()
sys.stdout.close()
del statObj
sys.stdout = priorStdOut
os.remove(statsFile)
def formatInterval(self, a, b, dec):
if a is NIL:
return "(nil)"
if isnan(dec) or isinf(dec): dec = 4
if a == b: # not an interval
return Locale.format_decimal(self.modelXbrl.locale, a, 1, max(dec,0))
return "[{}, {}]".format( # show as an interval
Locale.format_decimal(self.modelXbrl.locale, a, 1, max(dec,0)),
Locale.format_decimal(self.modelXbrl.locale, b, 1, max(dec,0)))
def formatInterval(self, a, b, dec):
if a is NIL:
return "(nil)"
if isnan(dec) or isinf(dec): dec = 4
if a == b: # not an interval
return Locale.format_decimal(self.modelXbrl.locale, a, 1,
max(dec, 0))
return "[{}, {}]".format( # show as an interval
Locale.format_decimal(self.modelXbrl.locale, a, 1, max(dec, 0)),
Locale.format_decimal(self.modelXbrl.locale, b, 1, max(dec, 0)))
def __init__(self, cntlr):
self.cntlr = cntlr
self.validateDisclosureSystem = False
self.disclosureSystem = DisclosureSystem.DisclosureSystem(self)
self.validateCalcLB = False
self.validateInferDecimals = False
self.validateUtr = False
self.loadedModelXbrls = []
from arelle import Locale
self.locale = Locale.getUserLocale()
self.defaultLang = Locale.getLanguageCode()
def __init__(self, cntlr):
self.cntlr = cntlr
self.validateDisclosureSystem = False
self.disclosureSystem = DisclosureSystem.DisclosureSystem(self)
self.validateCalcLB = False
self.validateInferDecimals = False
self.validateUtr = False
self.loadedModelXbrls = []
from arelle import Locale
self.locale = Locale.getUserLocale()
self.defaultLang = Locale.getLanguageCode()
def __init__(self, cntlr):
self.cntlr = cntlr
self.validateDisclosureSystem = False
self.disclosureSystem = DisclosureSystem.DisclosureSystem(self)
self.validateCalcLB = False
self.validateInferDecimals = False
self.validateInfoset = False
self.validateUtr = False
self.abortOnMajorError = False
self.collectProfileStats = False
self.loadedModelXbrls = []
from arelle import Locale
self.locale = Locale.getUserLocale(cntlr.config.get("userInterfaceLocaleOverride",""))
self.defaultLang = Locale.getLanguageCode()
def __init__(self, cntlr):
self.cntlr = cntlr
self.validateDisclosureSystem = False
self.disclosureSystem = DisclosureSystem.DisclosureSystem(self)
self.validateCalcLB = False
self.validateInferDecimals = False
self.validateInfoset = False
self.validateUtr = False
self.skipDTS = False
self.abortOnMajorError = False
self.collectProfileStats = False
self.loadedModelXbrls = []
from arelle import Locale
self.locale = Locale.getUserLocale(cntlr.config.get("userInterfaceLocaleOverride",""))
self.defaultLang = Locale.getLanguageCode()
def profilerCommandLineRun(cntlr, options, sourceZipStream=None, **kwargs):
from arelle import Locale
import cProfile, pstats, sys, time
profileReportFile = getattr(options, "profilerReportFile", None)
if profileReportFile and not getattr(cntlr, "blockNestedProfiling", False):
startedAt = time.time()
cntlr.addToLog(_("invoking command processing under profiler"))
statsFile = profileReportFile + ".bin"
cntlr.blockNestedProfiling = True
cProfile.runctx("cntlr.run(options, sourceZipStream)", globals(), locals(), statsFile)
cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale,
_("profiled command processing completed in %.2f secs"),
time.time() - startedAt))
# specify a file for log
priorStdOut = sys.stdout
sys.stdout = open(profileReportFile, "w")
statObj = pstats.Stats(statsFile)
statObj.strip_dirs()
statObj.sort_stats("time")
statObj.print_stats()
statObj.print_callees()
statObj.print_callers()
sys.stdout.flush()
sys.stdout.close()
del statObj
sys.stdout = priorStdOut
os.remove(statsFile)
del cntlr.blockNestedProfiling
sys.exit() # raise SYSTEM_EXIT to stop outer execution
def profilerCommandLineRun(cntlr, options, sourceZipStream=None, **kwargs):
from arelle import Locale
import cProfile, pstats, sys, time
profileReportFile = getattr(options, "profilerReportFile", None)
if profileReportFile and not getattr(cntlr, "blockNestedProfiling", False):
startedAt = time.time()
cntlr.addToLog(_("invoking command processing under profiler"))
statsFile = profileReportFile + ".bin"
cntlr.blockNestedProfiling = True
cProfile.runctx("cntlr.run(options, sourceZipStream)", globals(),
locals(), statsFile)
cntlr.addToLog(
Locale.format_string(
cntlr.modelManager.locale,
_("profiled command processing completed in %.2f secs"),
time.time() - startedAt))
# specify a file for log
priorStdOut = sys.stdout
sys.stdout = open(profileReportFile, "w")
statObj = pstats.Stats(statsFile)
statObj.strip_dirs()
statObj.sort_stats("time")
statObj.print_stats()
statObj.print_callees()
statObj.print_callers()
sys.stdout.flush()
sys.stdout.close()
del statObj
sys.stdout = priorStdOut
os.remove(statsFile)
del cntlr.blockNestedProfiling
sys.exit() # raise SYSTEM_EXIT to stop outer execution
def parsePackage(mainWin, metadataFile):
unNamedCounter = 1
NS = "{http://www.corefiling.com/xbrl/taxonomypackage/v1}"
pkg = {}
currentLang = Locale.getLanguageCode()
tree = etree.parse(metadataFile)
root = tree.getroot()
for eltName in ("name", "description", "version"):
pkg[eltName] = ""
for m in root.iterchildren(tag=NS + eltName):
pkg[eltName] = m.text.strip()
break # take first entry if several
remappings = dict((m.get("prefix"), m.get("replaceWith")) for m in tree.iter(tag=NS + "remapping"))
pkg["remappings"] = remappings
nameToUrls = {}
pkg["nameToUrls"] = nameToUrls
for entryPointSpec in tree.iter(tag=NS + "entryPoint"):
name = None
# find closest match name node given xml:lang match to current language or no xml:lang
for nameNode in entryPointSpec.iter(tag=NS + "name"):
xmlLang = nameNode.get("{http://www.w3.org/XML/1998/namespace}lang")
if name is None or not xmlLang or currentLang == xmlLang:
name = nameNode.text
if currentLang == xmlLang: # most prefer one with the current locale's language
break
if not name:
name = _("<unnamed {0}>").format(unNamedCounter)
unNamedCounter += 1
epDocCount = 0
for epDoc in entryPointSpec.iterchildren(NS + "entryPointDocument"):
if epDocCount:
mainWin.addToLog(_("WARNING: skipping multiple-document entry point (not supported)"))
continue
epDocCount += 1
epUrl = epDoc.get("href")
base = epDoc.get("{http://www.w3.org/XML/1998/namespace}base") # cope with xml:base
if base:
resolvedUrl = urljoin(base, epUrl)
else:
resolvedUrl = epUrl
# perform prefix remappings
remappedUrl = resolvedUrl
for prefix, replace in remappings.items():
remappedUrl = remappedUrl.replace(prefix, replace, 1)
nameToUrls[name] = (remappedUrl, resolvedUrl)
return pkg
def parseTxmyPkg(mainWin, metadataFile):
unNamedCounter = 1
currentLang = Locale.getLanguageCode()
tree = etree.parse(metadataFile)
remappings = dict(
(m.get("prefix"), m.get("replaceWith")) for m in tree.iter(
tag="{http://www.corefiling.com/xbrl/taxonomypackage/v1}remapping")
)
result = {}
for entryPointSpec in tree.iter(
tag="{http://www.corefiling.com/xbrl/taxonomypackage/v1}entryPoint"
):
name = None
# find closest match name node given xml:lang match to current language or no xml:lang
for nameNode in entryPointSpec.iter(
tag="{http://www.corefiling.com/xbrl/taxonomypackage/v1}name"):
xmlLang = nameNode.get(
'{http://www.w3.org/XML/1998/namespace}lang')
if name is None or not xmlLang or currentLang == xmlLang:
name = nameNode.text
if currentLang == xmlLang: # most prefer one with the current locale's language
break
if not name:
name = _("<unnamed {0}>").format(unNamedCounter)
unNamedCounter += 1
epDocCount = 0
for epDoc in entryPointSpec.iterchildren(
"{http://www.corefiling.com/xbrl/taxonomypackage/v1}entryPointDocument"
):
if epDocCount:
mainWin.addToLog(
_("WARNING: skipping multiple-document entry point (not supported)"
))
continue
epDocCount += 1
epUrl = epDoc.get('href')
base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base'
) # cope with xml:base
if base:
resolvedUrl = urljoin(base, epUrl)
else:
resolvedUrl = epUrl
#perform prefix remappings
remappedUrl = resolvedUrl
for prefix, replace in remappings.items():
remappedUrl = resolvedUrl.replace(prefix, replace, 1)
result[name] = (remappedUrl, resolvedUrl)
return result
def factCheck(val, fact):
concept = fact.concept
context = fact.context
if concept is None or context is None:
return # not checkable
try:
if fact.isNumeric and not fact.isNil and fact.xValue is not None:
# 2.4.1 decimal disagreement
if fact.decimals and fact.decimals != "INF":
vf = float(fact.value)
if _ISFINITE(vf):
dec = _INT(fact.decimals)
vround = round(vf, dec)
if vf != vround:
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.4.1",
_("Decimal disagreement %(fact)s in context %(contextID)s unit %(unitID)s value %(value)s has insignificant value %(insignificantValue)s"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID,
value=fact.effectiveValue, insignificantValue=Locale.format(val.modelXbrl.locale, "%.*f",
(dec + 2 if dec > 0 else 0, vf - vround),
True))
# 2.5.1 fractions disallowed on a disclosure
if fact.isFraction:
if any(val.linroleDefinitionIsDisclosure.match(roleType.definition)
for rel in val.modelXbrl.relationshipSet(XbrlConst.parentChild).toModelObject(concept)
for roleType in val.modelXbrl.roleTypes.get(rel.linkrole,())):
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.5.1",
_("Disclosure %(fact)s in context %(contextID)s value %(value)s is a fraction"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, value=fact.value)
# deprecated concept
if concept.qname.namespaceURI == ugtNamespace:
if concept.name in val.usgaapDeprecations:
val.deprecatedFactConcepts[concept].append(fact)
elif concept.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedFactConcepts[concept].append(fact)
if fact.isItem and fact.context is not None:
for dimConcept, modelDim in fact.context.segDimValues.items():
if dimConcept.qname.namespaceURI == ugtNamespace:
if dimConcept.name in val.usgaapDeprecations:
val.deprecatedDimensions[dimConcept].append(fact)
elif dimConcept.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedDimensions[dimConcept].append(fact)
if modelDim.isExplicit:
member = modelDim.member
if member is not None:
if member.qname.namespaceURI == ugtNamespace:
if member.name in val.usgaapDeprecations:
val.deprecatedMembers[member].append(fact)
elif member.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedMembers[member].append(fact)
except Exception as err:
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.testingException",
_("%(fact)s in context %(contextID)s unit %(unitID)s value %(value)s cannot be tested due to: %(err)s"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID,
value=fact.effectiveValue, err=err)
def label(self,preferredLabel=None,fallbackToQname=True,lang=None,strip=False,linkrole=None):
if preferredLabel is None: preferredLabel = XbrlConst.standardLabel
if preferredLabel == XbrlConst.conceptNameLabelRole: return str(self.qname)
labelsRelationshipSet = self.modelXbrl.relationshipSet(XbrlConst.conceptLabel,linkrole)
if labelsRelationshipSet:
label = labelsRelationshipSet.label(self, preferredLabel, lang)
if label is not None:
if strip: return label.strip()
return Locale.rtlString(label, lang=lang)
return str(self.qname) if fallbackToQname else None
def parseTxmyPkg(mainWin, metadataFile):
unNamedCounter = 1
currentLang = Locale.getLanguageCode()
tree = etree.parse(metadataFile)
remappings = dict(
(m.get("prefix"), m.get("replaceWith"))
for m in tree.iter(tag="{http://www.corefiling.com/xbrl/taxonomypackage/v1}remapping")
)
result = {}
for entryPointSpec in tree.iter(tag="{http://www.corefiling.com/xbrl/taxonomypackage/v1}entryPoint"):
name = None
# find closest match name node given xml:lang match to current language or no xml:lang
for nameNode in entryPointSpec.iter(tag="{http://www.corefiling.com/xbrl/taxonomypackage/v1}name"):
xmlLang = nameNode.get("{http://www.w3.org/XML/1998/namespace}lang")
if name is None or not xmlLang or currentLang == xmlLang:
name = nameNode.text
if currentLang == xmlLang: # most prefer one with the current locale's language
break
if not name:
name = _("<unnamed {0}>").format(unNamedCounter)
unNamedCounter += 1
epDocCount = 0
for epDoc in entryPointSpec.iterchildren(
"{http://www.corefiling.com/xbrl/taxonomypackage/v1}entryPointDocument"
):
if epDocCount:
mainWin.addToLog(_("WARNING: skipping multiple-document entry point (not supported)"))
continue
epDocCount += 1
epUrl = epDoc.get("href")
base = epDoc.get("{http://www.w3.org/XML/1998/namespace}base") # cope with xml:base
if base:
resolvedUrl = urljoin(base, epUrl)
else:
resolvedUrl = epUrl
# perform prefix remappings
remappedUrl = resolvedUrl
for prefix, replace in remappings.items():
remappedUrl = resolvedUrl.replace(prefix, replace, 1)
result[name] = (remappedUrl, resolvedUrl)
return result
def genLabel(self,role=None,fallbackToQname=False,fallbackToXlinkLabel=False,lang=None,strip=False,linkrole=None, linkroleHint=None):
from arelle import XbrlConst
if role is None: role = XbrlConst.genStandardLabel
if role == XbrlConst.conceptNameLabelRole: return str(self.qname)
labelsRelationshipSet = self.modelXbrl.relationshipSet(XbrlConst.elementLabel,linkrole)
if labelsRelationshipSet:
label = labelsRelationshipSet.label(self, linkroleHint or role, lang)
if label is not None:
if strip: return label.strip()
return Locale.rtlString(label, lang=lang)
if fallbackToQname:
return str(self.qname)
elif fallbackToXlinkLabel and hasattr(self,"xlinkLabel"):
return self.xlinkLabel
else:
return None
def label(self,
preferredLabel=None,
fallbackToQname=True,
lang=None,
strip=False,
linkrole=None):
if preferredLabel is None: preferredLabel = XbrlConst.standardLabel
if preferredLabel == XbrlConst.conceptNameLabelRole:
return str(self.qname)
labelsRelationshipSet = self.modelXbrl.relationshipSet(
XbrlConst.conceptLabel, linkrole)
if labelsRelationshipSet:
label = labelsRelationshipSet.label(self, preferredLabel, lang)
if label is not None:
if strip: return label.strip()
return Locale.rtlString(label, lang=lang)
return str(self.qname) if fallbackToQname else None
def effectiveValue(self):
concept = self.concept
if concept is None or concept.isTuple:
return None
if self.isNil:
return "(nil)"
if concept.isNumeric:
val = self.value
try:
num = float(val)
dec = self.decimals
if dec is None or dec == "INF":
dec = len(val.partition(".")[2])
else:
dec = int(dec) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
return Locale.format(self.modelXbrl.locale, "%.*f", (dec, num), True)
except ValueError:
return "(error)"
return self.value
def effectiveValue(self):
concept = self.concept
if concept is None or concept.isTuple:
return None
if self.isNil:
return "(nil)"
if concept.isNumeric:
val = self.value
try:
num = float(val)
dec = self.decimals
if dec is None or dec == "INF":
dec = len(val.partition(".")[2])
else:
dec = int(
dec
) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
return Locale.format(self.modelXbrl.locale, "%.*f", (dec, num),
True)
except ValueError:
return "(error)"
return self.value
def effectiveValue(self):
"""(str) -- Effective value for views, (nil) if isNil, None if no value,
locale-formatted string of decimal value (if decimals specified) , otherwise string value"""
concept = self.concept
if concept is None or concept.isTuple:
return None
if self.isNil:
return "(nil)"
try:
if concept.isNumeric:
val = self.value
try:
num = float(val)
dec = self.decimals
if dec is None or dec == "INF":
dec = len(val.partition(".")[2])
else:
dec = int(dec) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
return Locale.format(self.modelXbrl.locale, "%.*f", (dec, num), True)
except ValueError:
return "(error)"
return self.value
except Exception as ex:
return str(ex) # could be transform value of inline fact
def genFact(dts, concept, preferredLabel, arcrole, relationshipSet, level, visited, elrInfo):
try:
if concept is not None:
if concept.isHypercubeItem:
elrInfo["inCube"] = level
elrInfo["dims"] = {}
elrInfo["lineItems"] =False
elrInfo.pop("instant", None)
elrInfo.pop("duration", None)
elif concept.isDimensionItem:
elrInfo["currentDim"] = concept
if concept.isTypedDimension:
elrInfo["dims"][concept.qname] = (concept, concept.typedDomainElement)
if concept.typedDomainElement.isNumeric:
elrInfo["domainIter"] = 1
elif concept.name.endswith("Member"): # don't generate entries for default dim (Domain) (for now)
dimConcept = elrInfo["currentDim"]
if dimConcept.qname not in elrInfo["dims"]:
elrInfo["dims"][dimConcept.qname] = (dimConcept, concept)
else:
if concept.name.endswith("LineItems"):
elrInfo["lineItems"] = True
elif ((not elrInfo["inCube"] or # before any hypercube
elrInfo["lineItems"]) # in Cube and within Line Items
and not concept.isAbstract): # or within line items
# generate a fact
sampVals = sampleDataValues[elrInfo.get("domainIter",1)] # use first entry if no domain iter
if concept.periodType not in elrInfo:
qnameDims = {}
for _dimConcept, _domConcept in elrInfo["dims"].values():
if _dimConcept.isExplicitDimension:
_memVal = _domConcept.qname
else:
if _domConcept.type is not None and not _domConcept.isNumeric:
_memEltVal = genSampleValue(sampVals, _domConcept)
else:
_memEltVal = str(elrInfo["domainIter"])
_memVal = XmlUtil.addChild(dts.modelDocument.xmlRootElement,
_domConcept.qname,
text=_memEltVal,
appendChild=False)
_dimObj = DimValuePrototype(dts, None, _dimConcept.qname, _memVal, "segment")
qnameDims[_dimConcept.qname] = _dimObj
elrInfo[concept.periodType] = dts.createContext(
"http://treasury.gov", "entityId",
concept.periodType,
sampVals["periodStart"] if concept.periodType == "duration"
else None,
sampVals["periodEnd"],
concept.qname, qnameDims, [], [])
cntx = elrInfo[concept.periodType]
cntxId = cntx.id
if concept.isNumeric:
if concept.isMonetary:
unitMeasure = qname(XbrlConst.iso4217, "USD")
unitMeasure.prefix = "iso4217" # want to save with a recommended prefix
decimals = 2
elif concept.isShares:
unitMeasure = XbrlConst.qnXbrliShares
decimals = 0
else:
unitMeasure = XbrlConst.qnXbrliPure
decimals = 0
prevUnit = dts.matchUnit([unitMeasure], [])
if prevUnit is not None:
unitId = prevUnit.id
else:
newUnit = dts.createUnit([unitMeasure], [])
unitId = newUnit.id
value = genSampleValue(sampVals, concept)
attrs = [("contextRef", cntxId)]
if concept.isNumeric:
attrs.append(("unitRef", unitId))
attrs.append(("decimals", decimals))
value = Locale.atof(dts.locale, str(value), str.strip)
newFact = dts.createFact(concept.qname, attributes=attrs, text=value)
if concept not in visited:
visited.add(concept)
rels = relationshipSet.fromModelObject(concept)
lenRels = len(rels)
iRel = 0
iFirstLineItem = None
while iRel <= lenRels:
if iRel == lenRels: # check if cube needs re-iterating
if iFirstLineItem is None or elrInfo.get("domainIter",0) >= 2:
break
reIterateCube = True # cube can re-iterate
else:
modelRel = rels[iRel]
toConcept = modelRel.toModelObject
reIterateCube = (toConcept.isHypercubeItem and # finished prior line items and hitting next table
iFirstLineItem is not None and
elrInfo["lineItems"] and 1 <= elrInfo.get("domainIter",0) < 2)
if reIterateCube: # repeat typed dim container
iRel = iFirstLineItem
elrInfo["domainIter"] += 1
elrInfo.pop("instant", None) # want new contexts for next iteration
elrInfo.pop("duration", None)
isFirstLineItem = not elrInfo["lineItems"]
genFact(dts, toConcept, modelRel.preferredLabel, arcrole, relationshipSet, level+1, visited, elrInfo)
if isFirstLineItem and elrInfo["lineItems"] and elrInfo.get("domainIter",0) > 0:
iFirstLineItem = iRel
iRel += 1
visited.remove(concept)
except AttributeError as ex: # bad relationship
print ("[exception] {}".format(ex))
return
def viewConcept(self, concept, modelObject, labelPrefix, preferredLabel,
parentnode, n, relationshipSet, visited):
if concept is None:
return
try:
isRelation = isinstance(modelObject,
ModelDtsObject.ModelRelationship)
isModelTable = False
filingIndicatorCode = ""
if isinstance(concept, ModelDtsObject.ModelConcept):
text = labelPrefix + concept.label(
preferredLabel,
lang=self.lang,
linkroleHint=relationshipSet.linkrole)
if (self.arcrole
in ("XBRL-dimensions", XbrlConst.hypercubeDimension)
and concept.isTypedDimension
and concept.typedDomainElement is not None):
text += " (typedDomain={0})".format(
concept.typedDomainElement.qname)
elif isinstance(concept, ModelInstanceObject.ModelFact):
if concept.concept is not None:
text = labelPrefix + concept.concept.label(
preferredLabel,
lang=self.lang,
linkroleHint=relationshipSet.linkrole)
else:
text = str(concept.qname)
if concept.contextID:
text += " [" + concept.contextID + "] = " + concept.effectiveValue
elif self.arcrole == "Table-rendering":
text = concept.localName
elif isinstance(concept, ModelRenderingObject.ModelTable):
text = (concept.genLabel(lang=self.lang, strip=True)
or concept.localName)
isModelTable = True
elif isinstance(concept, ModelDtsObject.ModelResource):
if self.showReferences:
text = (concept.viewText() or concept.localName)
else:
text = (Locale.rtlString(concept.textValue.strip(),
lang=concept.xmlLang)
or concept.localName)
else: # just a resource
text = concept.localName
childnode = self.treeView.insert(parentnode,
"end",
modelObject.objectId(self.id),
text=text,
tags=("odd" if n
& 1 else "even", ))
# Check if we need special rendering of this item
for pluginXbrlMethod in pluginClassMethods(
"CntlrWinMain.Rendering.RenderConcept"):
stopPlugin = pluginXbrlMethod(isModelTable, concept, text,
self, self.modelXbrl, childnode)
if stopPlugin == True:
break
childRelationshipSet = relationshipSet
if self.arcrole == XbrlConst.parentChild: # extra columns
if isRelation:
preferredLabel = modelObject.preferredLabel
if preferredLabel and preferredLabel.startswith(
"http://www.xbrl.org/2003/role/"):
preferredLabel = os.path.basename(preferredLabel)
self.treeView.set(childnode, "preferredLabel",
preferredLabel)
self.treeView.set(childnode, "type", concept.niceType)
self.treeView.set(childnode, "references",
viewReferences(concept))
elif self.arcrole == XbrlConst.summationItem:
if isRelation:
self.treeView.set(childnode, "weight",
"{:+0g} ".format(modelObject.weight))
self.treeView.set(childnode, "balance", concept.balance)
elif self.arcrole == "XBRL-dimensions" and isRelation: # extra columns
relArcrole = modelObject.arcrole
self.treeView.set(childnode, "arcrole",
os.path.basename(relArcrole))
if relArcrole in (XbrlConst.all, XbrlConst.notAll):
self.treeView.set(childnode, "contextElement",
modelObject.contextElement)
self.treeView.set(childnode, "closed", modelObject.closed)
elif relArcrole in (XbrlConst.dimensionDomain,
XbrlConst.domainMember):
self.treeView.set(childnode, "usable", modelObject.usable)
childRelationshipSet = self.modelXbrl.relationshipSet(
XbrlConst.consecutiveArcrole.get(relArcrole,
"XBRL-dimensions"),
modelObject.consecutiveLinkrole)
elif self.arcrole == "Table-rendering": # extra columns
try:
header = concept.header(lang=self.lang,
strip=True,
evaluate=False)
except AttributeError:
header = None # could be a filter
if isRelation and header is None:
header = "{0} {1}".format(
os.path.basename(modelObject.arcrole),
concept.xlinkLabel)
self.treeView.set(childnode, "header", header)
if concept.get("abstract") == "true":
self.treeView.set(childnode, "abstract",
'\u2713') # checkmark unicode character
if concept.get("merge") == "true":
self.treeView.set(childnode, "merge",
'\u2713') # checkmark unicode character
if isRelation:
self.treeView.set(childnode, "axis",
modelObject.axisDisposition)
if isinstance(concept,
(ModelEuAxisCoord, ModelRuleDefinitionNode)):
self.treeView.set(
childnode, "priItem",
concept.aspectValue(None, Aspect.CONCEPT))
self.treeView.set(
childnode, "dims", ' '.join(
("{0},{1}".format(
dim, concept.aspectValue(None, dim))
for dim in (concept.aspectValue(
None, Aspect.DIMENSIONS, inherit=False)
or []))))
elif self.isResourceArcrole: # resource columns
if isRelation:
self.treeView.set(childnode, "arcrole",
os.path.basename(modelObject.arcrole))
if isinstance(concept, ModelDtsObject.ModelResource):
self.treeView.set(childnode, "resource", concept.localName)
self.treeView.set(childnode, "resourcerole",
os.path.basename(concept.role or ''))
self.treeView.set(childnode, "lang", concept.xmlLang)
self.id += 1
self.tag_has[modelObject.objectId()].append(childnode)
if isRelation:
self.tag_has[modelObject.toModelObject.objectId()].append(
childnode)
if concept not in visited:
visited.add(concept)
for modelRel in childRelationshipSet.fromModelObject(concept):
nestedRelationshipSet = childRelationshipSet
targetRole = modelRel.targetRole
if self.arcrole == XbrlConst.summationItem:
childPrefix = "({:0g}) ".format(
modelRel.weight
) # format without .0 on integer weights
elif targetRole is None or len(targetRole) == 0:
targetRole = relationshipSet.linkrole
childPrefix = ""
else:
nestedRelationshipSet = self.modelXbrl.relationshipSet(
childRelationshipSet.arcrole, targetRole)
childPrefix = "(via targetRole) "
toConcept = modelRel.toModelObject
if toConcept in visited:
childPrefix += "(loop)"
labelrole = modelRel.preferredLabel
if not labelrole or self.labelrole == conceptNameLabelRole:
labelrole = self.labelrole
n += 1 # child has opposite row style of parent
self.viewConcept(toConcept, modelRel, childPrefix,
labelrole, childnode, n,
nestedRelationshipSet, visited)
visited.remove(concept)
except AttributeError:
return # bad object, don't try to display
def savePickle(cntlr, modelXbrl, pickleFile):
if modelXbrl.fileSource.isArchive:
return
import io, time, pickle
from arelle import Locale
startedAt = time.time()
fh = io.open(pickleFile, u"wb")
try:
pickle.dump(modelXbrl, fh)
except Exception, ex:
cntlr.addToLog(u"Exception " + unicode(ex))
fh.close()
cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale,
_(u"profiled command processing completed in %.2f secs"),
time.time() - startedAt))
def savePickleMenuEntender(cntlr, menu):
# Extend menu with an item for the save infoset plugin
menu.add_command(label=u"Save pickled modelXbrl",
underline=0,
command=lambda: savePickleMenuCommand(cntlr) )
def savePickleMenuCommand(cntlr):
# save Infoset menu item has been invoked
from arelle.ModelDocument import Type
if cntlr.modelManager is None or cntlr.modelManager.modelXbrl is None or cntlr.modelManager.modelXbrl.modelDocument.type != Type.INSTANCE:
cntlr.addToLog(u"No instance loaded.")
return
def genFact(dts, concept, preferredLabel, arcrole, relationshipSet, level, visited, elrInfo):
try:
if concept is not None:
if concept.isHypercubeItem:
elrInfo["inCube"] = level
elrInfo["dims"] = {}
elrInfo["lineItems"] =False
elrInfo.pop("instant", None)
elrInfo.pop("duration", None)
elif concept.isDimensionItem:
elrInfo["currentDim"] = concept
if concept.isTypedDimension:
elrInfo["dims"][concept.qname] = (concept, concept.typedDomainElement)
if concept.typedDomainElement.isNumeric:
elrInfo["domainIter"] = 1
elif concept.name.endswith("Member") or concept.name.endswith("_member"): # don't generate entries for default dim (Domain) (for now)
dimConcept = elrInfo["currentDim"]
if dimConcept.qname not in elrInfo["dims"]:
elrInfo["dims"][dimConcept.qname] = (dimConcept, concept)
else:
if concept.name.endswith("LineItems") or concept.name.endswith("_line_items"):
elrInfo["lineItems"] = True
elif ((not elrInfo["inCube"] or # before any hypercube
elrInfo["lineItems"]) # in Cube and within Line Items
and not concept.isAbstract): # or within line items
# generate a fact
sampVals = sampleDataValues[elrInfo.get("domainIter",1)] # use first entry if no domain iter
if concept.periodType not in elrInfo:
qnameDims = {}
for _dimConcept, _domConcept in elrInfo["dims"].values():
if _dimConcept.isExplicitDimension:
_memVal = _domConcept.qname
else:
if _domConcept.type is not None and not _domConcept.isNumeric:
_memEltVal = genSampleValue(sampVals, _domConcept)
else:
_memEltVal = str(elrInfo["domainIter"])
_memVal = XmlUtil.addChild(dts.modelDocument.xmlRootElement,
_domConcept.qname,
text=_memEltVal,
appendChild=False)
_dimObj = DimValuePrototype(dts, None, _dimConcept.qname, _memVal, "segment")
qnameDims[_dimConcept.qname] = _dimObj
elrInfo[concept.periodType] = dts.createContext(
dts.conceptSampleScheme or "http://www.treasury.gov",
"entityId",
concept.periodType,
sampVals["periodStart"] if concept.periodType == "duration"
else None,
sampVals["periodEnd"],
concept.qname, qnameDims, [], [])
cntx = elrInfo[concept.periodType]
cntxId = cntx.id
if concept.isNumeric:
if concept.isMonetary:
unitMeasure = qname(XbrlConst.iso4217, "USD")
unitMeasure.prefix = "iso4217" # want to save with a recommended prefix
decimals = 2
elif concept.isShares:
unitMeasure = XbrlConst.qnXbrliShares
decimals = 0
else:
unitMeasure = XbrlConst.qnXbrliPure
decimals = 0
# check if utr unitId is specified
utrUnitId = genSampleUtrUnitId(concept)
if utrUnitId is not None:
_utrEntries = dts.modelManager.disclosureSystem.utrItemTypeEntries[concept.type.name]
if _utrEntries:
for _utrEntry in _utrEntries.values():
if _utrEntry.unitId == utrUnitId and _utrEntry.isSimple:
unitMeasure = qname(_utrEntry.nsUnit, _utrEntry.unitId)
break
prevUnit = dts.matchUnit([unitMeasure], [])
if prevUnit is not None:
unitId = prevUnit.id
else:
newUnit = dts.createUnit([unitMeasure], [])
unitId = newUnit.id
value = genSampleValue(sampVals, concept)
attrs = [("contextRef", cntxId)]
if concept.isNumeric:
attrs.append(("unitRef", unitId))
attrs.append(("decimals", decimals))
value = Locale.atof(dts.locale, str(value), str.strip)
newFact = dts.createFact(concept.qname, attributes=attrs, text=value)
if concept not in visited:
visited.add(concept)
rels = relationshipSet.fromModelObject(concept)
lenRels = len(rels)
iRel = 0
iFirstLineItem = None
while iRel <= lenRels:
if iRel == lenRels: # check if cube needs re-iterating
if iFirstLineItem is None or elrInfo.get("domainIter",0) >= 2:
break
reIterateCube = True # cube can re-iterate
else:
modelRel = rels[iRel]
toConcept = modelRel.toModelObject
reIterateCube = (toConcept.isHypercubeItem and # finished prior line items and hitting next table
iFirstLineItem is not None and
elrInfo["lineItems"] and 1 <= elrInfo.get("domainIter",0) < 2)
if reIterateCube: # repeat typed dim container
iRel = iFirstLineItem
elrInfo["domainIter"] += 1
elrInfo.pop("instant", None) # want new contexts for next iteration
elrInfo.pop("duration", None)
isFirstLineItem = not elrInfo["lineItems"]
genFact(dts, toConcept, modelRel.preferredLabel, arcrole, relationshipSet, level+1, visited, elrInfo)
if isFirstLineItem and elrInfo["lineItems"] and elrInfo.get("domainIter",0) > 0:
iFirstLineItem = iRel
iRel += 1
visited.remove(concept)
except AttributeError as ex: # bad relationship
print ("[exception] {}".format(ex))
return
def backgroundSaveInstance(self, newFilename=None):
cntlr = self.modelXbrl.modelManager.cntlr
if newFilename:
self.modelXbrl.modelManager.showStatus(
_("creating new instance {0}").format(
os.path.basename(newFilename)))
self.modelXbrl.modelManager.cntlr.waitForUiThreadQueue(
) # force status update
self.modelXbrl.createInstance(
newFilename
) # creates an instance as this modelXbrl's entrypoing
instance = self.modelXbrl
cntlr.showStatus(
_("Saving {0}").format(instance.modelDocument.basename))
cntlr.waitForUiThreadQueue() # force status update
newCntx = ModelXbrl.AUTO_LOCATE_ELEMENT
newUnit = ModelXbrl.AUTO_LOCATE_ELEMENT
# check user keyed changes
for bodyCell in self.gridBody.winfo_children():
if isinstance(bodyCell, gridCell) and bodyCell.isChanged:
value = bodyCell.value
objId = bodyCell.objectId
if objId:
if objId[0] == "f":
factPrototypeIndex = int(objId[1:])
factPrototype = self.factPrototypes[factPrototypeIndex]
concept = factPrototype.concept
entityIdentScheme = self.newFactItemOptions.entityIdentScheme
entityIdentValue = self.newFactItemOptions.entityIdentValue
periodType = factPrototype.concept.periodType
periodStart = self.newFactItemOptions.startDateDate if periodType == "duration" else None
periodEndInstant = self.newFactItemOptions.endDateDate
qnameDims = factPrototype.context.qnameDims
prevCntx = instance.matchContext(
entityIdentScheme, entityIdentValue, periodType,
periodStart, periodEndInstant, qnameDims, [], [])
if prevCntx is not None:
cntxId = prevCntx.id
else: # need new context
newCntx = instance.createContext(
entityIdentScheme,
entityIdentValue,
periodType,
periodStart,
periodEndInstant,
concept.qname,
qnameDims, [], [],
afterSibling=newCntx)
cntxId = newCntx.id
# new context
if concept.isNumeric:
if concept.isMonetary:
unitMeasure = qname(
XbrlConst.iso4217,
self.newFactItemOptions.monetaryUnit)
unitMeasure.prefix = "iso4217" # want to save with a recommended prefix
decimals = self.newFactItemOptions.monetaryDecimals
elif concept.isShares:
unitMeasure = XbrlConst.qnXbrliShares
decimals = self.newFactItemOptions.nonMonetaryDecimals
else:
unitMeasure = XbrlConst.qnXbrliPure
decimals = self.newFactItemOptions.nonMonetaryDecimals
prevUnit = instance.matchUnit([unitMeasure], [])
if prevUnit is not None:
unitId = prevUnit.id
else:
newUnit = instance.createUnit(
[unitMeasure], [], afterSibling=newUnit)
unitId = newUnit.id
attrs = [("contextRef", cntxId)]
if concept.isNumeric:
attrs.append(("unitRef", unitId))
attrs.append(("decimals", decimals))
value = Locale.atof(self.modelXbrl.locale, value,
str.strip)
newFact = instance.createFact(concept.qname,
attributes=attrs,
text=value)
bodyCell.objectId = newFact.objectId(
) # switch cell to now use fact ID
if self.factPrototypes[factPrototypeIndex] is not None:
self.factPrototypes[factPrototypeIndex].clear()
self.factPrototypes[
factPrototypeIndex] = None #dereference fact prototype
else: # instance fact, not prototype
fact = self.modelXbrl.modelObject(objId)
if fact.concept.isNumeric:
value = Locale.atof(self.modelXbrl.locale, value,
str.strip)
if fact.value != value:
fact.text = value
XmlValidate.validate(instance, fact)
bodyCell.isChanged = False # clear change flag
instance.saveInstance()
cntlr.showStatus(_("Saved {0}").format(
instance.modelDocument.basename),
clearAfter=3000)
def viewConcept(self, concept, modelObject, labelPrefix, preferredLabel, parentnode, n, relationshipSet, visited):
if concept is None:
return
try:
isRelation = isinstance(modelObject, ModelDtsObject.ModelRelationship)
if isinstance(concept, ModelDtsObject.ModelConcept):
text = labelPrefix + concept.label(preferredLabel,lang=self.lang,linkroleHint=relationshipSet.linkrole)
if (self.arcrole in ("XBRL-dimensions", XbrlConst.hypercubeDimension) and
concept.isTypedDimension and
concept.typedDomainElement is not None):
text += " (typedDomain={0})".format(concept.typedDomainElement.qname)
elif isinstance(concept, ModelInstanceObject.ModelFact):
if concept.concept is not None:
text = labelPrefix + concept.concept.label(preferredLabel,lang=self.lang,linkroleHint=relationshipSet.linkrole)
else:
text = str(concept.qname)
if concept.contextID:
text += " [" + concept.contextID + "] = " + concept.effectiveValue
elif self.arcrole == "Table-rendering":
text = concept.localName
elif isinstance(concept, ModelRenderingObject.ModelTable):
text = (concept.genLabel(lang=self.lang, strip=True) or concept.localName)
elif isinstance(concept, ModelDtsObject.ModelResource):
if self.showReferences:
text = (concept.viewText() or concept.localName)
else:
text = (Locale.rtlString(concept.textValue.strip(), lang=concept.xmlLang) or concept.localName)
else: # just a resource
text = concept.localName
childnode = self.treeView.insert(parentnode, "end", modelObject.objectId(self.id), text=text, tags=("odd" if n & 1 else "even",))
childRelationshipSet = relationshipSet
if self.arcrole == XbrlConst.parentChild: # extra columns
if isRelation:
preferredLabel = modelObject.preferredLabel
if preferredLabel and preferredLabel.startswith("http://www.xbrl.org/2003/role/"):
preferredLabel = os.path.basename(preferredLabel)
self.treeView.set(childnode, "preferredLabel", preferredLabel)
self.treeView.set(childnode, "type", concept.niceType)
self.treeView.set(childnode, "references", viewReferences(concept))
elif self.arcrole == XbrlConst.summationItem:
if isRelation:
self.treeView.set(childnode, "weight", "{:+0g} ".format(modelObject.weight))
self.treeView.set(childnode, "balance", concept.balance)
elif self.arcrole == "XBRL-dimensions" and isRelation: # extra columns
relArcrole = modelObject.arcrole
self.treeView.set(childnode, "arcrole", os.path.basename(relArcrole))
if relArcrole in (XbrlConst.all, XbrlConst.notAll):
self.treeView.set(childnode, "contextElement", modelObject.contextElement)
self.treeView.set(childnode, "closed", modelObject.closed)
elif relArcrole in (XbrlConst.dimensionDomain, XbrlConst.domainMember):
self.treeView.set(childnode, "usable", modelObject.usable)
childRelationshipSet = self.modelXbrl.relationshipSet(XbrlConst.consecutiveArcrole.get(relArcrole,"XBRL-dimensions"),
modelObject.consecutiveLinkrole)
elif self.arcrole == "Table-rendering": # extra columns
try:
header = concept.header(lang=self.lang,strip=True,evaluate=False)
except AttributeError:
header = None # could be a filter
if isRelation and header is None:
header = "{0} {1}".format(os.path.basename(modelObject.arcrole), concept.xlinkLabel)
self.treeView.set(childnode, "header", header)
if concept.get("abstract") == "true":
self.treeView.set(childnode, "abstract", '\u2713') # checkmark unicode character
if concept.get("merge") == "true":
self.treeView.set(childnode, "merge", '\u2713') # checkmark unicode character
if isRelation:
self.treeView.set(childnode, "axis", modelObject.axisDisposition)
if isinstance(concept, (ModelEuAxisCoord,ModelRuleDefinitionNode)):
self.treeView.set(childnode, "priItem", concept.aspectValue(None, Aspect.CONCEPT))
self.treeView.set(childnode, "dims", ' '.join(("{0},{1}".format(dim, concept.aspectValue(None, dim))
for dim in (concept.aspectValue(None, Aspect.DIMENSIONS, inherit=False) or []))))
elif self.isResourceArcrole: # resource columns
if isRelation:
self.treeView.set(childnode, "arcrole", os.path.basename(modelObject.arcrole))
if isinstance(concept, ModelDtsObject.ModelResource):
self.treeView.set(childnode, "resource", concept.localName)
self.treeView.set(childnode, "resourcerole", os.path.basename(concept.role or ''))
self.treeView.set(childnode, "lang", concept.xmlLang)
self.id += 1
self.tag_has[modelObject.objectId()].append(childnode)
if isRelation:
self.tag_has[modelObject.toModelObject.objectId()].append(childnode)
if concept not in visited:
visited.add(concept)
for modelRel in childRelationshipSet.fromModelObject(concept):
nestedRelationshipSet = childRelationshipSet
targetRole = modelRel.targetRole
if self.arcrole == XbrlConst.summationItem:
childPrefix = "({:0g}) ".format(modelRel.weight) # format without .0 on integer weights
elif targetRole is None or len(targetRole) == 0:
targetRole = relationshipSet.linkrole
childPrefix = ""
else:
nestedRelationshipSet = self.modelXbrl.relationshipSet(childRelationshipSet.arcrole, targetRole)
childPrefix = "(via targetRole) "
toConcept = modelRel.toModelObject
if toConcept in visited:
childPrefix += "(loop)"
labelrole = modelRel.preferredLabel
if not labelrole: labelrole = self.labelrole
n += 1 # child has opposite row style of parent
self.viewConcept(toConcept, modelRel, childPrefix, labelrole, childnode, n, nestedRelationshipSet, visited)
visited.remove(concept)
except AttributeError:
return # bad object, don't try to display
def factCheck(val, fact):
concept = fact.concept
context = fact.context
if concept is None or context is None or not val.validateLoggingSemantic:
return # not checkable
try:
if fact.isNumeric:
# 2.3.3 additional unit tests beyond UTR spec
unit = fact.unit
if unit is not None and concept.type is not None and val.validateUTR:
typeName = concept.type.name
if typeName == "perUnitItemType" and any(m.namespaceURI == XbrlConst.iso4217 or
m in (XbrlConst.qnXbrliPure, XbrlConst.qnXbrliShares)
for m in unit.measures[1]):
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.3.3.perUnitItemType",
_("PureItemType fact %(fact)s in context %(contextID)s unit %(unitID)s value %(value)s has disallowed unit denominator %(denominator)s"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID,
value=fact.effectiveValue, denominator=", ".join((str(m) for m in unit.measures[1])))
if not fact.isNil and getattr(fact, "xValue", None) is not None:
# 2.4.1 decimal disagreement
if fact.decimals and fact.decimals != "INF":
vf = float(fact.value)
if _ISFINITE(vf):
dec = _INT(fact.decimals)
vround = round(vf, dec)
if vf != vround:
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.4.1",
_("Decimal disagreement %(fact)s in context %(contextID)s unit %(unitID)s value %(value)s has insignificant value %(insignificantValue)s"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID,
value=fact.effectiveValue, insignificantValue=Locale.format(val.modelXbrl.locale, "%.*f",
(dec + 2 if dec > 0 else 0, vf - vround),
True))
# 2.5.1 fractions disallowed on a disclosure
if fact.isFraction:
if any(val.linroleDefinitionIsDisclosure.match(roleType.definition)
for rel in val.modelXbrl.relationshipSet(XbrlConst.parentChild).toModelObject(concept)
for roleType in val.modelXbrl.roleTypes.get(rel.linkrole,())):
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.5.1",
_("Disclosure %(fact)s in context %(contextID)s value %(value)s is a fraction"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, value=fact.value)
# deprecated concept
if concept.qname.namespaceURI == val.ugtNamespace:
if concept.name in val.usgaapDeprecations:
val.deprecatedFactConcepts[concept].append(fact)
elif concept.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedFactConcepts[concept].append(fact)
if fact.isItem and fact.context is not None:
for dimConcept, modelDim in fact.context.segDimValues.items():
if dimConcept.qname.namespaceURI == val.ugtNamespace:
if dimConcept.name in val.usgaapDeprecations:
val.deprecatedDimensions[dimConcept].append(fact)
elif dimConcept.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedDimensions[dimConcept].append(fact)
if modelDim.isExplicit:
member = modelDim.member
if member is not None:
if member.qname.namespaceURI == val.ugtNamespace:
if member.name in val.usgaapDeprecations:
val.deprecatedMembers[member].append(fact)
elif member.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedMembers[member].append(fact)
except Exception as err:
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.testingException",
_("%(fact)s in context %(contextID)s unit %(unitID)s value %(value)s cannot be tested due to: %(err)s"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID,
value=fact.effectiveValue, err=err)
def loadXbrlFromDB(self, loadDBsaveToFile):
# load from database
modelXbrl = self.modelXbrl
# find instance in DB
instanceURI = os.path.basename(loadDBsaveToFile)
results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant"
" FROM dInstance WHERE FileName = '{}'"
.format(instanceURI))
instanceId = moduleId = None
for instanceId, moduleId, entScheme, entId, datePerEnd in results:
break
# find module in DB
results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId))
xbrlSchemaRef = None
for result in results:
xbrlSchemaRef = result[0]
break
if not instanceId or not xbrlSchemaRef:
raise XPDBException("sqlDB:MissingDTS",
_("The instance and module were not found for %(instanceURI)"),
instanceURI = instanceURI)
if modelXbrl.skipDTS:
# find prefixes and namespaces in DB
results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]")
dpmPrefixedNamespaces = dict((prefix, namespace)
for owner, prefix, namespace in results)
# create the instance document and resulting filing
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(modelXbrl,
Type.INSTANCE,
loadDBsaveToFile,
schemaRefs=[xbrlSchemaRef],
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement,
'xbrl-streamable-instance',
'version="1.0" contextBuffer="1"')
# add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue)
# filing indicator code IDs
# get filing indicators
results = self.execute("SELECT mToT.TemplateOrTableCode "
" FROM dFilingIndicator dFI, mTemplateOrTable mToT "
" WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID"
.format(instanceId))
filingIndicatorCodes = [code[0] for code in results]
if filingIndicatorCodes:
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
{}, [], [],
id='c')
filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators)
for filingIndicatorCode in filingIndicatorCodes:
modelXbrl.createFact(qnFindFilingIndicator,
parent=filingIndicatorsTuple,
attributes={"contextRef": "c"},
text=filingIndicatorCode)
# facts in this instance
factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards,"
" Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue "
"FROM dFact WHERE InstanceID = {} "
"ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL "
" THEN DataPointSignature"
" ELSE DataPointSignatureWithValuesForWildcards"
" END, instr(DataPointSignature,'|') + 1)"
.format(instanceId))
# results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal
# get typed dimension values
prefixedNamespaces = modelXbrl.prefixedNamespaces
prefixedNamespaces["iso4217"] = XbrlConst.iso4217
if modelXbrl.skipDTS:
prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed
cntxTbl = {} # index by d
unitTbl = {}
def typedDimElt(s):
# add xmlns into s for known qnames
tag, angleBrkt, rest = s[1:].partition('>')
text, angleBrkt, rest = rest.partition("<")
qn = qname(tag, prefixedNamespaces)
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
# contexts and facts
for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl:
metric, sep, dims = (dpSigTypedDims or dpSig).partition('|')
conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces)
concept = modelXbrl.qnameConcepts.get(conceptQn)
isNumeric = isBool = isDateTime = isQName = isText = False
if concept is not None:
if concept.isNumeric:
isNumeric = True
else:
baseXbrliType = concept.baseXbrliType
if baseXbrliType == "booleanItemType":
isBool = True
elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
isDateTime = True
elif baseXbrliType == "QNameItemType":
isQName = True
else:
c = conceptQn.localName[0]
if c == 'm':
isNumeric = True
elif c == 'd':
isDateTime = True
elif c == 'b':
isBool = True
elif c == 'e':
isQName = True
isText = not (isNumeric or isBool or isDateTime or isQName)
if isinstance(datePerEnd, _STR_BASE):
datePerEnd = datetimeValue(datePerEnd, addOneDay=True)
cntxKey = (dims, entId, datePerEnd)
if cntxKey in cntxTbl:
cntxId = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
cntxTbl[cntxKey] = cntxId
qnameDims = {}
if dims:
for dim in dims.split('|'):
dQn, sep, dVal = dim[:-1].partition('(')
dimQname = qname(dQn, prefixedNamespaces)
if dVal.startswith('<'):
mem = typedDimElt(dVal) # typed dim
else:
mem = qname(dVal, prefixedNamespaces) # explicit dim (even if treat-as-typed)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario")
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId)
if unit:
if unit in unitTbl:
unitId = unitTbl[unit]
else:
unitQn = qname(unit, prefixedNamespaces)
unitId = 'u{}'.format(unitQn.localName)
unitTbl[unit] = unitId
modelXbrl.createUnit([unitQn], [], id=unitId)
else:
unitId = None
attrs = {"contextRef": cntxId}
if unitId:
attrs["unitRef"] = unitId
if dec is not None:
if isinstance(dec, float): # must be an integer
dec = int(dec)
elif isinstance(dec, _STR_BASE) and '.' in dec:
dec = dec.partition('.')[0] # drop .0 from any SQLite string
attrs["decimals"] = str(dec) # somehow it is float from the database
if False: # fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
elif numVal is not None:
num = roundValue(numVal, None, dec) # round using reported decimals
if dec is None or dec == "INF": # show using decimals or reported format
dec = len(numVal.partition(".")[2])
else: # max decimals at 28
dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num))
elif dateVal is not None:
text = dateVal
elif boolVal is not None:
text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false'
else:
if isQName: # declare namespace
addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces))
text = textVal
modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
# add footnotes if any
# save to file
modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
return modelXbrl.modelDocument
def parsePackage(cntlr, filesource, metadataFile, fileBase):
global ArchiveFileIOError
if ArchiveFileIOError is None:
from arelle.FileSource import ArchiveFileIOError
unNamedCounter = 1
txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1",
"http://xbrl.org/PWD/2014-01-15/taxonomy-package",
"http://xbrl.org/PWD/2015-01-14/taxonomy-package")
catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog",)
pkg = {}
currentLang = Locale.getLanguageCode()
_file = filesource.file(metadataFile)[0] # URL in zip, plain file in file system or web
tree = etree.parse(_file)
root = tree.getroot()
ns = root.tag.partition("}")[0][1:]
nsPrefix = "{{{}}}".format(ns)
if ns in txmyPkgNSes: # package file
for eltName in ("name", "description", "version"):
pkg[eltName] = ''
for m in root.iterchildren(tag=nsPrefix + eltName):
pkg[eltName] = m.text.strip()
break # take first entry if several
else: # oasis catalog, use dirname as the package name
# metadataFile may be a File object (with name) or string filename
fileName = getattr(metadataFile, 'fileName', # for FileSource named objects
getattr(metadataFile, 'name', # for io.file named objects
metadataFile)) # for string
pkg["name"] = os.path.basename(os.path.dirname(fileName))
pkg["description"] = "oasis catalog"
pkg["version"] = "(none)"
remappings = {}
rewriteTree = tree
catalogFile = metadataFile
if ns in ("http://xbrl.org/PWD/2015-01-14/taxonomy-package",):
catalogFile = metadataFile.replace('taxonomyPackage.xml','catalog.xml')
try:
rewriteTree = etree.parse(filesource.file(catalogFile)[0])
except ArchiveFileIOError:
pass
for tag, prefixAttr, replaceAttr in (
(nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package
("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem", "systemIdStartString", "rewritePrefix"),
("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI", "uriStartString", "rewritePrefix")): # oasis catalog
for m in rewriteTree.iter(tag=tag):
prefixValue = m.get(prefixAttr)
replaceValue = m.get(replaceAttr)
if prefixValue and replaceValue is not None:
if prefixValue not in remappings:
base = baseForElement(m)
if base:
replaceValue = os.path.join(base, replaceValue)
if replaceValue: # neither None nor ''
if not isAbsolute(replaceValue):
if not os.path.isabs(replaceValue):
replaceValue = fileBase + replaceValue
replaceValue = replaceValue.replace("/", os.sep)
_normedValue = os.path.normpath(replaceValue)
if replaceValue.endswith(os.sep) and not _normedValue.endswith(os.sep):
_normedValue += os.sep
remappings[prefixValue] = _normedValue
else:
cntlr.addToLog(_("Package catalog duplicate rewrite start string %(rewriteStartString)s"),
messageArgs={"rewriteStartString": prefixValue},
messageCode="arelle.catalogDuplicateRewrite",
file=os.path.basename(catalogFile),
level=logging.ERROR)
pkg["remappings"] = remappings
nameToUrls = {}
pkg["nameToUrls"] = nameToUrls
for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"):
name = None
# find closest match name node given xml:lang match to current language or no xml:lang
for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"):
xmlLang = nameNode.get('{http://www.w3.org/XML/1998/namespace}lang')
if name is None or not xmlLang or currentLang == xmlLang:
name = nameNode.text
if currentLang == xmlLang: # most prefer one with the current locale's language
break
if not name:
name = _("<unnamed {0}>").format(unNamedCounter)
unNamedCounter += 1
epDocCount = 0
for epDoc in entryPointSpec.iterchildren(nsPrefix + "entryPointDocument"):
epUrl = epDoc.get('href')
base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base') # cope with xml:base
if base:
resolvedUrl = urljoin(base, epUrl)
else:
resolvedUrl = epUrl
if epDocCount:
cntlr.addToLog(_("Skipping multiple-document entry point (not supported) %(href)s"),
messageArgs={"href": epUrl},
messageCode="arelle.packageMultipleDocumentEntryPoints",
file=os.path.basename(metadataFile),
level=logging.WARNING)
continue
epDocCount += 1
#perform prefix remappings
remappedUrl = resolvedUrl
longestPrefix = 0
for mapFrom, mapTo in remappings.items():
if remappedUrl.startswith(mapFrom):
prefixLength = len(mapFrom)
if prefixLength > longestPrefix:
_remappedUrl = remappedUrl[prefixLength:]
if not (_remappedUrl[0] in (os.sep, '/') or mapTo[-1] in (os.sep, '/')):
_remappedUrl = mapTo + os.sep + _remappedUrl
else:
_remappedUrl = mapTo + _remappedUrl
longestPrefix = prefixLength
if longestPrefix:
remappedUrl = _remappedUrl.replace(os.sep, "/") # always used as FileSource select
nameToUrls[name] = (remappedUrl, resolvedUrl)
return pkg
def parsePackage(mainWin, metadataFile):
unNamedCounter = 1
txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1",
"http://xbrl.org/PWD/2014-01-15/taxonomy-package")
catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog",)
pkg = {}
currentLang = Locale.getLanguageCode()
tree = etree.parse(metadataFile)
root = tree.getroot()
ns = root.tag.partition("}")[0][1:]
nsPrefix = "{{{}}}".format(ns)
if ns in txmyPkgNSes: # package file
for eltName in ("name", "description", "version"):
pkg[eltName] = ''
for m in root.iterchildren(tag=nsPrefix + eltName):
pkg[eltName] = m.text.strip()
break # take first entry if several
else: # oasis catalog, use dirname as the package name
# metadataFile may be a File object (with name) or string filename
fileName = getattr(metadataFile, 'fileName', # for FileSource named objects
getattr(metadataFile, 'name', # for io.file named objects
metadataFile)) # for string
pkg["name"] = os.path.basename(os.path.dirname(fileName))
pkg["description"] = "oasis catalog"
pkg["version"] = "(none)"
remappings = {}
for tag, prefixAttr, replaceAttr in (
(nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package
(nsPrefix + "rewriteSystem", "systemIdStartString", "rewritePrefix")): # oasis catalog
for m in tree.iter(tag=tag):
prefixValue = m.get(prefixAttr)
replaceValue = m.get(replaceAttr)
if prefixValue and replaceValue is not None:
remappings[prefixValue] = replaceValue
pkg["remappings"] = remappings
nameToUrls = {}
pkg["nameToUrls"] = nameToUrls
for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"):
name = None
# find closest match name node given xml:lang match to current language or no xml:lang
for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"):
xmlLang = nameNode.get('{http://www.w3.org/XML/1998/namespace}lang')
if name is None or not xmlLang or currentLang == xmlLang:
name = nameNode.text
if currentLang == xmlLang: # most prefer one with the current locale's language
break
if not name:
name = _("<unnamed {0}>").format(unNamedCounter)
unNamedCounter += 1
epDocCount = 0
for epDoc in entryPointSpec.iterchildren(nsPrefix + "entryPointDocument"):
if epDocCount:
mainWin.addToLog(_("WARNING: skipping multiple-document entry point (not supported)"))
continue
epDocCount += 1
epUrl = epDoc.get('href')
base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base') # cope with xml:base
if base:
resolvedUrl = urljoin(base, epUrl)
else:
resolvedUrl = epUrl
#perform prefix remappings
remappedUrl = resolvedUrl
for prefix, replace in remappings.items():
remappedUrl = remappedUrl.replace(prefix, replace, 1)
nameToUrls[name] = (remappedUrl, resolvedUrl)
return pkg
def loadXbrlFromDB(self, loadDBsaveToFile):
# load from database
modelXbrl = self.modelXbrl
# find instance in DB
instanceURI = os.path.basename(loadDBsaveToFile)
results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant"
" FROM dInstance WHERE FileName = '{}'"
.format(instanceURI))
instanceId = moduleId = None
for instanceId, moduleId, entScheme, entId, datePerEnd in results:
break
# find module in DB
results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId))
xbrlSchemaRef = None
for result in results:
xbrlSchemaRef = result[0]
break
if not instanceId or not xbrlSchemaRef:
raise XPDBException("sqlDB:MissingDTS",
_("The instance and module were not found for %(instanceURI)"),
instanceURI = instanceURI)
if modelXbrl.skipDTS:
# find prefixes and namespaces in DB
results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]")
dpmPrefixedNamespaces = dict((prefix, namespace)
for owner, prefix, namespace in results)
# create the instance document and resulting filing
modelXbrl.blockDpmDBrecursion = True
modelXbrl.modelDocument = createModelDocument(modelXbrl,
Type.INSTANCE,
loadDBsaveToFile,
schemaRefs=[xbrlSchemaRef],
isEntry=True)
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement,
'xbrl-streamable-instance',
'version="1.0" contextBuffer="1"')
# add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue)
# filing indicator code IDs
# get filing indicators
results = self.execute("SELECT mToT.TemplateOrTableCode "
" FROM dFilingIndicator dFI, mTemplateOrTable mToT "
" WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID"
.format(instanceId))
filingIndicatorCodes = [code[0] for code in results]
if filingIndicatorCodes:
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
{}, [], [],
id='c')
filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators)
for filingIndicatorCode in filingIndicatorCodes:
modelXbrl.createFact(qnFindFilingIndicator,
parent=filingIndicatorsTuple,
attributes={"contextRef": "c"},
text=filingIndicatorCode)
# facts in this instance
factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards,"
" Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue "
"FROM dFact WHERE InstanceID = {} "
"ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL "
" THEN DataPointSignature"
" ELSE DataPointSignatureWithValuesForWildcards"
" END, instr(DataPointSignature,'|') + 1)"
.format(instanceId))
# results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal
# get typed dimension values
prefixedNamespaces = modelXbrl.prefixedNamespaces
prefixedNamespaces["iso4217"] = XbrlConst.iso4217
if modelXbrl.skipDTS:
prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed
cntxTbl = {} # index by d
unitTbl = {}
def typedDimElt(s):
# add xmlns into s for known qnames
tag, angleBrkt, rest = s[1:].partition('>')
text, angleBrkt, rest = rest.partition("<")
qn = qname(tag, prefixedNamespaces)
# a modelObject xml element is needed for all of the instance functions to manage the typed dim
return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False)
# contexts and facts
for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl:
metric, sep, dims = (dpSigTypedDims or dpSig).partition('|')
conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces)
concept = modelXbrl.qnameConcepts.get(conceptQn)
isNumeric = isBool = isDateTime = isQName = isText = False
if concept is not None:
if concept.isNumeric:
isNumeric = True
else:
baseXbrliType = concept.baseXbrliType
if baseXbrliType == "booleanItemType":
isBool = True
elif baseXbrliType == "dateTimeItemType": # also is dateItemType?
isDateTime = True
elif baseXbrliType == "QNameItemType":
isQName = True
else:
c = conceptQn.localName[0]
if c == 'm':
isNumeric = True
elif c == 'd':
isDateTime = True
elif c == 'b':
isBool = True
elif c == 'e':
isQName = True
isText = not (isNumeric or isBool or isDateTime or isQName)
if isinstance(datePerEnd, _STR_BASE):
datePerEnd = datetimeValue(datePerEnd, addOneDay=True)
cntxKey = (dims, entId, datePerEnd)
if cntxKey in cntxTbl:
cntxId = cntxTbl[cntxKey]
else:
cntxId = 'c-{:02}'.format(len(cntxTbl) + 1)
cntxTbl[cntxKey] = cntxId
qnameDims = {}
if dims:
for dim in dims.split('|'):
dQn, sep, dVal = dim[:-1].partition('(')
dimQname = qname(dQn, prefixedNamespaces)
if dVal.startswith('<'): # typed dim
mem = typedDimElt(dVal)
else:
mem = qname(dVal, prefixedNamespaces)
qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario")
modelXbrl.createContext(entScheme,
entId,
'instant',
None,
datePerEnd,
None, # no dimensional validity checking (like formula does)
qnameDims, [], [],
id=cntxId)
if unit:
if unit in unitTbl:
unitId = unitTbl[unit]
else:
unitQn = qname(unit, prefixedNamespaces)
unitId = 'u{}'.format(unitQn.localName)
unitTbl[unit] = unitId
modelXbrl.createUnit([unitQn], [], id=unitId)
else:
unitId = None
attrs = {"contextRef": cntxId}
if unitId:
attrs["unitRef"] = unitId
if dec is not None:
if isinstance(dec, float): # must be an integer
dec = int(dec)
elif isinstance(dec, _STR_BASE) and '.' in dec:
dec = dec.partition('.')[0] # drop .0 from any SQLite string
attrs["decimals"] = str(dec) # somehow it is float from the database
if False: # fact.isNil:
attrs[XbrlConst.qnXsiNil] = "true"
text = None
elif numVal is not None:
num = roundValue(numVal, None, dec) # round using reported decimals
if dec is None or dec == "INF": # show using decimals or reported format
dec = len(numVal.partition(".")[2])
else: # max decimals at 28
dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here
text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num))
elif dateVal is not None:
text = dateVal
elif boolVal is not None:
text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false'
else:
if isQName: # declare namespace
addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces))
text = textVal
modelXbrl.createFact(conceptQn, attributes=attrs, text=text)
# add footnotes if any
# save to file
modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile)
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
return modelXbrl.modelDocument
def parsePackage(cntlr, filesource, metadataFile, fileBase):
global ArchiveFileIOError
if ArchiveFileIOError is None:
from arelle.FileSource import ArchiveFileIOError
unNamedCounter = 1
txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1",
"http://xbrl.org/PWD/2014-01-15/taxonomy-package",
"http://xbrl.org/PWD/2015-01-14/taxonomy-package")
catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog", )
pkg = {}
currentLang = Locale.getLanguageCode()
_file = filesource.file(metadataFile)[
0] # URL in zip, plain file in file system or web
tree = etree.parse(_file)
root = tree.getroot()
ns = root.tag.partition("}")[0][1:]
nsPrefix = "{{{}}}".format(ns)
if ns in txmyPkgNSes: # package file
for eltName in ("name", "description", "version"):
pkg[eltName] = ''
for m in root.iterchildren(tag=nsPrefix + eltName):
pkg[eltName] = m.text.strip()
break # take first entry if several
else: # oasis catalog, use dirname as the package name
# metadataFile may be a File object (with name) or string filename
fileName = getattr(
metadataFile,
'fileName', # for FileSource named objects
getattr(
metadataFile,
'name', # for io.file named objects
metadataFile)) # for string
pkg["name"] = os.path.basename(os.path.dirname(fileName))
pkg["description"] = "oasis catalog"
pkg["version"] = "(none)"
remappings = {}
rewriteTree = tree
catalogFile = metadataFile
if ns in ("http://xbrl.org/PWD/2015-01-14/taxonomy-package", ):
catalogFile = metadataFile.replace('taxonomyPackage.xml',
'catalog.xml')
try:
rewriteTree = etree.parse(filesource.file(catalogFile)[0])
except ArchiveFileIOError:
pass
for tag, prefixAttr, replaceAttr in (
(nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package
("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem",
"systemIdStartString", "rewritePrefix"),
("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI",
"uriStartString", "rewritePrefix")): # oasis catalog
for m in rewriteTree.iter(tag=tag):
prefixValue = m.get(prefixAttr)
replaceValue = m.get(replaceAttr)
if prefixValue and replaceValue is not None:
if prefixValue not in remappings:
base = baseForElement(m)
if base:
replaceValue = os.path.join(base, replaceValue)
if replaceValue: # neither None nor ''
if not isAbsolute(replaceValue):
if not os.path.isabs(replaceValue):
replaceValue = fileBase + replaceValue
replaceValue = replaceValue.replace("/", os.sep)
_normedValue = os.path.normpath(replaceValue)
if replaceValue.endswith(
os.sep) and not _normedValue.endswith(os.sep):
_normedValue += os.sep
remappings[prefixValue] = _normedValue
else:
cntlr.addToLog(_(
"Package catalog duplicate rewrite start string %(rewriteStartString)s"
),
messageArgs={
"rewriteStartString": prefixValue
},
messageCode="arelle.catalogDuplicateRewrite",
file=os.path.basename(catalogFile),
level=logging.ERROR)
pkg["remappings"] = remappings
nameToUrls = {}
pkg["nameToUrls"] = nameToUrls
for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"):
name = None
# find closest match name node given xml:lang match to current language or no xml:lang
for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"):
xmlLang = nameNode.get(
'{http://www.w3.org/XML/1998/namespace}lang')
if name is None or not xmlLang or currentLang == xmlLang:
name = nameNode.text
if currentLang == xmlLang: # most prefer one with the current locale's language
break
if not name:
name = _("<unnamed {0}>").format(unNamedCounter)
unNamedCounter += 1
epDocCount = 0
for epDoc in entryPointSpec.iterchildren(nsPrefix +
"entryPointDocument"):
epUrl = epDoc.get('href')
base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base'
) # cope with xml:base
if base:
resolvedUrl = urljoin(base, epUrl)
else:
resolvedUrl = epUrl
if epDocCount:
cntlr.addToLog(
_("Skipping multiple-document entry point (not supported) %(href)s"
),
messageArgs={"href": epUrl},
messageCode="arelle.packageMultipleDocumentEntryPoints",
file=os.path.basename(metadataFile),
level=logging.WARNING)
continue
epDocCount += 1
#perform prefix remappings
remappedUrl = resolvedUrl
longestPrefix = 0
for mapFrom, mapTo in remappings.items():
if remappedUrl.startswith(mapFrom):
prefixLength = len(mapFrom)
if prefixLength > longestPrefix:
_remappedUrl = remappedUrl[prefixLength:]
if not (_remappedUrl[0] in (os.sep, '/')
or mapTo[-1] in (os.sep, '/')):
_remappedUrl = mapTo + os.sep + _remappedUrl
else:
_remappedUrl = mapTo + _remappedUrl
longestPrefix = prefixLength
if longestPrefix:
remappedUrl = _remappedUrl.replace(
os.sep, "/") # always used as FileSource select
nameToUrls[name] = (remappedUrl, resolvedUrl)
return pkg
def validate(self):
if not self.modelXbrl.contexts and not self.modelXbrl.facts:
return # skip if no contexts or facts
if not self.inferDecimals: # infering precision is now contrary to XBRL REC section 5.2.5.2
self.modelXbrl.info("xbrl.5.2.5.2:inferringPrecision","Validating calculations inferring precision.")
# identify equal contexts
self.modelXbrl.profileActivity()
uniqueContextHashes = {}
for context in self.modelXbrl.contexts.values():
h = context.contextDimAwareHash
if h in uniqueContextHashes:
if context.isEqualTo(uniqueContextHashes[h]):
self.mapContext[context] = uniqueContextHashes[h]
else:
uniqueContextHashes[h] = context
del uniqueContextHashes
self.modelXbrl.profileActivity("... identify equal contexts", minTimeToShow=1.0)
# identify equal contexts
uniqueUnitHashes = {}
for unit in self.modelXbrl.units.values():
h = unit.hash
if h in uniqueUnitHashes:
if unit.isEqualTo(uniqueUnitHashes[h]):
self.mapUnit[unit] = uniqueUnitHashes[h]
else:
uniqueUnitHashes[h] = unit
self.modelXbrl.profileActivity("... identify equal units", minTimeToShow=1.0)
# identify concepts participating in essence-alias relationships
# identify calcluation & essence-alias base sets (by key)
for baseSetKey in self.modelXbrl.baseSets.keys():
arcrole, ELR, linkqname, arcqname = baseSetKey
if ELR and linkqname and arcqname:
if arcrole in (XbrlConst.essenceAlias, XbrlConst.requiresElement):
conceptsSet = {XbrlConst.essenceAlias:self.conceptsInEssencesAlias,
XbrlConst.requiresElement:self.conceptsInRequiresElement}[arcrole]
for modelRel in self.modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname).modelRelationships:
for concept in (modelRel.fromModelObject, modelRel.toModelObject):
if concept is not None and concept.qname is not None:
conceptsSet.add(concept)
self.modelXbrl.profileActivity("... identify requires-element and esseance-aliased concepts", minTimeToShow=1.0)
self.bindFacts(self.modelXbrl.facts,[self.modelXbrl.modelDocument.xmlRootElement])
self.modelXbrl.profileActivity("... bind facts", minTimeToShow=1.0)
# identify calcluation & essence-alias base sets (by key)
for baseSetKey in self.modelXbrl.baseSets.keys():
arcrole, ELR, linkqname, arcqname = baseSetKey
if ELR and linkqname and arcqname:
if arcrole in (XbrlConst.summationItem, XbrlConst.essenceAlias, XbrlConst.requiresElement):
relsSet = self.modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname)
if arcrole == XbrlConst.summationItem:
fromRelationships = relsSet.fromModelObjects()
for sumConcept, modelRels in fromRelationships.items():
sumBindingKeys = self.sumConceptBindKeys[sumConcept]
dupBindingKeys = set()
boundSumKeys = set()
# determine boundSums
for modelRel in modelRels:
itemConcept = modelRel.toModelObject
if itemConcept is not None and itemConcept.qname is not None:
itemBindingKeys = self.itemConceptBindKeys[itemConcept]
boundSumKeys |= sumBindingKeys & itemBindingKeys
# add up rounded items
boundSums = defaultdict(decimal.Decimal) # sum of facts meeting factKey
boundSummationItems = defaultdict(list) # corresponding fact refs for messages
for modelRel in modelRels:
weight = modelRel.weightDecimal
itemConcept = modelRel.toModelObject
if itemConcept is not None:
for itemBindKey in boundSumKeys:
ancestor, contextHash, unit = itemBindKey
factKey = (itemConcept, ancestor, contextHash, unit)
if factKey in self.itemFacts:
for fact in self.itemFacts[factKey]:
if fact in self.duplicatedFacts:
dupBindingKeys.add(itemBindKey)
else:
roundedValue = roundFact(fact, self.inferDecimals)
boundSums[itemBindKey] += roundedValue * weight
boundSummationItems[itemBindKey].append(wrappedFactWithWeight(fact,weight,roundedValue))
for sumBindKey in boundSumKeys:
ancestor, contextHash, unit = sumBindKey
factKey = (sumConcept, ancestor, contextHash, unit)
if factKey in self.sumFacts:
sumFacts = self.sumFacts[factKey]
for fact in sumFacts:
if fact in self.duplicatedFacts:
dupBindingKeys.add(sumBindKey)
elif sumBindKey not in dupBindingKeys:
roundedSum = roundFact(fact, self.inferDecimals)
roundedItemsSum = roundFact(fact, self.inferDecimals, vDecimal=boundSums[sumBindKey])
if roundedItemsSum != roundFact(fact, self.inferDecimals):
d = inferredDecimals(fact)
if isnan(d) or isinf(d): d = 4
_boundSummationItems = boundSummationItems[sumBindKey]
unreportedContribingItemQnames = [] # list the missing/unreported contributors in relationship order
for modelRel in modelRels:
itemConcept = modelRel.toModelObject
if (itemConcept is not None and
(itemConcept, ancestor, contextHash, unit) not in self.itemFacts):
unreportedContribingItemQnames.append(str(itemConcept.qname))
self.modelXbrl.log('INCONSISTENCY', "xbrl.5.2.5.2:calcInconsistency",
_("Calculation inconsistent from %(concept)s in link role %(linkrole)s reported sum %(reportedSum)s computed sum %(computedSum)s context %(contextID)s unit %(unitID)s unreportedContributingItems %(unreportedContributors)s"),
modelObject=wrappedSummationAndItems(fact, roundedSum, _boundSummationItems),
concept=sumConcept.qname, linkrole=ELR,
linkroleDefinition=self.modelXbrl.roleTypeDefinition(ELR),
reportedSum=Locale.format_decimal(self.modelXbrl.locale, roundedSum, 1, max(d,0)),
computedSum=Locale.format_decimal(self.modelXbrl.locale, roundedItemsSum, 1, max(d,0)),
contextID=fact.context.id, unitID=fact.unit.id,
unreportedContributors=", ".join(unreportedContribingItemQnames) or "none")
del unreportedContribingItemQnames[:]
boundSummationItems.clear() # dereference facts in list
elif arcrole == XbrlConst.essenceAlias:
for modelRel in relsSet.modelRelationships:
essenceConcept = modelRel.fromModelObject
aliasConcept = modelRel.toModelObject
essenceBindingKeys = self.esAlConceptBindKeys[essenceConcept]
aliasBindingKeys = self.esAlConceptBindKeys[aliasConcept]
for esAlBindKey in essenceBindingKeys & aliasBindingKeys:
ancestor, contextHash = esAlBindKey
essenceFactsKey = (essenceConcept, ancestor, contextHash)
aliasFactsKey = (aliasConcept, ancestor, contextHash)
if essenceFactsKey in self.esAlFacts and aliasFactsKey in self.esAlFacts:
for eF in self.esAlFacts[essenceFactsKey]:
for aF in self.esAlFacts[aliasFactsKey]:
essenceUnit = self.mapUnit.get(eF.unit,eF.unit)
aliasUnit = self.mapUnit.get(aF.unit,aF.unit)
if essenceUnit != aliasUnit:
self.modelXbrl.log('INCONSISTENCY', "xbrl.5.2.6.2.2:essenceAliasUnitsInconsistency",
_("Essence-Alias inconsistent units from %(essenceConcept)s to %(aliasConcept)s in link role %(linkrole)s context %(contextID)s"),
modelObject=(modelRel, eF, aF),
essenceConcept=essenceConcept.qname, aliasConcept=aliasConcept.qname,
linkrole=ELR,
linkroleDefinition=self.modelXbrl.roleTypeDefinition(ELR),
contextID=eF.context.id)
if not XbrlUtil.vEqual(eF, aF):
self.modelXbrl.log('INCONSISTENCY', "xbrl.5.2.6.2.2:essenceAliasUnitsInconsistency",
_("Essence-Alias inconsistent value from %(essenceConcept)s to %(aliasConcept)s in link role %(linkrole)s context %(contextID)s"),
modelObject=(modelRel, eF, aF),
essenceConcept=essenceConcept.qname, aliasConcept=aliasConcept.qname,
linkrole=ELR,
linkroleDefinition=self.modelXbrl.roleTypeDefinition(ELR),
contextID=eF.context.id)
elif arcrole == XbrlConst.requiresElement:
for modelRel in relsSet.modelRelationships:
sourceConcept = modelRel.fromModelObject
requiredConcept = modelRel.toModelObject
if sourceConcept in self.requiresElementFacts and \
not requiredConcept in self.requiresElementFacts:
self.modelXbrl.log('INCONSISTENCY', "xbrl.5.2.6.2.4:requiresElementInconsistency",
_("Requires-Element %(requiringConcept)s missing required fact for %(requiredConcept)s in link role %(linkrole)s"),
modelObject=sourceConcept,
requiringConcept=sourceConcept.qname, requiredConcept=requiredConcept.qname,
linkrole=ELR,
linkroleDefinition=self.modelXbrl.roleTypeDefinition(ELR))
self.modelXbrl.profileActivity("... find inconsistencies", minTimeToShow=1.0)
self.modelXbrl.profileActivity() # reset
def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
global ArchiveFileIOError
if ArchiveFileIOError is None:
from arelle.FileSource import ArchiveFileIOError
unNamedCounter = 1
txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1",
"http://xbrl.org/PWD/2014-01-15/taxonomy-package",
"http://xbrl.org/PWD/2015-01-14/taxonomy-package",
"http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package")
catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog",)
pkg = {}
currentLang = Locale.getLanguageCode()
_file = filesource.file(metadataFile)[0] # URL in zip, plain file in file system or web
try:
tree = etree.parse(_file)
except etree.XMLSyntaxError as err:
cntlr.addToLog(_("Package catalog syntax error %(error)s"),
messageArgs={"error": str(err)},
messageCode="tpe:invalidMetaDataFile",
file=os.path.basename(metadataFile),
level=logging.ERROR)
errors.append("tpe:invalidMetaDataFile")
raise # reraise error
root = tree.getroot()
ns = root.tag.partition("}")[0][1:]
nsPrefix = "{{{}}}".format(ns)
if ns in txmyPkgNSes: # package file
for eltName in ("identifier", "version", "license", "publisher", "publisherURL", "publisherCountry", "publicationDate"):
pkg[eltName] = ''
for m in root.iterchildren(tag=nsPrefix + eltName):
if eltName == "license":
pkg[eltName] = m.get("name")
else:
pkg[eltName] = (m.text or "").strip()
break # take first entry if several
for eltName in ("name", "description"):
closest = ''
closestLen = 0
for m in root.iterchildren(tag=nsPrefix + eltName):
s = (m.text or "").strip()
l = langCloseness(xmlLang(m), currentLang)
if l > closestLen:
closestLen = l
closest = s
pkg[eltName] = closest
for eltName in ("supersededTaxonomyPackages", "versioningReports"):
pkg[eltName] = []
for m in root.iterchildren(tag=nsPrefix + "supersededTaxonomyPackages"):
pkg['supersededTaxonomyPackages'] = [
r.text.strip()
for r in m.iterchildren(tag=nsPrefix + "taxonomyPackageRef")]
for m in root.iterchildren(tag=nsPrefix + "versioningReports"):
pkg['versioningReports'] = [
r.get("href")
for r in m.iterchildren(tag=nsPrefix + "versioningReport")]
# check for duplicate multi-lingual elements (among children of nodes)
langElts = defaultdict(list)
for n in root.iter(tag=nsPrefix + "*"):
for eltName in ("name", "description"):
langElts.clear()
for m in n.iterchildren(tag=nsPrefix + eltName):
langElts[xmlLang(m)].append(m)
for lang, elts in langElts.items():
if not lang:
cntlr.addToLog(_("Multi-lingual element %(element)s has no in-scope xml:lang attribute"),
messageArgs={"element": eltName},
messageCode="tpe:missingLanguageAttribute",
refs=[{"href":os.path.basename(metadataFile), "sourceLine":m.sourceline} for m in elts],
level=logging.ERROR)
errors.append("tpe:missingLanguageAttribute")
elif len(elts) > 1:
cntlr.addToLog(_("Multi-lingual element %(element)s has multiple (%(count)s) in-scope xml:lang %(lang)s elements"),
messageArgs={"element": eltName, "lang": lang, "count": len(elts)},
messageCode="tpe:duplicateLanguagesForElement",
refs=[{"href":os.path.basename(metadataFile), "sourceLine":m.sourceline} for m in elts],
level=logging.ERROR)
errors.append("tpe:duplicateLanguagesForElement")
del langElts # dereference
else: # oasis catalog, use dirname as the package name
# metadataFile may be a File object (with name) or string filename
fileName = getattr(metadataFile, 'fileName', # for FileSource named objects
getattr(metadataFile, 'name', # for io.file named objects
metadataFile)) # for string
pkg["name"] = os.path.basename(os.path.dirname(fileName))
pkg["description"] = "oasis catalog"
pkg["version"] = "(none)"
remappings = {}
rewriteTree = tree
catalogFile = metadataFile
if ns in ("http://xbrl.org/PWD/2015-01-14/taxonomy-package",
"http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package"):
catalogFile = metadataFile.replace('taxonomyPackage.xml','catalog.xml')
try:
rewriteTree = etree.parse(filesource.file(catalogFile)[0])
except ArchiveFileIOError:
pass
for tag, prefixAttr, replaceAttr in (
(nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package
("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem", "systemIdStartString", "rewritePrefix"),
("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI", "uriStartString", "rewritePrefix")): # oasis catalog
for m in rewriteTree.iter(tag=tag):
prefixValue = m.get(prefixAttr)
replaceValue = m.get(replaceAttr)
if prefixValue and replaceValue is not None:
if prefixValue not in remappings:
base = baseForElement(m)
if base:
replaceValue = os.path.join(base, replaceValue)
if replaceValue: # neither None nor ''
if not isAbsolute(replaceValue):
if not os.path.isabs(replaceValue):
replaceValue = fileBase + replaceValue
replaceValue = replaceValue.replace("/", os.sep)
_normedValue = os.path.normpath(replaceValue)
if replaceValue.endswith(os.sep) and not _normedValue.endswith(os.sep):
_normedValue += os.sep
remappings[prefixValue] = _normedValue
else:
cntlr.addToLog(_("Package catalog duplicate rewrite start string %(rewriteStartString)s"),
messageArgs={"rewriteStartString": prefixValue},
messageCode="tpe:multipleRewriteURIsForStartString",
file=os.path.basename(catalogFile),
level=logging.ERROR)
errors.append("tpe:multipleRewriteURIsForStartString")
pkg["remappings"] = remappings
nameToUrls = {}
pkg["nameToUrls"] = nameToUrls
for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"):
name = None
closestLen = 0
# find closest match name node given xml:lang match to current language or no xml:lang
for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"):
s = (nameNode.text or "").strip()
l = langCloseness(xmlLang(nameNode), currentLang)
if l > closestLen:
closestLen = l
name = s
if not name:
name = _("<unnamed {0}>").format(unNamedCounter)
unNamedCounter += 1
epDocCount = 0
for epDoc in entryPointSpec.iterchildren(nsPrefix + "entryPointDocument"):
epUrl = epDoc.get('href')
base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base') # cope with xml:base
if base:
resolvedUrl = urljoin(base, epUrl)
else:
resolvedUrl = epUrl
if epDocCount:
cntlr.addToLog(_("Skipping multiple-document entry point (not supported) %(href)s"),
messageArgs={"href": epUrl},
messageCode="arelle.packageMultipleDocumentEntryPoints",
file=os.path.basename(metadataFile),
level=logging.WARNING)
errors.append("arelle.packageMultipleDocumentEntryPoints")
continue
epDocCount += 1
#perform prefix remappings
remappedUrl = resolvedUrl
longestPrefix = 0
for mapFrom, mapTo in remappings.items():
if remappedUrl.startswith(mapFrom):
prefixLength = len(mapFrom)
if prefixLength > longestPrefix:
_remappedUrl = remappedUrl[prefixLength:]
if not (_remappedUrl[0] in (os.sep, '/') or mapTo[-1] in (os.sep, '/')):
_remappedUrl = mapTo + os.sep + _remappedUrl
else:
_remappedUrl = mapTo + _remappedUrl
longestPrefix = prefixLength
if longestPrefix:
remappedUrl = _remappedUrl.replace(os.sep, "/") # always used as FileSource select
nameToUrls[name] = (remappedUrl, resolvedUrl)
return pkg
def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
global ArchiveFileIOError
if ArchiveFileIOError is None:
from arelle.FileSource import ArchiveFileIOError
unNamedCounter = 1
txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1",
"http://xbrl.org/PWD/2014-01-15/taxonomy-package",
"http://xbrl.org/PWD/2015-01-14/taxonomy-package",
"http://xbrl.org/PR/2015-12-09/taxonomy-package",
"http://xbrl.org/2016/taxonomy-package",
"http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package")
catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog", )
pkg = {}
currentLang = Locale.getLanguageCode()
_file = filesource.file(metadataFile)[
0] # URL in zip, plain file in file system or web
parser = lxmlResolvingParser(cntlr)
try:
tree = etree.parse(_file, parser=parser)
# schema validate tp xml
xsdTree = etree.parse(TP_XSD, parser=parser)
etree.XMLSchema(xsdTree).assertValid(tree)
except (etree.XMLSyntaxError, etree.DocumentInvalid) as err:
cntlr.addToLog(_("Taxonomy package file syntax error %(error)s"),
messageArgs={"error": str(err)},
messageCode="tpe:invalidMetaDataFile",
file=os.path.basename(metadataFile),
level=logging.ERROR)
errors.append("tpe:invalidMetaDataFile")
return pkg
root = tree.getroot()
ns = root.tag.partition("}")[0][1:]
nsPrefix = "{{{}}}".format(ns)
if ns in txmyPkgNSes: # package file
for eltName in ("identifier", "version", "license", "publisher",
"publisherURL", "publisherCountry", "publicationDate"):
pkg[eltName] = ''
for m in root.iterchildren(tag=nsPrefix + eltName):
if eltName == "license":
pkg[eltName] = m.get("name")
else:
pkg[eltName] = (m.text or "").strip()
break # take first entry if several
for eltName in ("name", "description"):
closest = ''
closestLen = 0
for m in root.iterchildren(tag=nsPrefix + eltName):
s = (m.text or "").strip()
eltLang = xmlLang(m)
l = langCloseness(eltLang, currentLang)
if l > closestLen:
closestLen = l
closest = s
elif closestLen == 0 and eltLang.startswith("en"):
closest = s # pick english if nothing better
if not closest and eltName == "name": # assign default name when none in taxonomy package
closest = os.path.splitext(os.path.basename(
filesource.baseurl))[0]
pkg[eltName] = closest
for eltName in ("supersededTaxonomyPackages", "versioningReports"):
pkg[eltName] = []
for m in root.iterchildren(tag=nsPrefix +
"supersededTaxonomyPackages"):
pkg['supersededTaxonomyPackages'] = [
r.text.strip()
for r in m.iterchildren(tag=nsPrefix + "taxonomyPackageRef")
]
for m in root.iterchildren(tag=nsPrefix + "versioningReports"):
pkg['versioningReports'] = [
r.get("href")
for r in m.iterchildren(tag=nsPrefix + "versioningReport")
]
# check for duplicate multi-lingual elements (among children of nodes)
langElts = defaultdict(list)
for n in root.iter(tag=nsPrefix + "*"):
for eltName in ("name", "description", "publisher"):
langElts.clear()
for m in n.iterchildren(tag=nsPrefix + eltName):
langElts[xmlLang(m)].append(m)
for lang, elts in langElts.items():
if not lang:
cntlr.addToLog(
_("Multi-lingual element %(element)s has no in-scope xml:lang attribute"
),
messageArgs={"element": eltName},
messageCode="tpe:missingLanguageAttribute",
refs=[{
"href": os.path.basename(metadataFile),
"sourceLine": m.sourceline
} for m in elts],
level=logging.ERROR)
errors.append("tpe:missingLanguageAttribute")
elif len(elts) > 1:
cntlr.addToLog(
_("Multi-lingual element %(element)s has multiple (%(count)s) in-scope xml:lang %(lang)s elements"
),
messageArgs={
"element": eltName,
"lang": lang,
"count": len(elts)
},
messageCode="tpe:duplicateLanguagesForElement",
refs=[{
"href": os.path.basename(metadataFile),
"sourceLine": m.sourceline
} for m in elts],
level=logging.ERROR)
errors.append("tpe:duplicateLanguagesForElement")
del langElts # dereference
else: # oasis catalog, use dirname as the package name
# metadataFile may be a File object (with name) or string filename
fileName = getattr(
metadataFile,
'fileName', # for FileSource named objects
getattr(
metadataFile,
'name', # for io.file named objects
metadataFile)) # for string
pkg["name"] = os.path.basename(os.path.dirname(fileName))
pkg["description"] = "oasis catalog"
pkg["version"] = "(none)"
remappings = {}
rewriteTree = tree
catalogFile = metadataFile
if ns in ("http://xbrl.org/PWD/2015-01-14/taxonomy-package",
"http://xbrl.org/PR/2015-12-09/taxonomy-package",
"http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package",
"http://xbrl.org/2016/taxonomy-package",
"http://xbrl.org/REC/2016-04-19/taxonomy-package"):
catalogFile = metadataFile.replace('taxonomyPackage.xml',
'catalog.xml')
try:
rewriteTree = etree.parse(filesource.file(catalogFile)[0],
parser=parser)
# schema validate tp xml
xsdTree = etree.parse(CAT_XSD, parser=parser)
etree.XMLSchema(xsdTree).assertValid(rewriteTree)
except (etree.XMLSyntaxError, etree.DocumentInvalid) as err:
cntlr.addToLog(_("Catalog file syntax error %(error)s"),
messageArgs={"error": str(err)},
messageCode="tpe:invalidCatalogFile",
file=os.path.basename(metadataFile),
level=logging.ERROR)
errors.append("tpe:invalidCatalogFile")
except ArchiveFileIOError:
pass
for tag, prefixAttr, replaceAttr in (
(nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package
("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem",
"systemIdStartString", "rewritePrefix"),
("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI",
"uriStartString", "rewritePrefix")): # oasis catalog
for m in rewriteTree.iter(tag=tag):
prefixValue = m.get(prefixAttr)
replaceValue = m.get(replaceAttr)
if prefixValue and replaceValue is not None:
if prefixValue not in remappings:
base = baseForElement(m)
if base:
replaceValue = os.path.join(base, replaceValue)
if replaceValue: # neither None nor ''
if not isAbsolute(replaceValue):
if not os.path.isabs(replaceValue):
replaceValue = fileBase + replaceValue
replaceValue = replaceValue.replace("/", os.sep)
_normedValue = cntlr.webCache.normalizeUrl(replaceValue)
if replaceValue.endswith(
os.sep) and not _normedValue.endswith(os.sep):
_normedValue += os.sep
remappings[prefixValue] = _normedValue
else:
cntlr.addToLog(
_("Package catalog duplicate rewrite start string %(rewriteStartString)s"
),
messageArgs={"rewriteStartString": prefixValue},
messageCode="tpe:multipleRewriteURIsForStartString",
file=os.path.basename(catalogFile),
level=logging.ERROR)
errors.append("tpe:multipleRewriteURIsForStartString")
pkg["remappings"] = remappings
entryPoints = defaultdict(list)
pkg["entryPoints"] = entryPoints
for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"):
name = None
closestLen = 0
# find closest match name node given xml:lang match to current language or no xml:lang
for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"):
s = (nameNode.text or "").strip()
nameLang = xmlLang(nameNode)
l = langCloseness(nameLang, currentLang)
if l > closestLen:
closestLen = l
name = s
elif closestLen == 0 and nameLang.startswith("en"):
name = s # pick english if nothing better
if not name:
name = _("<unnamed {0}>").format(unNamedCounter)
unNamedCounter += 1
epDocCount = 0
for epDoc in entryPointSpec.iterchildren(nsPrefix +
"entryPointDocument"):
epUrl = epDoc.get('href')
base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base'
) # cope with xml:base
if base:
resolvedUrl = urljoin(base, epUrl)
else:
resolvedUrl = epUrl
epDocCount += 1
#perform prefix remappings
remappedUrl = resolvedUrl
longestPrefix = 0
for mapFrom, mapTo in remappings.items():
if remappedUrl.startswith(mapFrom):
prefixLength = len(mapFrom)
if prefixLength > longestPrefix:
_remappedUrl = remappedUrl[prefixLength:]
if not (_remappedUrl[0] in (os.sep, '/')
or mapTo[-1] in (os.sep, '/')):
_remappedUrl = mapTo + os.sep + _remappedUrl
else:
_remappedUrl = mapTo + _remappedUrl
longestPrefix = prefixLength
if longestPrefix:
remappedUrl = _remappedUrl.replace(
os.sep, "/") # always used as FileSource select
# find closest language description
closest = ''
closestLen = 0
for m in entryPointSpec.iterchildren(tag=nsPrefix + "description"):
s = (m.text or "").strip()
eltLang = xmlLang(m)
l = langCloseness(eltLang, currentLang)
if l > closestLen:
closestLen = l
closest = s
elif closestLen == 0 and eltLang.startswith("en"):
closest = s # pick english if nothing better
if not closest and name: # assign default name when none in taxonomy package
closest = name
entryPoints[name].append((remappedUrl, resolvedUrl, closest))
return pkg
def parsePackage(mainWin, metadataFile):
unNamedCounter = 1
txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1",
"http://xbrl.org/PWD/2014-01-15/taxonomy-package")
catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog", )
pkg = {}
currentLang = Locale.getLanguageCode()
tree = etree.parse(metadataFile)
root = tree.getroot()
ns = root.tag.partition("}")[0][1:]
nsPrefix = "{{{}}}".format(ns)
if ns in txmyPkgNSes: # package file
for eltName in ("name", "description", "version"):
pkg[eltName] = ''
for m in root.iterchildren(tag=nsPrefix + eltName):
pkg[eltName] = m.text.strip()
break # take first entry if several
else: # oasis catalog, use dirname as the package name
# metadataFile may be a File object (with name) or string filename
fileName = getattr(
metadataFile,
'fileName', # for FileSource named objects
getattr(
metadataFile,
'name', # for io.file named objects
metadataFile)) # for string
pkg["name"] = os.path.basename(os.path.dirname(fileName))
pkg["description"] = "oasis catalog"
pkg["version"] = "(none)"
remappings = {}
for tag, prefixAttr, replaceAttr in (
(nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package
(nsPrefix + "rewriteSystem", "systemIdStartString",
"rewritePrefix")): # oasis catalog
for m in tree.iter(tag=tag):
prefixValue = m.get(prefixAttr)
replaceValue = m.get(replaceAttr)
if prefixValue and replaceValue is not None:
remappings[prefixValue] = replaceValue
pkg["remappings"] = remappings
nameToUrls = {}
pkg["nameToUrls"] = nameToUrls
for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"):
name = None
# find closest match name node given xml:lang match to current language or no xml:lang
for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"):
xmlLang = nameNode.get(
'{http://www.w3.org/XML/1998/namespace}lang')
if name is None or not xmlLang or currentLang == xmlLang:
name = nameNode.text
if currentLang == xmlLang: # most prefer one with the current locale's language
break
if not name:
name = _("<unnamed {0}>").format(unNamedCounter)
unNamedCounter += 1
epDocCount = 0
for epDoc in entryPointSpec.iterchildren(nsPrefix +
"entryPointDocument"):
if epDocCount:
mainWin.addToLog(
_("WARNING: skipping multiple-document entry point (not supported)"
))
continue
epDocCount += 1
epUrl = epDoc.get('href')
base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base'
) # cope with xml:base
if base:
resolvedUrl = urljoin(base, epUrl)
else:
resolvedUrl = epUrl
#perform prefix remappings
remappedUrl = resolvedUrl
for prefix, replace in remappings.items():
remappedUrl = remappedUrl.replace(prefix, replace, 1)
nameToUrls[name] = (remappedUrl, resolvedUrl)
return pkg
def backgroundSaveInstance(self, newFilename=None):
cntlr = self.modelXbrl.modelManager.cntlr
if newFilename and self.modelXbrl.modelDocument.type != ModelDocument.Type.INSTANCE:
self.modelXbrl.modelManager.showStatus(_("creating new instance {0}").format(os.path.basename(newFilename)))
self.modelXbrl.modelManager.cntlr.waitForUiThreadQueue() # force status update
self.modelXbrl.createInstance(newFilename) # creates an instance as this modelXbrl's entrypoing
instance = self.modelXbrl
cntlr.showStatus(_("Saving {0}").format(instance.modelDocument.basename))
cntlr.waitForUiThreadQueue() # force status update
newCntx = ModelXbrl.AUTO_LOCATE_ELEMENT
newUnit = ModelXbrl.AUTO_LOCATE_ELEMENT
# check user keyed changes
for bodyCell in self.gridBody.winfo_children():
if isinstance(bodyCell, gridCell) and bodyCell.isChanged:
value = bodyCell.value
objId = bodyCell.objectId
if objId:
if objId[0] == "f":
factPrototypeIndex = int(objId[1:])
factPrototype = self.factPrototypes[factPrototypeIndex]
concept = factPrototype.concept
entityIdentScheme = self.newFactItemOptions.entityIdentScheme
entityIdentValue = self.newFactItemOptions.entityIdentValue
periodType = factPrototype.concept.periodType
periodStart = self.newFactItemOptions.startDateDate if periodType == "duration" else None
periodEndInstant = self.newFactItemOptions.endDateDate
qnameDims = factPrototype.context.qnameDims
prevCntx = instance.matchContext(
entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant,
qnameDims, [], [])
if prevCntx is not None:
cntxId = prevCntx.id
else: # need new context
newCntx = instance.createContext(entityIdentScheme, entityIdentValue,
periodType, periodStart, periodEndInstant,
concept.qname, qnameDims, [], [],
afterSibling=newCntx)
cntxId = newCntx.id
# new context
if concept.isNumeric:
if concept.isMonetary:
unitMeasure = qname(XbrlConst.iso4217, self.newFactItemOptions.monetaryUnit)
unitMeasure.prefix = "iso4217" # want to save with a recommended prefix
decimals = self.newFactItemOptions.monetaryDecimals
elif concept.isShares:
unitMeasure = XbrlConst.qnXbrliShares
decimals = self.newFactItemOptions.nonMonetaryDecimals
else:
unitMeasure = XbrlConst.qnXbrliPure
decimals = self.newFactItemOptions.nonMonetaryDecimals
prevUnit = instance.matchUnit([unitMeasure],[])
if prevUnit is not None:
unitId = prevUnit.id
else:
newUnit = instance.createUnit([unitMeasure],[], afterSibling=newUnit)
unitId = newUnit.id
attrs = [("contextRef", cntxId)]
if concept.isNumeric:
attrs.append(("unitRef", unitId))
attrs.append(("decimals", decimals))
value = Locale.atof(self.modelXbrl.locale, value, str.strip)
newFact = instance.createFact(concept.qname, attributes=attrs, text=value)
bodyCell.objectId = newFact.objectId() # switch cell to now use fact ID
if self.factPrototypes[factPrototypeIndex] is not None:
self.factPrototypes[factPrototypeIndex].clear()
self.factPrototypes[factPrototypeIndex] = None #dereference fact prototype
else: # instance fact, not prototype
fact = self.modelXbrl.modelObject(objId)
if fact.concept.isNumeric:
value = Locale.atof(self.modelXbrl.locale, value, str.strip)
if fact.value != value:
if fact.concept.isNumeric and fact.isNil != (not value):
fact.isNil = not value
if value: # had been nil, now it needs decimals
fact.decimals = (self.newFactItemOptions.monetaryDecimals
if fact.concept.isMonetary else
self.newFactItemOptions.nonMonetaryDecimals)
fact.text = value
XmlValidate.validate(instance, fact)
bodyCell.isChanged = False # clear change flag
instance.saveInstance(newFilename) # may override prior filename for instance from main menu
cntlr.showStatus(_("Saved {0}").format(instance.modelDocument.basename), clearAfter=3000)
def factCheck(val, fact):
concept = fact.concept
context = fact.context
if concept is None or context is None or not val.validateLoggingSemantic:
return # not checkable
try:
if fact.isNumeric:
# 2.3.3 additional unit tests beyond UTR spec
unit = fact.unit
if unit is not None and concept.type is not None and val.validateUTR:
typeName = concept.type.name
if typeName == "perUnitItemType" and any(m.namespaceURI == XbrlConst.iso4217 or
m in (XbrlConst.qnXbrliPure, XbrlConst.qnXbrliShares)
for m in unit.measures[1]):
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.3.3.perUnitItemType",
_("PureItemType fact %(fact)s in context %(contextID)s unit %(unitID)s value %(value)s has disallowed unit denominator %(denominator)s"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID,
value=fact.effectiveValue, denominator=", ".join((str(m) for m in unit.measures[1])))
if not fact.isNil and getattr(fact, "xValue", None) is not None:
# 2.4.1 decimal disagreement
if fact.decimals and fact.decimals != "INF":
vf = float(fact.value)
if _ISFINITE(vf):
dec = _INT(fact.decimals)
vround = round(vf, dec)
if vf != vround:
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.4.1",
_("Decimal disagreement %(fact)s in context %(contextID)s unit %(unitID)s value %(value)s has insignificant value %(insignificantValue)s"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID,
value=fact.effectiveValue, insignificantValue=Locale.format(val.modelXbrl.locale, "%.*f",
(dec + 2 if dec > 0 else 0, vf - vround),
True))
# 2.5.1 fractions disallowed on a disclosure
if fact.isFraction:
if any(val.linroleDefinitionIsDisclosure.match(roleType.definition)
for rel in val.modelXbrl.relationshipSet(XbrlConst.parentChild).toModelObject(concept)
for roleType in val.modelXbrl.roleTypes.get(rel.linkrole,())):
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.5.1",
_("Disclosure %(fact)s in context %(contextID)s value %(value)s is a fraction"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, value=fact.value)
# deprecated concept
if concept.qname.namespaceURI == val.ugtNamespace:
if concept.name in val.usgaapDeprecations:
val.deprecatedFactConcepts[concept].append(fact)
elif concept.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedFactConcepts[concept].append(fact)
if fact.isItem and fact.context is not None:
for dimConcept, modelDim in fact.context.segDimValues.items():
if dimConcept.qname.namespaceURI == val.ugtNamespace:
if dimConcept.name in val.usgaapDeprecations:
val.deprecatedDimensions[dimConcept].append(fact)
elif dimConcept.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedDimensions[dimConcept].append(fact)
if modelDim.isExplicit:
member = modelDim.member
if member is not None:
if member.qname.namespaceURI == val.ugtNamespace:
if member.name in val.usgaapDeprecations:
val.deprecatedMembers[member].append(fact)
elif member.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"):
val.deprecatedMembers[member].append(fact)
except Exception as err:
val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.testingException",
_("%(fact)s in context %(contextID)s unit %(unitID)s value %(value)s cannot be tested due to: %(err)s"),
modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID,
value=fact.effectiveValue, err=err)
