def backgroundProfileFormula(cntlr, profileReportFile, maxRunTime, excludeCompileTime): from arelle import Locale, XPathParser, ValidateXbrlDimensions, ValidateFormula # build grammar before profiling (if this is the first pass, so it doesn't count in profile statistics) XPathParser.initializeParser(cntlr.modelManager) # load dimension defaults ValidateXbrlDimensions.loadDimensionDefaults(cntlr.modelManager) import cProfile, pstats, sys, time # a minimal validation class for formula validator parameters that are needed class Validate: def __init__(self, modelXbrl, maxRunTime): self.modelXbrl = modelXbrl self.parameters = None self.validateSBRNL = False self.maxFormulaRunTime = maxRunTime def close(self): self.__dict__.clear() val = Validate(cntlr.modelManager.modelXbrl, maxRunTime) formulaOptions = val.modelXbrl.modelManager.formulaOptions if excludeCompileTime: startedAt = time.time() cntlr.addToLog(_("pre-compiling formulas before profiling")) val.validateFormulaCompileOnly = True ValidateFormula.validate(val) del val.validateFormulaCompileOnly cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale, _("formula pre-compiling completed in %.2f secs"), time.time() - startedAt)) cntlr.addToLog(_("executing formulas for profiling")) else: cntlr.addToLog(_("compiling and executing formulas for profiling")) startedAt = time.time() statsFile = profileReportFile + ".bin" cProfile.runctx("ValidateFormula.validate(val)", globals(), locals(), statsFile) cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale, _("formula profiling completed in %.2f secs"), time.time() - startedAt)) # dereference val val.close() # specify a file for log priorStdOut = sys.stdout sys.stdout = open(profileReportFile, "w") statObj = pstats.Stats(statsFile) statObj.strip_dirs() statObj.sort_stats("time") statObj.print_stats() statObj.print_callees() statObj.print_callers() sys.stdout.flush() sys.stdout.close() del statObj sys.stdout = priorStdOut os.remove(statsFile)
def formatInterval(self, a, b, dec): if a is NIL: return "(nil)" if isnan(dec) or isinf(dec): dec = 4 if a == b: # not an interval return Locale.format_decimal(self.modelXbrl.locale, a, 1, max(dec,0)) return "[{}, {}]".format( # show as an interval Locale.format_decimal(self.modelXbrl.locale, a, 1, max(dec,0)), Locale.format_decimal(self.modelXbrl.locale, b, 1, max(dec,0)))
def formatInterval(self, a, b, dec): if a is NIL: return "(nil)" if isnan(dec) or isinf(dec): dec = 4 if a == b: # not an interval return Locale.format_decimal(self.modelXbrl.locale, a, 1, max(dec, 0)) return "[{}, {}]".format( # show as an interval Locale.format_decimal(self.modelXbrl.locale, a, 1, max(dec, 0)), Locale.format_decimal(self.modelXbrl.locale, b, 1, max(dec, 0)))
def __init__(self, cntlr): self.cntlr = cntlr self.validateDisclosureSystem = False self.disclosureSystem = DisclosureSystem.DisclosureSystem(self) self.validateCalcLB = False self.validateInferDecimals = False self.validateUtr = False self.loadedModelXbrls = [] from arelle import Locale self.locale = Locale.getUserLocale() self.defaultLang = Locale.getLanguageCode()
def __init__(self, cntlr): self.cntlr = cntlr self.validateDisclosureSystem = False self.disclosureSystem = DisclosureSystem.DisclosureSystem(self) self.validateCalcLB = False self.validateInferDecimals = False self.validateInfoset = False self.validateUtr = False self.abortOnMajorError = False self.collectProfileStats = False self.loadedModelXbrls = [] from arelle import Locale self.locale = Locale.getUserLocale(cntlr.config.get("userInterfaceLocaleOverride","")) self.defaultLang = Locale.getLanguageCode()
def __init__(self, cntlr): self.cntlr = cntlr self.validateDisclosureSystem = False self.disclosureSystem = DisclosureSystem.DisclosureSystem(self) self.validateCalcLB = False self.validateInferDecimals = False self.validateInfoset = False self.validateUtr = False self.skipDTS = False self.abortOnMajorError = False self.collectProfileStats = False self.loadedModelXbrls = [] from arelle import Locale self.locale = Locale.getUserLocale(cntlr.config.get("userInterfaceLocaleOverride","")) self.defaultLang = Locale.getLanguageCode()
def profilerCommandLineRun(cntlr, options, sourceZipStream=None, **kwargs): from arelle import Locale import cProfile, pstats, sys, time profileReportFile = getattr(options, "profilerReportFile", None) if profileReportFile and not getattr(cntlr, "blockNestedProfiling", False): startedAt = time.time() cntlr.addToLog(_("invoking command processing under profiler")) statsFile = profileReportFile + ".bin" cntlr.blockNestedProfiling = True cProfile.runctx("cntlr.run(options, sourceZipStream)", globals(), locals(), statsFile) cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale, _("profiled command processing completed in %.2f secs"), time.time() - startedAt)) # specify a file for log priorStdOut = sys.stdout sys.stdout = open(profileReportFile, "w") statObj = pstats.Stats(statsFile) statObj.strip_dirs() statObj.sort_stats("time") statObj.print_stats() statObj.print_callees() statObj.print_callers() sys.stdout.flush() sys.stdout.close() del statObj sys.stdout = priorStdOut os.remove(statsFile) del cntlr.blockNestedProfiling sys.exit() # raise SYSTEM_EXIT to stop outer execution
def profilerCommandLineRun(cntlr, options, sourceZipStream=None, **kwargs): from arelle import Locale import cProfile, pstats, sys, time profileReportFile = getattr(options, "profilerReportFile", None) if profileReportFile and not getattr(cntlr, "blockNestedProfiling", False): startedAt = time.time() cntlr.addToLog(_("invoking command processing under profiler")) statsFile = profileReportFile + ".bin" cntlr.blockNestedProfiling = True cProfile.runctx("cntlr.run(options, sourceZipStream)", globals(), locals(), statsFile) cntlr.addToLog( Locale.format_string( cntlr.modelManager.locale, _("profiled command processing completed in %.2f secs"), time.time() - startedAt)) # specify a file for log priorStdOut = sys.stdout sys.stdout = open(profileReportFile, "w") statObj = pstats.Stats(statsFile) statObj.strip_dirs() statObj.sort_stats("time") statObj.print_stats() statObj.print_callees() statObj.print_callers() sys.stdout.flush() sys.stdout.close() del statObj sys.stdout = priorStdOut os.remove(statsFile) del cntlr.blockNestedProfiling sys.exit() # raise SYSTEM_EXIT to stop outer execution
def parsePackage(mainWin, metadataFile): unNamedCounter = 1 NS = "{http://www.corefiling.com/xbrl/taxonomypackage/v1}" pkg = {} currentLang = Locale.getLanguageCode() tree = etree.parse(metadataFile) root = tree.getroot() for eltName in ("name", "description", "version"): pkg[eltName] = "" for m in root.iterchildren(tag=NS + eltName): pkg[eltName] = m.text.strip() break # take first entry if several remappings = dict((m.get("prefix"), m.get("replaceWith")) for m in tree.iter(tag=NS + "remapping")) pkg["remappings"] = remappings nameToUrls = {} pkg["nameToUrls"] = nameToUrls for entryPointSpec in tree.iter(tag=NS + "entryPoint"): name = None # find closest match name node given xml:lang match to current language or no xml:lang for nameNode in entryPointSpec.iter(tag=NS + "name"): xmlLang = nameNode.get("{http://www.w3.org/XML/1998/namespace}lang") if name is None or not xmlLang or currentLang == xmlLang: name = nameNode.text if currentLang == xmlLang: # most prefer one with the current locale's language break if not name: name = _("<unnamed {0}>").format(unNamedCounter) unNamedCounter += 1 epDocCount = 0 for epDoc in entryPointSpec.iterchildren(NS + "entryPointDocument"): if epDocCount: mainWin.addToLog(_("WARNING: skipping multiple-document entry point (not supported)")) continue epDocCount += 1 epUrl = epDoc.get("href") base = epDoc.get("{http://www.w3.org/XML/1998/namespace}base") # cope with xml:base if base: resolvedUrl = urljoin(base, epUrl) else: resolvedUrl = epUrl # perform prefix remappings remappedUrl = resolvedUrl for prefix, replace in remappings.items(): remappedUrl = remappedUrl.replace(prefix, replace, 1) nameToUrls[name] = (remappedUrl, resolvedUrl) return pkg
def parseTxmyPkg(mainWin, metadataFile): unNamedCounter = 1 currentLang = Locale.getLanguageCode() tree = etree.parse(metadataFile) remappings = dict( (m.get("prefix"), m.get("replaceWith")) for m in tree.iter( tag="{http://www.corefiling.com/xbrl/taxonomypackage/v1}remapping") ) result = {} for entryPointSpec in tree.iter( tag="{http://www.corefiling.com/xbrl/taxonomypackage/v1}entryPoint" ): name = None # find closest match name node given xml:lang match to current language or no xml:lang for nameNode in entryPointSpec.iter( tag="{http://www.corefiling.com/xbrl/taxonomypackage/v1}name"): xmlLang = nameNode.get( '{http://www.w3.org/XML/1998/namespace}lang') if name is None or not xmlLang or currentLang == xmlLang: name = nameNode.text if currentLang == xmlLang: # most prefer one with the current locale's language break if not name: name = _("<unnamed {0}>").format(unNamedCounter) unNamedCounter += 1 epDocCount = 0 for epDoc in entryPointSpec.iterchildren( "{http://www.corefiling.com/xbrl/taxonomypackage/v1}entryPointDocument" ): if epDocCount: mainWin.addToLog( _("WARNING: skipping multiple-document entry point (not supported)" )) continue epDocCount += 1 epUrl = epDoc.get('href') base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base' ) # cope with xml:base if base: resolvedUrl = urljoin(base, epUrl) else: resolvedUrl = epUrl #perform prefix remappings remappedUrl = resolvedUrl for prefix, replace in remappings.items(): remappedUrl = resolvedUrl.replace(prefix, replace, 1) result[name] = (remappedUrl, resolvedUrl) return result
def factCheck(val, fact): concept = fact.concept context = fact.context if concept is None or context is None: return # not checkable try: if fact.isNumeric and not fact.isNil and fact.xValue is not None: # 2.4.1 decimal disagreement if fact.decimals and fact.decimals != "INF": vf = float(fact.value) if _ISFINITE(vf): dec = _INT(fact.decimals) vround = round(vf, dec) if vf != vround: val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.4.1", _("Decimal disagreement %(fact)s in context %(contextID)s unit %(unitID)s value %(value)s has insignificant value %(insignificantValue)s"), modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID, value=fact.effectiveValue, insignificantValue=Locale.format(val.modelXbrl.locale, "%.*f", (dec + 2 if dec > 0 else 0, vf - vround), True)) # 2.5.1 fractions disallowed on a disclosure if fact.isFraction: if any(val.linroleDefinitionIsDisclosure.match(roleType.definition) for rel in val.modelXbrl.relationshipSet(XbrlConst.parentChild).toModelObject(concept) for roleType in val.modelXbrl.roleTypes.get(rel.linkrole,())): val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.5.1", _("Disclosure %(fact)s in context %(contextID)s value %(value)s is a fraction"), modelObject=fact, fact=fact.qname, contextID=fact.contextID, value=fact.value) # deprecated concept if concept.qname.namespaceURI == ugtNamespace: if concept.name in val.usgaapDeprecations: val.deprecatedFactConcepts[concept].append(fact) elif concept.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"): val.deprecatedFactConcepts[concept].append(fact) if fact.isItem and fact.context is not None: for dimConcept, modelDim in fact.context.segDimValues.items(): if dimConcept.qname.namespaceURI == ugtNamespace: if dimConcept.name in val.usgaapDeprecations: val.deprecatedDimensions[dimConcept].append(fact) elif dimConcept.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"): val.deprecatedDimensions[dimConcept].append(fact) if modelDim.isExplicit: member = modelDim.member if member is not None: if member.qname.namespaceURI == ugtNamespace: if member.name in val.usgaapDeprecations: val.deprecatedMembers[member].append(fact) elif member.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"): val.deprecatedMembers[member].append(fact) except Exception as err: val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.testingException", _("%(fact)s in context %(contextID)s unit %(unitID)s value %(value)s cannot be tested due to: %(err)s"), modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID, value=fact.effectiveValue, err=err)
def label(self,preferredLabel=None,fallbackToQname=True,lang=None,strip=False,linkrole=None): if preferredLabel is None: preferredLabel = XbrlConst.standardLabel if preferredLabel == XbrlConst.conceptNameLabelRole: return str(self.qname) labelsRelationshipSet = self.modelXbrl.relationshipSet(XbrlConst.conceptLabel,linkrole) if labelsRelationshipSet: label = labelsRelationshipSet.label(self, preferredLabel, lang) if label is not None: if strip: return label.strip() return Locale.rtlString(label, lang=lang) return str(self.qname) if fallbackToQname else None
def parseTxmyPkg(mainWin, metadataFile): unNamedCounter = 1 currentLang = Locale.getLanguageCode() tree = etree.parse(metadataFile) remappings = dict( (m.get("prefix"), m.get("replaceWith")) for m in tree.iter(tag="{http://www.corefiling.com/xbrl/taxonomypackage/v1}remapping") ) result = {} for entryPointSpec in tree.iter(tag="{http://www.corefiling.com/xbrl/taxonomypackage/v1}entryPoint"): name = None # find closest match name node given xml:lang match to current language or no xml:lang for nameNode in entryPointSpec.iter(tag="{http://www.corefiling.com/xbrl/taxonomypackage/v1}name"): xmlLang = nameNode.get("{http://www.w3.org/XML/1998/namespace}lang") if name is None or not xmlLang or currentLang == xmlLang: name = nameNode.text if currentLang == xmlLang: # most prefer one with the current locale's language break if not name: name = _("<unnamed {0}>").format(unNamedCounter) unNamedCounter += 1 epDocCount = 0 for epDoc in entryPointSpec.iterchildren( "{http://www.corefiling.com/xbrl/taxonomypackage/v1}entryPointDocument" ): if epDocCount: mainWin.addToLog(_("WARNING: skipping multiple-document entry point (not supported)")) continue epDocCount += 1 epUrl = epDoc.get("href") base = epDoc.get("{http://www.w3.org/XML/1998/namespace}base") # cope with xml:base if base: resolvedUrl = urljoin(base, epUrl) else: resolvedUrl = epUrl # perform prefix remappings remappedUrl = resolvedUrl for prefix, replace in remappings.items(): remappedUrl = resolvedUrl.replace(prefix, replace, 1) result[name] = (remappedUrl, resolvedUrl) return result
def genLabel(self,role=None,fallbackToQname=False,fallbackToXlinkLabel=False,lang=None,strip=False,linkrole=None, linkroleHint=None): from arelle import XbrlConst if role is None: role = XbrlConst.genStandardLabel if role == XbrlConst.conceptNameLabelRole: return str(self.qname) labelsRelationshipSet = self.modelXbrl.relationshipSet(XbrlConst.elementLabel,linkrole) if labelsRelationshipSet: label = labelsRelationshipSet.label(self, linkroleHint or role, lang) if label is not None: if strip: return label.strip() return Locale.rtlString(label, lang=lang) if fallbackToQname: return str(self.qname) elif fallbackToXlinkLabel and hasattr(self,"xlinkLabel"): return self.xlinkLabel else: return None
def label(self, preferredLabel=None, fallbackToQname=True, lang=None, strip=False, linkrole=None): if preferredLabel is None: preferredLabel = XbrlConst.standardLabel if preferredLabel == XbrlConst.conceptNameLabelRole: return str(self.qname) labelsRelationshipSet = self.modelXbrl.relationshipSet( XbrlConst.conceptLabel, linkrole) if labelsRelationshipSet: label = labelsRelationshipSet.label(self, preferredLabel, lang) if label is not None: if strip: return label.strip() return Locale.rtlString(label, lang=lang) return str(self.qname) if fallbackToQname else None
def effectiveValue(self): concept = self.concept if concept is None or concept.isTuple: return None if self.isNil: return "(nil)" if concept.isNumeric: val = self.value try: num = float(val) dec = self.decimals if dec is None or dec == "INF": dec = len(val.partition(".")[2]) else: dec = int(dec) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here return Locale.format(self.modelXbrl.locale, "%.*f", (dec, num), True) except ValueError: return "(error)" return self.value
def effectiveValue(self): concept = self.concept if concept is None or concept.isTuple: return None if self.isNil: return "(nil)" if concept.isNumeric: val = self.value try: num = float(val) dec = self.decimals if dec is None or dec == "INF": dec = len(val.partition(".")[2]) else: dec = int( dec ) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here return Locale.format(self.modelXbrl.locale, "%.*f", (dec, num), True) except ValueError: return "(error)" return self.value
def effectiveValue(self): """(str) -- Effective value for views, (nil) if isNil, None if no value, locale-formatted string of decimal value (if decimals specified) , otherwise string value""" concept = self.concept if concept is None or concept.isTuple: return None if self.isNil: return "(nil)" try: if concept.isNumeric: val = self.value try: num = float(val) dec = self.decimals if dec is None or dec == "INF": dec = len(val.partition(".")[2]) else: dec = int(dec) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here return Locale.format(self.modelXbrl.locale, "%.*f", (dec, num), True) except ValueError: return "(error)" return self.value except Exception as ex: return str(ex) # could be transform value of inline fact
def genFact(dts, concept, preferredLabel, arcrole, relationshipSet, level, visited, elrInfo): try: if concept is not None: if concept.isHypercubeItem: elrInfo["inCube"] = level elrInfo["dims"] = {} elrInfo["lineItems"] =False elrInfo.pop("instant", None) elrInfo.pop("duration", None) elif concept.isDimensionItem: elrInfo["currentDim"] = concept if concept.isTypedDimension: elrInfo["dims"][concept.qname] = (concept, concept.typedDomainElement) if concept.typedDomainElement.isNumeric: elrInfo["domainIter"] = 1 elif concept.name.endswith("Member"): # don't generate entries for default dim (Domain) (for now) dimConcept = elrInfo["currentDim"] if dimConcept.qname not in elrInfo["dims"]: elrInfo["dims"][dimConcept.qname] = (dimConcept, concept) else: if concept.name.endswith("LineItems"): elrInfo["lineItems"] = True elif ((not elrInfo["inCube"] or # before any hypercube elrInfo["lineItems"]) # in Cube and within Line Items and not concept.isAbstract): # or within line items # generate a fact sampVals = sampleDataValues[elrInfo.get("domainIter",1)] # use first entry if no domain iter if concept.periodType not in elrInfo: qnameDims = {} for _dimConcept, _domConcept in elrInfo["dims"].values(): if _dimConcept.isExplicitDimension: _memVal = _domConcept.qname else: if _domConcept.type is not None and not _domConcept.isNumeric: _memEltVal = genSampleValue(sampVals, _domConcept) else: _memEltVal = str(elrInfo["domainIter"]) _memVal = XmlUtil.addChild(dts.modelDocument.xmlRootElement, _domConcept.qname, text=_memEltVal, appendChild=False) _dimObj = DimValuePrototype(dts, None, _dimConcept.qname, _memVal, "segment") qnameDims[_dimConcept.qname] = _dimObj elrInfo[concept.periodType] = dts.createContext( "http://treasury.gov", "entityId", concept.periodType, sampVals["periodStart"] if concept.periodType == "duration" else None, sampVals["periodEnd"], concept.qname, qnameDims, [], []) cntx = elrInfo[concept.periodType] cntxId = cntx.id if concept.isNumeric: if concept.isMonetary: unitMeasure = qname(XbrlConst.iso4217, "USD") unitMeasure.prefix = "iso4217" # want to save with a recommended prefix decimals = 2 elif concept.isShares: unitMeasure = XbrlConst.qnXbrliShares decimals = 0 else: unitMeasure = XbrlConst.qnXbrliPure decimals = 0 prevUnit = dts.matchUnit([unitMeasure], []) if prevUnit is not None: unitId = prevUnit.id else: newUnit = dts.createUnit([unitMeasure], []) unitId = newUnit.id value = genSampleValue(sampVals, concept) attrs = [("contextRef", cntxId)] if concept.isNumeric: attrs.append(("unitRef", unitId)) attrs.append(("decimals", decimals)) value = Locale.atof(dts.locale, str(value), str.strip) newFact = dts.createFact(concept.qname, attributes=attrs, text=value) if concept not in visited: visited.add(concept) rels = relationshipSet.fromModelObject(concept) lenRels = len(rels) iRel = 0 iFirstLineItem = None while iRel <= lenRels: if iRel == lenRels: # check if cube needs re-iterating if iFirstLineItem is None or elrInfo.get("domainIter",0) >= 2: break reIterateCube = True # cube can re-iterate else: modelRel = rels[iRel] toConcept = modelRel.toModelObject reIterateCube = (toConcept.isHypercubeItem and # finished prior line items and hitting next table iFirstLineItem is not None and elrInfo["lineItems"] and 1 <= elrInfo.get("domainIter",0) < 2) if reIterateCube: # repeat typed dim container iRel = iFirstLineItem elrInfo["domainIter"] += 1 elrInfo.pop("instant", None) # want new contexts for next iteration elrInfo.pop("duration", None) isFirstLineItem = not elrInfo["lineItems"] genFact(dts, toConcept, modelRel.preferredLabel, arcrole, relationshipSet, level+1, visited, elrInfo) if isFirstLineItem and elrInfo["lineItems"] and elrInfo.get("domainIter",0) > 0: iFirstLineItem = iRel iRel += 1 visited.remove(concept) except AttributeError as ex: # bad relationship print ("[exception] {}".format(ex)) return
def viewConcept(self, concept, modelObject, labelPrefix, preferredLabel, parentnode, n, relationshipSet, visited): if concept is None: return try: isRelation = isinstance(modelObject, ModelDtsObject.ModelRelationship) isModelTable = False filingIndicatorCode = "" if isinstance(concept, ModelDtsObject.ModelConcept): text = labelPrefix + concept.label( preferredLabel, lang=self.lang, linkroleHint=relationshipSet.linkrole) if (self.arcrole in ("XBRL-dimensions", XbrlConst.hypercubeDimension) and concept.isTypedDimension and concept.typedDomainElement is not None): text += " (typedDomain={0})".format( concept.typedDomainElement.qname) elif isinstance(concept, ModelInstanceObject.ModelFact): if concept.concept is not None: text = labelPrefix + concept.concept.label( preferredLabel, lang=self.lang, linkroleHint=relationshipSet.linkrole) else: text = str(concept.qname) if concept.contextID: text += " [" + concept.contextID + "] = " + concept.effectiveValue elif self.arcrole == "Table-rendering": text = concept.localName elif isinstance(concept, ModelRenderingObject.ModelTable): text = (concept.genLabel(lang=self.lang, strip=True) or concept.localName) isModelTable = True elif isinstance(concept, ModelDtsObject.ModelResource): if self.showReferences: text = (concept.viewText() or concept.localName) else: text = (Locale.rtlString(concept.textValue.strip(), lang=concept.xmlLang) or concept.localName) else: # just a resource text = concept.localName childnode = self.treeView.insert(parentnode, "end", modelObject.objectId(self.id), text=text, tags=("odd" if n & 1 else "even", )) # Check if we need special rendering of this item for pluginXbrlMethod in pluginClassMethods( "CntlrWinMain.Rendering.RenderConcept"): stopPlugin = pluginXbrlMethod(isModelTable, concept, text, self, self.modelXbrl, childnode) if stopPlugin == True: break childRelationshipSet = relationshipSet if self.arcrole == XbrlConst.parentChild: # extra columns if isRelation: preferredLabel = modelObject.preferredLabel if preferredLabel and preferredLabel.startswith( "http://www.xbrl.org/2003/role/"): preferredLabel = os.path.basename(preferredLabel) self.treeView.set(childnode, "preferredLabel", preferredLabel) self.treeView.set(childnode, "type", concept.niceType) self.treeView.set(childnode, "references", viewReferences(concept)) elif self.arcrole == XbrlConst.summationItem: if isRelation: self.treeView.set(childnode, "weight", "{:+0g} ".format(modelObject.weight)) self.treeView.set(childnode, "balance", concept.balance) elif self.arcrole == "XBRL-dimensions" and isRelation: # extra columns relArcrole = modelObject.arcrole self.treeView.set(childnode, "arcrole", os.path.basename(relArcrole)) if relArcrole in (XbrlConst.all, XbrlConst.notAll): self.treeView.set(childnode, "contextElement", modelObject.contextElement) self.treeView.set(childnode, "closed", modelObject.closed) elif relArcrole in (XbrlConst.dimensionDomain, XbrlConst.domainMember): self.treeView.set(childnode, "usable", modelObject.usable) childRelationshipSet = self.modelXbrl.relationshipSet( XbrlConst.consecutiveArcrole.get(relArcrole, "XBRL-dimensions"), modelObject.consecutiveLinkrole) elif self.arcrole == "Table-rendering": # extra columns try: header = concept.header(lang=self.lang, strip=True, evaluate=False) except AttributeError: header = None # could be a filter if isRelation and header is None: header = "{0} {1}".format( os.path.basename(modelObject.arcrole), concept.xlinkLabel) self.treeView.set(childnode, "header", header) if concept.get("abstract") == "true": self.treeView.set(childnode, "abstract", '\u2713') # checkmark unicode character if concept.get("merge") == "true": self.treeView.set(childnode, "merge", '\u2713') # checkmark unicode character if isRelation: self.treeView.set(childnode, "axis", modelObject.axisDisposition) if isinstance(concept, (ModelEuAxisCoord, ModelRuleDefinitionNode)): self.treeView.set( childnode, "priItem", concept.aspectValue(None, Aspect.CONCEPT)) self.treeView.set( childnode, "dims", ' '.join( ("{0},{1}".format( dim, concept.aspectValue(None, dim)) for dim in (concept.aspectValue( None, Aspect.DIMENSIONS, inherit=False) or [])))) elif self.isResourceArcrole: # resource columns if isRelation: self.treeView.set(childnode, "arcrole", os.path.basename(modelObject.arcrole)) if isinstance(concept, ModelDtsObject.ModelResource): self.treeView.set(childnode, "resource", concept.localName) self.treeView.set(childnode, "resourcerole", os.path.basename(concept.role or '')) self.treeView.set(childnode, "lang", concept.xmlLang) self.id += 1 self.tag_has[modelObject.objectId()].append(childnode) if isRelation: self.tag_has[modelObject.toModelObject.objectId()].append( childnode) if concept not in visited: visited.add(concept) for modelRel in childRelationshipSet.fromModelObject(concept): nestedRelationshipSet = childRelationshipSet targetRole = modelRel.targetRole if self.arcrole == XbrlConst.summationItem: childPrefix = "({:0g}) ".format( modelRel.weight ) # format without .0 on integer weights elif targetRole is None or len(targetRole) == 0: targetRole = relationshipSet.linkrole childPrefix = "" else: nestedRelationshipSet = self.modelXbrl.relationshipSet( childRelationshipSet.arcrole, targetRole) childPrefix = "(via targetRole) " toConcept = modelRel.toModelObject if toConcept in visited: childPrefix += "(loop)" labelrole = modelRel.preferredLabel if not labelrole or self.labelrole == conceptNameLabelRole: labelrole = self.labelrole n += 1 # child has opposite row style of parent self.viewConcept(toConcept, modelRel, childPrefix, labelrole, childnode, n, nestedRelationshipSet, visited) visited.remove(concept) except AttributeError: return # bad object, don't try to display
def savePickle(cntlr, modelXbrl, pickleFile): if modelXbrl.fileSource.isArchive: return import io, time, pickle from arelle import Locale startedAt = time.time() fh = io.open(pickleFile, u"wb") try: pickle.dump(modelXbrl, fh) except Exception, ex: cntlr.addToLog(u"Exception " + unicode(ex)) fh.close() cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale, _(u"profiled command processing completed in %.2f secs"), time.time() - startedAt)) def savePickleMenuEntender(cntlr, menu): # Extend menu with an item for the save infoset plugin menu.add_command(label=u"Save pickled modelXbrl", underline=0, command=lambda: savePickleMenuCommand(cntlr) ) def savePickleMenuCommand(cntlr): # save Infoset menu item has been invoked from arelle.ModelDocument import Type if cntlr.modelManager is None or cntlr.modelManager.modelXbrl is None or cntlr.modelManager.modelXbrl.modelDocument.type != Type.INSTANCE: cntlr.addToLog(u"No instance loaded.") return
def genFact(dts, concept, preferredLabel, arcrole, relationshipSet, level, visited, elrInfo): try: if concept is not None: if concept.isHypercubeItem: elrInfo["inCube"] = level elrInfo["dims"] = {} elrInfo["lineItems"] =False elrInfo.pop("instant", None) elrInfo.pop("duration", None) elif concept.isDimensionItem: elrInfo["currentDim"] = concept if concept.isTypedDimension: elrInfo["dims"][concept.qname] = (concept, concept.typedDomainElement) if concept.typedDomainElement.isNumeric: elrInfo["domainIter"] = 1 elif concept.name.endswith("Member") or concept.name.endswith("_member"): # don't generate entries for default dim (Domain) (for now) dimConcept = elrInfo["currentDim"] if dimConcept.qname not in elrInfo["dims"]: elrInfo["dims"][dimConcept.qname] = (dimConcept, concept) else: if concept.name.endswith("LineItems") or concept.name.endswith("_line_items"): elrInfo["lineItems"] = True elif ((not elrInfo["inCube"] or # before any hypercube elrInfo["lineItems"]) # in Cube and within Line Items and not concept.isAbstract): # or within line items # generate a fact sampVals = sampleDataValues[elrInfo.get("domainIter",1)] # use first entry if no domain iter if concept.periodType not in elrInfo: qnameDims = {} for _dimConcept, _domConcept in elrInfo["dims"].values(): if _dimConcept.isExplicitDimension: _memVal = _domConcept.qname else: if _domConcept.type is not None and not _domConcept.isNumeric: _memEltVal = genSampleValue(sampVals, _domConcept) else: _memEltVal = str(elrInfo["domainIter"]) _memVal = XmlUtil.addChild(dts.modelDocument.xmlRootElement, _domConcept.qname, text=_memEltVal, appendChild=False) _dimObj = DimValuePrototype(dts, None, _dimConcept.qname, _memVal, "segment") qnameDims[_dimConcept.qname] = _dimObj elrInfo[concept.periodType] = dts.createContext( dts.conceptSampleScheme or "http://www.treasury.gov", "entityId", concept.periodType, sampVals["periodStart"] if concept.periodType == "duration" else None, sampVals["periodEnd"], concept.qname, qnameDims, [], []) cntx = elrInfo[concept.periodType] cntxId = cntx.id if concept.isNumeric: if concept.isMonetary: unitMeasure = qname(XbrlConst.iso4217, "USD") unitMeasure.prefix = "iso4217" # want to save with a recommended prefix decimals = 2 elif concept.isShares: unitMeasure = XbrlConst.qnXbrliShares decimals = 0 else: unitMeasure = XbrlConst.qnXbrliPure decimals = 0 # check if utr unitId is specified utrUnitId = genSampleUtrUnitId(concept) if utrUnitId is not None: _utrEntries = dts.modelManager.disclosureSystem.utrItemTypeEntries[concept.type.name] if _utrEntries: for _utrEntry in _utrEntries.values(): if _utrEntry.unitId == utrUnitId and _utrEntry.isSimple: unitMeasure = qname(_utrEntry.nsUnit, _utrEntry.unitId) break prevUnit = dts.matchUnit([unitMeasure], []) if prevUnit is not None: unitId = prevUnit.id else: newUnit = dts.createUnit([unitMeasure], []) unitId = newUnit.id value = genSampleValue(sampVals, concept) attrs = [("contextRef", cntxId)] if concept.isNumeric: attrs.append(("unitRef", unitId)) attrs.append(("decimals", decimals)) value = Locale.atof(dts.locale, str(value), str.strip) newFact = dts.createFact(concept.qname, attributes=attrs, text=value) if concept not in visited: visited.add(concept) rels = relationshipSet.fromModelObject(concept) lenRels = len(rels) iRel = 0 iFirstLineItem = None while iRel <= lenRels: if iRel == lenRels: # check if cube needs re-iterating if iFirstLineItem is None or elrInfo.get("domainIter",0) >= 2: break reIterateCube = True # cube can re-iterate else: modelRel = rels[iRel] toConcept = modelRel.toModelObject reIterateCube = (toConcept.isHypercubeItem and # finished prior line items and hitting next table iFirstLineItem is not None and elrInfo["lineItems"] and 1 <= elrInfo.get("domainIter",0) < 2) if reIterateCube: # repeat typed dim container iRel = iFirstLineItem elrInfo["domainIter"] += 1 elrInfo.pop("instant", None) # want new contexts for next iteration elrInfo.pop("duration", None) isFirstLineItem = not elrInfo["lineItems"] genFact(dts, toConcept, modelRel.preferredLabel, arcrole, relationshipSet, level+1, visited, elrInfo) if isFirstLineItem and elrInfo["lineItems"] and elrInfo.get("domainIter",0) > 0: iFirstLineItem = iRel iRel += 1 visited.remove(concept) except AttributeError as ex: # bad relationship print ("[exception] {}".format(ex)) return
def backgroundSaveInstance(self, newFilename=None): cntlr = self.modelXbrl.modelManager.cntlr if newFilename: self.modelXbrl.modelManager.showStatus( _("creating new instance {0}").format( os.path.basename(newFilename))) self.modelXbrl.modelManager.cntlr.waitForUiThreadQueue( ) # force status update self.modelXbrl.createInstance( newFilename ) # creates an instance as this modelXbrl's entrypoing instance = self.modelXbrl cntlr.showStatus( _("Saving {0}").format(instance.modelDocument.basename)) cntlr.waitForUiThreadQueue() # force status update newCntx = ModelXbrl.AUTO_LOCATE_ELEMENT newUnit = ModelXbrl.AUTO_LOCATE_ELEMENT # check user keyed changes for bodyCell in self.gridBody.winfo_children(): if isinstance(bodyCell, gridCell) and bodyCell.isChanged: value = bodyCell.value objId = bodyCell.objectId if objId: if objId[0] == "f": factPrototypeIndex = int(objId[1:]) factPrototype = self.factPrototypes[factPrototypeIndex] concept = factPrototype.concept entityIdentScheme = self.newFactItemOptions.entityIdentScheme entityIdentValue = self.newFactItemOptions.entityIdentValue periodType = factPrototype.concept.periodType periodStart = self.newFactItemOptions.startDateDate if periodType == "duration" else None periodEndInstant = self.newFactItemOptions.endDateDate qnameDims = factPrototype.context.qnameDims prevCntx = instance.matchContext( entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, qnameDims, [], []) if prevCntx is not None: cntxId = prevCntx.id else: # need new context newCntx = instance.createContext( entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, concept.qname, qnameDims, [], [], afterSibling=newCntx) cntxId = newCntx.id # new context if concept.isNumeric: if concept.isMonetary: unitMeasure = qname( XbrlConst.iso4217, self.newFactItemOptions.monetaryUnit) unitMeasure.prefix = "iso4217" # want to save with a recommended prefix decimals = self.newFactItemOptions.monetaryDecimals elif concept.isShares: unitMeasure = XbrlConst.qnXbrliShares decimals = self.newFactItemOptions.nonMonetaryDecimals else: unitMeasure = XbrlConst.qnXbrliPure decimals = self.newFactItemOptions.nonMonetaryDecimals prevUnit = instance.matchUnit([unitMeasure], []) if prevUnit is not None: unitId = prevUnit.id else: newUnit = instance.createUnit( [unitMeasure], [], afterSibling=newUnit) unitId = newUnit.id attrs = [("contextRef", cntxId)] if concept.isNumeric: attrs.append(("unitRef", unitId)) attrs.append(("decimals", decimals)) value = Locale.atof(self.modelXbrl.locale, value, str.strip) newFact = instance.createFact(concept.qname, attributes=attrs, text=value) bodyCell.objectId = newFact.objectId( ) # switch cell to now use fact ID if self.factPrototypes[factPrototypeIndex] is not None: self.factPrototypes[factPrototypeIndex].clear() self.factPrototypes[ factPrototypeIndex] = None #dereference fact prototype else: # instance fact, not prototype fact = self.modelXbrl.modelObject(objId) if fact.concept.isNumeric: value = Locale.atof(self.modelXbrl.locale, value, str.strip) if fact.value != value: fact.text = value XmlValidate.validate(instance, fact) bodyCell.isChanged = False # clear change flag instance.saveInstance() cntlr.showStatus(_("Saved {0}").format( instance.modelDocument.basename), clearAfter=3000)
def viewConcept(self, concept, modelObject, labelPrefix, preferredLabel, parentnode, n, relationshipSet, visited): if concept is None: return try: isRelation = isinstance(modelObject, ModelDtsObject.ModelRelationship) if isinstance(concept, ModelDtsObject.ModelConcept): text = labelPrefix + concept.label(preferredLabel,lang=self.lang,linkroleHint=relationshipSet.linkrole) if (self.arcrole in ("XBRL-dimensions", XbrlConst.hypercubeDimension) and concept.isTypedDimension and concept.typedDomainElement is not None): text += " (typedDomain={0})".format(concept.typedDomainElement.qname) elif isinstance(concept, ModelInstanceObject.ModelFact): if concept.concept is not None: text = labelPrefix + concept.concept.label(preferredLabel,lang=self.lang,linkroleHint=relationshipSet.linkrole) else: text = str(concept.qname) if concept.contextID: text += " [" + concept.contextID + "] = " + concept.effectiveValue elif self.arcrole == "Table-rendering": text = concept.localName elif isinstance(concept, ModelRenderingObject.ModelTable): text = (concept.genLabel(lang=self.lang, strip=True) or concept.localName) elif isinstance(concept, ModelDtsObject.ModelResource): if self.showReferences: text = (concept.viewText() or concept.localName) else: text = (Locale.rtlString(concept.textValue.strip(), lang=concept.xmlLang) or concept.localName) else: # just a resource text = concept.localName childnode = self.treeView.insert(parentnode, "end", modelObject.objectId(self.id), text=text, tags=("odd" if n & 1 else "even",)) childRelationshipSet = relationshipSet if self.arcrole == XbrlConst.parentChild: # extra columns if isRelation: preferredLabel = modelObject.preferredLabel if preferredLabel and preferredLabel.startswith("http://www.xbrl.org/2003/role/"): preferredLabel = os.path.basename(preferredLabel) self.treeView.set(childnode, "preferredLabel", preferredLabel) self.treeView.set(childnode, "type", concept.niceType) self.treeView.set(childnode, "references", viewReferences(concept)) elif self.arcrole == XbrlConst.summationItem: if isRelation: self.treeView.set(childnode, "weight", "{:+0g} ".format(modelObject.weight)) self.treeView.set(childnode, "balance", concept.balance) elif self.arcrole == "XBRL-dimensions" and isRelation: # extra columns relArcrole = modelObject.arcrole self.treeView.set(childnode, "arcrole", os.path.basename(relArcrole)) if relArcrole in (XbrlConst.all, XbrlConst.notAll): self.treeView.set(childnode, "contextElement", modelObject.contextElement) self.treeView.set(childnode, "closed", modelObject.closed) elif relArcrole in (XbrlConst.dimensionDomain, XbrlConst.domainMember): self.treeView.set(childnode, "usable", modelObject.usable) childRelationshipSet = self.modelXbrl.relationshipSet(XbrlConst.consecutiveArcrole.get(relArcrole,"XBRL-dimensions"), modelObject.consecutiveLinkrole) elif self.arcrole == "Table-rendering": # extra columns try: header = concept.header(lang=self.lang,strip=True,evaluate=False) except AttributeError: header = None # could be a filter if isRelation and header is None: header = "{0} {1}".format(os.path.basename(modelObject.arcrole), concept.xlinkLabel) self.treeView.set(childnode, "header", header) if concept.get("abstract") == "true": self.treeView.set(childnode, "abstract", '\u2713') # checkmark unicode character if concept.get("merge") == "true": self.treeView.set(childnode, "merge", '\u2713') # checkmark unicode character if isRelation: self.treeView.set(childnode, "axis", modelObject.axisDisposition) if isinstance(concept, (ModelEuAxisCoord,ModelRuleDefinitionNode)): self.treeView.set(childnode, "priItem", concept.aspectValue(None, Aspect.CONCEPT)) self.treeView.set(childnode, "dims", ' '.join(("{0},{1}".format(dim, concept.aspectValue(None, dim)) for dim in (concept.aspectValue(None, Aspect.DIMENSIONS, inherit=False) or [])))) elif self.isResourceArcrole: # resource columns if isRelation: self.treeView.set(childnode, "arcrole", os.path.basename(modelObject.arcrole)) if isinstance(concept, ModelDtsObject.ModelResource): self.treeView.set(childnode, "resource", concept.localName) self.treeView.set(childnode, "resourcerole", os.path.basename(concept.role or '')) self.treeView.set(childnode, "lang", concept.xmlLang) self.id += 1 self.tag_has[modelObject.objectId()].append(childnode) if isRelation: self.tag_has[modelObject.toModelObject.objectId()].append(childnode) if concept not in visited: visited.add(concept) for modelRel in childRelationshipSet.fromModelObject(concept): nestedRelationshipSet = childRelationshipSet targetRole = modelRel.targetRole if self.arcrole == XbrlConst.summationItem: childPrefix = "({:0g}) ".format(modelRel.weight) # format without .0 on integer weights elif targetRole is None or len(targetRole) == 0: targetRole = relationshipSet.linkrole childPrefix = "" else: nestedRelationshipSet = self.modelXbrl.relationshipSet(childRelationshipSet.arcrole, targetRole) childPrefix = "(via targetRole) " toConcept = modelRel.toModelObject if toConcept in visited: childPrefix += "(loop)" labelrole = modelRel.preferredLabel if not labelrole: labelrole = self.labelrole n += 1 # child has opposite row style of parent self.viewConcept(toConcept, modelRel, childPrefix, labelrole, childnode, n, nestedRelationshipSet, visited) visited.remove(concept) except AttributeError: return # bad object, don't try to display
def factCheck(val, fact): concept = fact.concept context = fact.context if concept is None or context is None or not val.validateLoggingSemantic: return # not checkable try: if fact.isNumeric: # 2.3.3 additional unit tests beyond UTR spec unit = fact.unit if unit is not None and concept.type is not None and val.validateUTR: typeName = concept.type.name if typeName == "perUnitItemType" and any(m.namespaceURI == XbrlConst.iso4217 or m in (XbrlConst.qnXbrliPure, XbrlConst.qnXbrliShares) for m in unit.measures[1]): val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.3.3.perUnitItemType", _("PureItemType fact %(fact)s in context %(contextID)s unit %(unitID)s value %(value)s has disallowed unit denominator %(denominator)s"), modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID, value=fact.effectiveValue, denominator=", ".join((str(m) for m in unit.measures[1]))) if not fact.isNil and getattr(fact, "xValue", None) is not None: # 2.4.1 decimal disagreement if fact.decimals and fact.decimals != "INF": vf = float(fact.value) if _ISFINITE(vf): dec = _INT(fact.decimals) vround = round(vf, dec) if vf != vround: val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.4.1", _("Decimal disagreement %(fact)s in context %(contextID)s unit %(unitID)s value %(value)s has insignificant value %(insignificantValue)s"), modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID, value=fact.effectiveValue, insignificantValue=Locale.format(val.modelXbrl.locale, "%.*f", (dec + 2 if dec > 0 else 0, vf - vround), True)) # 2.5.1 fractions disallowed on a disclosure if fact.isFraction: if any(val.linroleDefinitionIsDisclosure.match(roleType.definition) for rel in val.modelXbrl.relationshipSet(XbrlConst.parentChild).toModelObject(concept) for roleType in val.modelXbrl.roleTypes.get(rel.linkrole,())): val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.2.5.1", _("Disclosure %(fact)s in context %(contextID)s value %(value)s is a fraction"), modelObject=fact, fact=fact.qname, contextID=fact.contextID, value=fact.value) # deprecated concept if concept.qname.namespaceURI == val.ugtNamespace: if concept.name in val.usgaapDeprecations: val.deprecatedFactConcepts[concept].append(fact) elif concept.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"): val.deprecatedFactConcepts[concept].append(fact) if fact.isItem and fact.context is not None: for dimConcept, modelDim in fact.context.segDimValues.items(): if dimConcept.qname.namespaceURI == val.ugtNamespace: if dimConcept.name in val.usgaapDeprecations: val.deprecatedDimensions[dimConcept].append(fact) elif dimConcept.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"): val.deprecatedDimensions[dimConcept].append(fact) if modelDim.isExplicit: member = modelDim.member if member is not None: if member.qname.namespaceURI == val.ugtNamespace: if member.name in val.usgaapDeprecations: val.deprecatedMembers[member].append(fact) elif member.get("{http://fasb.org/us-gaap/attributes}deprecatedDate"): val.deprecatedMembers[member].append(fact) except Exception as err: val.modelXbrl.log('WARNING-SEMANTIC', "US-BPG.testingException", _("%(fact)s in context %(contextID)s unit %(unitID)s value %(value)s cannot be tested due to: %(err)s"), modelObject=fact, fact=fact.qname, contextID=fact.contextID, unitID=fact.unitID, value=fact.effectiveValue, err=err)
def loadXbrlFromDB(self, loadDBsaveToFile): # load from database modelXbrl = self.modelXbrl # find instance in DB instanceURI = os.path.basename(loadDBsaveToFile) results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant" " FROM dInstance WHERE FileName = '{}'" .format(instanceURI)) instanceId = moduleId = None for instanceId, moduleId, entScheme, entId, datePerEnd in results: break # find module in DB results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId)) xbrlSchemaRef = None for result in results: xbrlSchemaRef = result[0] break if not instanceId or not xbrlSchemaRef: raise XPDBException("sqlDB:MissingDTS", _("The instance and module were not found for %(instanceURI)"), instanceURI = instanceURI) if modelXbrl.skipDTS: # find prefixes and namespaces in DB results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]") dpmPrefixedNamespaces = dict((prefix, namespace) for owner, prefix, namespace in results) # create the instance document and resulting filing modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument(modelXbrl, Type.INSTANCE, loadDBsaveToFile, schemaRefs=[xbrlSchemaRef], isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement, 'xbrl-streamable-instance', 'version="1.0" contextBuffer="1"') # add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue) # filing indicator code IDs # get filing indicators results = self.execute("SELECT mToT.TemplateOrTableCode " " FROM dFilingIndicator dFI, mTemplateOrTable mToT " " WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID" .format(instanceId)) filingIndicatorCodes = [code[0] for code in results] if filingIndicatorCodes: modelXbrl.createContext(entScheme, entId, 'instant', None, datePerEnd, None, # no dimensional validity checking (like formula does) {}, [], [], id='c') filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators) for filingIndicatorCode in filingIndicatorCodes: modelXbrl.createFact(qnFindFilingIndicator, parent=filingIndicatorsTuple, attributes={"contextRef": "c"}, text=filingIndicatorCode) # facts in this instance factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards," " Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue " "FROM dFact WHERE InstanceID = {} " "ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL " " THEN DataPointSignature" " ELSE DataPointSignatureWithValuesForWildcards" " END, instr(DataPointSignature,'|') + 1)" .format(instanceId)) # results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal # get typed dimension values prefixedNamespaces = modelXbrl.prefixedNamespaces prefixedNamespaces["iso4217"] = XbrlConst.iso4217 if modelXbrl.skipDTS: prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed cntxTbl = {} # index by d unitTbl = {} def typedDimElt(s): # add xmlns into s for known qnames tag, angleBrkt, rest = s[1:].partition('>') text, angleBrkt, rest = rest.partition("<") qn = qname(tag, prefixedNamespaces) # a modelObject xml element is needed for all of the instance functions to manage the typed dim return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False) # contexts and facts for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl: metric, sep, dims = (dpSigTypedDims or dpSig).partition('|') conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces) concept = modelXbrl.qnameConcepts.get(conceptQn) isNumeric = isBool = isDateTime = isQName = isText = False if concept is not None: if concept.isNumeric: isNumeric = True else: baseXbrliType = concept.baseXbrliType if baseXbrliType == "booleanItemType": isBool = True elif baseXbrliType == "dateTimeItemType": # also is dateItemType? isDateTime = True elif baseXbrliType == "QNameItemType": isQName = True else: c = conceptQn.localName[0] if c == 'm': isNumeric = True elif c == 'd': isDateTime = True elif c == 'b': isBool = True elif c == 'e': isQName = True isText = not (isNumeric or isBool or isDateTime or isQName) if isinstance(datePerEnd, _STR_BASE): datePerEnd = datetimeValue(datePerEnd, addOneDay=True) cntxKey = (dims, entId, datePerEnd) if cntxKey in cntxTbl: cntxId = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) cntxTbl[cntxKey] = cntxId qnameDims = {} if dims: for dim in dims.split('|'): dQn, sep, dVal = dim[:-1].partition('(') dimQname = qname(dQn, prefixedNamespaces) if dVal.startswith('<'): mem = typedDimElt(dVal) # typed dim else: mem = qname(dVal, prefixedNamespaces) # explicit dim (even if treat-as-typed) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario") modelXbrl.createContext(entScheme, entId, 'instant', None, datePerEnd, None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) if unit: if unit in unitTbl: unitId = unitTbl[unit] else: unitQn = qname(unit, prefixedNamespaces) unitId = 'u{}'.format(unitQn.localName) unitTbl[unit] = unitId modelXbrl.createUnit([unitQn], [], id=unitId) else: unitId = None attrs = {"contextRef": cntxId} if unitId: attrs["unitRef"] = unitId if dec is not None: if isinstance(dec, float): # must be an integer dec = int(dec) elif isinstance(dec, _STR_BASE) and '.' in dec: dec = dec.partition('.')[0] # drop .0 from any SQLite string attrs["decimals"] = str(dec) # somehow it is float from the database if False: # fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None elif numVal is not None: num = roundValue(numVal, None, dec) # round using reported decimals if dec is None or dec == "INF": # show using decimals or reported format dec = len(numVal.partition(".")[2]) else: # max decimals at 28 dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num)) elif dateVal is not None: text = dateVal elif boolVal is not None: text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false' else: if isQName: # declare namespace addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces)) text = textVal modelXbrl.createFact(conceptQn, attributes=attrs, text=text) # add footnotes if any # save to file modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000) return modelXbrl.modelDocument
def parsePackage(cntlr, filesource, metadataFile, fileBase): global ArchiveFileIOError if ArchiveFileIOError is None: from arelle.FileSource import ArchiveFileIOError unNamedCounter = 1 txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1", "http://xbrl.org/PWD/2014-01-15/taxonomy-package", "http://xbrl.org/PWD/2015-01-14/taxonomy-package") catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog",) pkg = {} currentLang = Locale.getLanguageCode() _file = filesource.file(metadataFile)[0] # URL in zip, plain file in file system or web tree = etree.parse(_file) root = tree.getroot() ns = root.tag.partition("}")[0][1:] nsPrefix = "{{{}}}".format(ns) if ns in txmyPkgNSes: # package file for eltName in ("name", "description", "version"): pkg[eltName] = '' for m in root.iterchildren(tag=nsPrefix + eltName): pkg[eltName] = m.text.strip() break # take first entry if several else: # oasis catalog, use dirname as the package name # metadataFile may be a File object (with name) or string filename fileName = getattr(metadataFile, 'fileName', # for FileSource named objects getattr(metadataFile, 'name', # for io.file named objects metadataFile)) # for string pkg["name"] = os.path.basename(os.path.dirname(fileName)) pkg["description"] = "oasis catalog" pkg["version"] = "(none)" remappings = {} rewriteTree = tree catalogFile = metadataFile if ns in ("http://xbrl.org/PWD/2015-01-14/taxonomy-package",): catalogFile = metadataFile.replace('taxonomyPackage.xml','catalog.xml') try: rewriteTree = etree.parse(filesource.file(catalogFile)[0]) except ArchiveFileIOError: pass for tag, prefixAttr, replaceAttr in ( (nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem", "systemIdStartString", "rewritePrefix"), ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI", "uriStartString", "rewritePrefix")): # oasis catalog for m in rewriteTree.iter(tag=tag): prefixValue = m.get(prefixAttr) replaceValue = m.get(replaceAttr) if prefixValue and replaceValue is not None: if prefixValue not in remappings: base = baseForElement(m) if base: replaceValue = os.path.join(base, replaceValue) if replaceValue: # neither None nor '' if not isAbsolute(replaceValue): if not os.path.isabs(replaceValue): replaceValue = fileBase + replaceValue replaceValue = replaceValue.replace("/", os.sep) _normedValue = os.path.normpath(replaceValue) if replaceValue.endswith(os.sep) and not _normedValue.endswith(os.sep): _normedValue += os.sep remappings[prefixValue] = _normedValue else: cntlr.addToLog(_("Package catalog duplicate rewrite start string %(rewriteStartString)s"), messageArgs={"rewriteStartString": prefixValue}, messageCode="arelle.catalogDuplicateRewrite", file=os.path.basename(catalogFile), level=logging.ERROR) pkg["remappings"] = remappings nameToUrls = {} pkg["nameToUrls"] = nameToUrls for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"): name = None # find closest match name node given xml:lang match to current language or no xml:lang for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"): xmlLang = nameNode.get('{http://www.w3.org/XML/1998/namespace}lang') if name is None or not xmlLang or currentLang == xmlLang: name = nameNode.text if currentLang == xmlLang: # most prefer one with the current locale's language break if not name: name = _("<unnamed {0}>").format(unNamedCounter) unNamedCounter += 1 epDocCount = 0 for epDoc in entryPointSpec.iterchildren(nsPrefix + "entryPointDocument"): epUrl = epDoc.get('href') base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base') # cope with xml:base if base: resolvedUrl = urljoin(base, epUrl) else: resolvedUrl = epUrl if epDocCount: cntlr.addToLog(_("Skipping multiple-document entry point (not supported) %(href)s"), messageArgs={"href": epUrl}, messageCode="arelle.packageMultipleDocumentEntryPoints", file=os.path.basename(metadataFile), level=logging.WARNING) continue epDocCount += 1 #perform prefix remappings remappedUrl = resolvedUrl longestPrefix = 0 for mapFrom, mapTo in remappings.items(): if remappedUrl.startswith(mapFrom): prefixLength = len(mapFrom) if prefixLength > longestPrefix: _remappedUrl = remappedUrl[prefixLength:] if not (_remappedUrl[0] in (os.sep, '/') or mapTo[-1] in (os.sep, '/')): _remappedUrl = mapTo + os.sep + _remappedUrl else: _remappedUrl = mapTo + _remappedUrl longestPrefix = prefixLength if longestPrefix: remappedUrl = _remappedUrl.replace(os.sep, "/") # always used as FileSource select nameToUrls[name] = (remappedUrl, resolvedUrl) return pkg
def parsePackage(mainWin, metadataFile): unNamedCounter = 1 txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1", "http://xbrl.org/PWD/2014-01-15/taxonomy-package") catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog",) pkg = {} currentLang = Locale.getLanguageCode() tree = etree.parse(metadataFile) root = tree.getroot() ns = root.tag.partition("}")[0][1:] nsPrefix = "{{{}}}".format(ns) if ns in txmyPkgNSes: # package file for eltName in ("name", "description", "version"): pkg[eltName] = '' for m in root.iterchildren(tag=nsPrefix + eltName): pkg[eltName] = m.text.strip() break # take first entry if several else: # oasis catalog, use dirname as the package name # metadataFile may be a File object (with name) or string filename fileName = getattr(metadataFile, 'fileName', # for FileSource named objects getattr(metadataFile, 'name', # for io.file named objects metadataFile)) # for string pkg["name"] = os.path.basename(os.path.dirname(fileName)) pkg["description"] = "oasis catalog" pkg["version"] = "(none)" remappings = {} for tag, prefixAttr, replaceAttr in ( (nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package (nsPrefix + "rewriteSystem", "systemIdStartString", "rewritePrefix")): # oasis catalog for m in tree.iter(tag=tag): prefixValue = m.get(prefixAttr) replaceValue = m.get(replaceAttr) if prefixValue and replaceValue is not None: remappings[prefixValue] = replaceValue pkg["remappings"] = remappings nameToUrls = {} pkg["nameToUrls"] = nameToUrls for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"): name = None # find closest match name node given xml:lang match to current language or no xml:lang for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"): xmlLang = nameNode.get('{http://www.w3.org/XML/1998/namespace}lang') if name is None or not xmlLang or currentLang == xmlLang: name = nameNode.text if currentLang == xmlLang: # most prefer one with the current locale's language break if not name: name = _("<unnamed {0}>").format(unNamedCounter) unNamedCounter += 1 epDocCount = 0 for epDoc in entryPointSpec.iterchildren(nsPrefix + "entryPointDocument"): if epDocCount: mainWin.addToLog(_("WARNING: skipping multiple-document entry point (not supported)")) continue epDocCount += 1 epUrl = epDoc.get('href') base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base') # cope with xml:base if base: resolvedUrl = urljoin(base, epUrl) else: resolvedUrl = epUrl #perform prefix remappings remappedUrl = resolvedUrl for prefix, replace in remappings.items(): remappedUrl = remappedUrl.replace(prefix, replace, 1) nameToUrls[name] = (remappedUrl, resolvedUrl) return pkg
def loadXbrlFromDB(self, loadDBsaveToFile): # load from database modelXbrl = self.modelXbrl # find instance in DB instanceURI = os.path.basename(loadDBsaveToFile) results = self.execute("SELECT InstanceID, ModuleID, EntityScheme, EntityIdentifier, PeriodEndDateOrInstant" " FROM dInstance WHERE FileName = '{}'" .format(instanceURI)) instanceId = moduleId = None for instanceId, moduleId, entScheme, entId, datePerEnd in results: break # find module in DB results = self.execute("SELECT XbrlSchemaRef FROM mModule WHERE ModuleID = {}".format(moduleId)) xbrlSchemaRef = None for result in results: xbrlSchemaRef = result[0] break if not instanceId or not xbrlSchemaRef: raise XPDBException("sqlDB:MissingDTS", _("The instance and module were not found for %(instanceURI)"), instanceURI = instanceURI) if modelXbrl.skipDTS: # find prefixes and namespaces in DB results = self.execute("SELECT * FROM [vwGetNamespacesPrefixes]") dpmPrefixedNamespaces = dict((prefix, namespace) for owner, prefix, namespace in results) # create the instance document and resulting filing modelXbrl.blockDpmDBrecursion = True modelXbrl.modelDocument = createModelDocument(modelXbrl, Type.INSTANCE, loadDBsaveToFile, schemaRefs=[xbrlSchemaRef], isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults addProcessingInstruction(modelXbrl.modelDocument.xmlRootElement, 'xbrl-streamable-instance', 'version="1.0" contextBuffer="1"') # add roleRef and arcroleRef (e.g. for footnotes, if any, see inlineXbrlDocue) # filing indicator code IDs # get filing indicators results = self.execute("SELECT mToT.TemplateOrTableCode " " FROM dFilingIndicator dFI, mTemplateOrTable mToT " " WHERE dFI.InstanceID = {} AND mTot.TemplateOrTableID = dFI.BusinessTemplateID" .format(instanceId)) filingIndicatorCodes = [code[0] for code in results] if filingIndicatorCodes: modelXbrl.createContext(entScheme, entId, 'instant', None, datePerEnd, None, # no dimensional validity checking (like formula does) {}, [], [], id='c') filingIndicatorsTuple = modelXbrl.createFact(qnFindFilingIndicators) for filingIndicatorCode in filingIndicatorCodes: modelXbrl.createFact(qnFindFilingIndicator, parent=filingIndicatorsTuple, attributes={"contextRef": "c"}, text=filingIndicatorCode) # facts in this instance factsTbl = self.execute("SELECT DataPointSignature, DataPointSignatureWithValuesForWildcards," " Unit, Decimals, NumericValue, DateTimeValue, BooleanValue, TextValue " "FROM dFact WHERE InstanceID = {} " "ORDER BY substr(CASE WHEN DataPointSignatureWithValuesForWildcards IS NULL " " THEN DataPointSignature" " ELSE DataPointSignatureWithValuesForWildcards" " END, instr(DataPointSignature,'|') + 1)" .format(instanceId)) # results tuple: factId, dec, varId, dpKey, entId, datePerEnd, unit, numVal, dateVal, boolVal, textVal # get typed dimension values prefixedNamespaces = modelXbrl.prefixedNamespaces prefixedNamespaces["iso4217"] = XbrlConst.iso4217 if modelXbrl.skipDTS: prefixedNamespaces.update(dpmPrefixedNamespaces) # for skipDTS this is always needed cntxTbl = {} # index by d unitTbl = {} def typedDimElt(s): # add xmlns into s for known qnames tag, angleBrkt, rest = s[1:].partition('>') text, angleBrkt, rest = rest.partition("<") qn = qname(tag, prefixedNamespaces) # a modelObject xml element is needed for all of the instance functions to manage the typed dim return addChild(modelXbrl.modelDocument, qn, text=text, appendChild=False) # contexts and facts for dpSig, dpSigTypedDims, unit, dec, numVal, dateVal, boolVal, textVal in factsTbl: metric, sep, dims = (dpSigTypedDims or dpSig).partition('|') conceptQn = qname(metric.partition('(')[2][:-1], prefixedNamespaces) concept = modelXbrl.qnameConcepts.get(conceptQn) isNumeric = isBool = isDateTime = isQName = isText = False if concept is not None: if concept.isNumeric: isNumeric = True else: baseXbrliType = concept.baseXbrliType if baseXbrliType == "booleanItemType": isBool = True elif baseXbrliType == "dateTimeItemType": # also is dateItemType? isDateTime = True elif baseXbrliType == "QNameItemType": isQName = True else: c = conceptQn.localName[0] if c == 'm': isNumeric = True elif c == 'd': isDateTime = True elif c == 'b': isBool = True elif c == 'e': isQName = True isText = not (isNumeric or isBool or isDateTime or isQName) if isinstance(datePerEnd, _STR_BASE): datePerEnd = datetimeValue(datePerEnd, addOneDay=True) cntxKey = (dims, entId, datePerEnd) if cntxKey in cntxTbl: cntxId = cntxTbl[cntxKey] else: cntxId = 'c-{:02}'.format(len(cntxTbl) + 1) cntxTbl[cntxKey] = cntxId qnameDims = {} if dims: for dim in dims.split('|'): dQn, sep, dVal = dim[:-1].partition('(') dimQname = qname(dQn, prefixedNamespaces) if dVal.startswith('<'): # typed dim mem = typedDimElt(dVal) else: mem = qname(dVal, prefixedNamespaces) qnameDims[dimQname] = DimValuePrototype(modelXbrl, None, dimQname, mem, "scenario") modelXbrl.createContext(entScheme, entId, 'instant', None, datePerEnd, None, # no dimensional validity checking (like formula does) qnameDims, [], [], id=cntxId) if unit: if unit in unitTbl: unitId = unitTbl[unit] else: unitQn = qname(unit, prefixedNamespaces) unitId = 'u{}'.format(unitQn.localName) unitTbl[unit] = unitId modelXbrl.createUnit([unitQn], [], id=unitId) else: unitId = None attrs = {"contextRef": cntxId} if unitId: attrs["unitRef"] = unitId if dec is not None: if isinstance(dec, float): # must be an integer dec = int(dec) elif isinstance(dec, _STR_BASE) and '.' in dec: dec = dec.partition('.')[0] # drop .0 from any SQLite string attrs["decimals"] = str(dec) # somehow it is float from the database if False: # fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None elif numVal is not None: num = roundValue(numVal, None, dec) # round using reported decimals if dec is None or dec == "INF": # show using decimals or reported format dec = len(numVal.partition(".")[2]) else: # max decimals at 28 dec = max( min(int(float(dec)), 28), -28) # 2.7 wants short int, 3.2 takes regular int, don't use _INT here text = Locale.format(self.modelXbrl.locale, "%.*f", (dec, num)) elif dateVal is not None: text = dateVal elif boolVal is not None: text = 'true' if boolVal.lower() in ('t', 'true', '1') else 'false' else: if isQName: # declare namespace addQnameValue(modelXbrl.modelDocument, qname(textVal, prefixedNamespaces)) text = textVal modelXbrl.createFact(conceptQn, attributes=attrs, text=text) # add footnotes if any # save to file modelXbrl.saveInstance(overrideFilepath=loadDBsaveToFile) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000) return modelXbrl.modelDocument
def parsePackage(cntlr, filesource, metadataFile, fileBase): global ArchiveFileIOError if ArchiveFileIOError is None: from arelle.FileSource import ArchiveFileIOError unNamedCounter = 1 txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1", "http://xbrl.org/PWD/2014-01-15/taxonomy-package", "http://xbrl.org/PWD/2015-01-14/taxonomy-package") catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog", ) pkg = {} currentLang = Locale.getLanguageCode() _file = filesource.file(metadataFile)[ 0] # URL in zip, plain file in file system or web tree = etree.parse(_file) root = tree.getroot() ns = root.tag.partition("}")[0][1:] nsPrefix = "{{{}}}".format(ns) if ns in txmyPkgNSes: # package file for eltName in ("name", "description", "version"): pkg[eltName] = '' for m in root.iterchildren(tag=nsPrefix + eltName): pkg[eltName] = m.text.strip() break # take first entry if several else: # oasis catalog, use dirname as the package name # metadataFile may be a File object (with name) or string filename fileName = getattr( metadataFile, 'fileName', # for FileSource named objects getattr( metadataFile, 'name', # for io.file named objects metadataFile)) # for string pkg["name"] = os.path.basename(os.path.dirname(fileName)) pkg["description"] = "oasis catalog" pkg["version"] = "(none)" remappings = {} rewriteTree = tree catalogFile = metadataFile if ns in ("http://xbrl.org/PWD/2015-01-14/taxonomy-package", ): catalogFile = metadataFile.replace('taxonomyPackage.xml', 'catalog.xml') try: rewriteTree = etree.parse(filesource.file(catalogFile)[0]) except ArchiveFileIOError: pass for tag, prefixAttr, replaceAttr in ( (nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem", "systemIdStartString", "rewritePrefix"), ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI", "uriStartString", "rewritePrefix")): # oasis catalog for m in rewriteTree.iter(tag=tag): prefixValue = m.get(prefixAttr) replaceValue = m.get(replaceAttr) if prefixValue and replaceValue is not None: if prefixValue not in remappings: base = baseForElement(m) if base: replaceValue = os.path.join(base, replaceValue) if replaceValue: # neither None nor '' if not isAbsolute(replaceValue): if not os.path.isabs(replaceValue): replaceValue = fileBase + replaceValue replaceValue = replaceValue.replace("/", os.sep) _normedValue = os.path.normpath(replaceValue) if replaceValue.endswith( os.sep) and not _normedValue.endswith(os.sep): _normedValue += os.sep remappings[prefixValue] = _normedValue else: cntlr.addToLog(_( "Package catalog duplicate rewrite start string %(rewriteStartString)s" ), messageArgs={ "rewriteStartString": prefixValue }, messageCode="arelle.catalogDuplicateRewrite", file=os.path.basename(catalogFile), level=logging.ERROR) pkg["remappings"] = remappings nameToUrls = {} pkg["nameToUrls"] = nameToUrls for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"): name = None # find closest match name node given xml:lang match to current language or no xml:lang for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"): xmlLang = nameNode.get( '{http://www.w3.org/XML/1998/namespace}lang') if name is None or not xmlLang or currentLang == xmlLang: name = nameNode.text if currentLang == xmlLang: # most prefer one with the current locale's language break if not name: name = _("<unnamed {0}>").format(unNamedCounter) unNamedCounter += 1 epDocCount = 0 for epDoc in entryPointSpec.iterchildren(nsPrefix + "entryPointDocument"): epUrl = epDoc.get('href') base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base' ) # cope with xml:base if base: resolvedUrl = urljoin(base, epUrl) else: resolvedUrl = epUrl if epDocCount: cntlr.addToLog( _("Skipping multiple-document entry point (not supported) %(href)s" ), messageArgs={"href": epUrl}, messageCode="arelle.packageMultipleDocumentEntryPoints", file=os.path.basename(metadataFile), level=logging.WARNING) continue epDocCount += 1 #perform prefix remappings remappedUrl = resolvedUrl longestPrefix = 0 for mapFrom, mapTo in remappings.items(): if remappedUrl.startswith(mapFrom): prefixLength = len(mapFrom) if prefixLength > longestPrefix: _remappedUrl = remappedUrl[prefixLength:] if not (_remappedUrl[0] in (os.sep, '/') or mapTo[-1] in (os.sep, '/')): _remappedUrl = mapTo + os.sep + _remappedUrl else: _remappedUrl = mapTo + _remappedUrl longestPrefix = prefixLength if longestPrefix: remappedUrl = _remappedUrl.replace( os.sep, "/") # always used as FileSource select nameToUrls[name] = (remappedUrl, resolvedUrl) return pkg
def validate(self): if not self.modelXbrl.contexts and not self.modelXbrl.facts: return # skip if no contexts or facts if not self.inferDecimals: # infering precision is now contrary to XBRL REC section 5.2.5.2 self.modelXbrl.info("xbrl.5.2.5.2:inferringPrecision","Validating calculations inferring precision.") # identify equal contexts self.modelXbrl.profileActivity() uniqueContextHashes = {} for context in self.modelXbrl.contexts.values(): h = context.contextDimAwareHash if h in uniqueContextHashes: if context.isEqualTo(uniqueContextHashes[h]): self.mapContext[context] = uniqueContextHashes[h] else: uniqueContextHashes[h] = context del uniqueContextHashes self.modelXbrl.profileActivity("... identify equal contexts", minTimeToShow=1.0) # identify equal contexts uniqueUnitHashes = {} for unit in self.modelXbrl.units.values(): h = unit.hash if h in uniqueUnitHashes: if unit.isEqualTo(uniqueUnitHashes[h]): self.mapUnit[unit] = uniqueUnitHashes[h] else: uniqueUnitHashes[h] = unit self.modelXbrl.profileActivity("... identify equal units", minTimeToShow=1.0) # identify concepts participating in essence-alias relationships # identify calcluation & essence-alias base sets (by key) for baseSetKey in self.modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if ELR and linkqname and arcqname: if arcrole in (XbrlConst.essenceAlias, XbrlConst.requiresElement): conceptsSet = {XbrlConst.essenceAlias:self.conceptsInEssencesAlias, XbrlConst.requiresElement:self.conceptsInRequiresElement}[arcrole] for modelRel in self.modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname).modelRelationships: for concept in (modelRel.fromModelObject, modelRel.toModelObject): if concept is not None and concept.qname is not None: conceptsSet.add(concept) self.modelXbrl.profileActivity("... identify requires-element and esseance-aliased concepts", minTimeToShow=1.0) self.bindFacts(self.modelXbrl.facts,[self.modelXbrl.modelDocument.xmlRootElement]) self.modelXbrl.profileActivity("... bind facts", minTimeToShow=1.0) # identify calcluation & essence-alias base sets (by key) for baseSetKey in self.modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if ELR and linkqname and arcqname: if arcrole in (XbrlConst.summationItem, XbrlConst.essenceAlias, XbrlConst.requiresElement): relsSet = self.modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname) if arcrole == XbrlConst.summationItem: fromRelationships = relsSet.fromModelObjects() for sumConcept, modelRels in fromRelationships.items(): sumBindingKeys = self.sumConceptBindKeys[sumConcept] dupBindingKeys = set() boundSumKeys = set() # determine boundSums for modelRel in modelRels: itemConcept = modelRel.toModelObject if itemConcept is not None and itemConcept.qname is not None: itemBindingKeys = self.itemConceptBindKeys[itemConcept] boundSumKeys |= sumBindingKeys & itemBindingKeys # add up rounded items boundSums = defaultdict(decimal.Decimal) # sum of facts meeting factKey boundSummationItems = defaultdict(list) # corresponding fact refs for messages for modelRel in modelRels: weight = modelRel.weightDecimal itemConcept = modelRel.toModelObject if itemConcept is not None: for itemBindKey in boundSumKeys: ancestor, contextHash, unit = itemBindKey factKey = (itemConcept, ancestor, contextHash, unit) if factKey in self.itemFacts: for fact in self.itemFacts[factKey]: if fact in self.duplicatedFacts: dupBindingKeys.add(itemBindKey) else: roundedValue = roundFact(fact, self.inferDecimals) boundSums[itemBindKey] += roundedValue * weight boundSummationItems[itemBindKey].append(wrappedFactWithWeight(fact,weight,roundedValue)) for sumBindKey in boundSumKeys: ancestor, contextHash, unit = sumBindKey factKey = (sumConcept, ancestor, contextHash, unit) if factKey in self.sumFacts: sumFacts = self.sumFacts[factKey] for fact in sumFacts: if fact in self.duplicatedFacts: dupBindingKeys.add(sumBindKey) elif sumBindKey not in dupBindingKeys: roundedSum = roundFact(fact, self.inferDecimals) roundedItemsSum = roundFact(fact, self.inferDecimals, vDecimal=boundSums[sumBindKey]) if roundedItemsSum != roundFact(fact, self.inferDecimals): d = inferredDecimals(fact) if isnan(d) or isinf(d): d = 4 _boundSummationItems = boundSummationItems[sumBindKey] unreportedContribingItemQnames = [] # list the missing/unreported contributors in relationship order for modelRel in modelRels: itemConcept = modelRel.toModelObject if (itemConcept is not None and (itemConcept, ancestor, contextHash, unit) not in self.itemFacts): unreportedContribingItemQnames.append(str(itemConcept.qname)) self.modelXbrl.log('INCONSISTENCY', "xbrl.5.2.5.2:calcInconsistency", _("Calculation inconsistent from %(concept)s in link role %(linkrole)s reported sum %(reportedSum)s computed sum %(computedSum)s context %(contextID)s unit %(unitID)s unreportedContributingItems %(unreportedContributors)s"), modelObject=wrappedSummationAndItems(fact, roundedSum, _boundSummationItems), concept=sumConcept.qname, linkrole=ELR, linkroleDefinition=self.modelXbrl.roleTypeDefinition(ELR), reportedSum=Locale.format_decimal(self.modelXbrl.locale, roundedSum, 1, max(d,0)), computedSum=Locale.format_decimal(self.modelXbrl.locale, roundedItemsSum, 1, max(d,0)), contextID=fact.context.id, unitID=fact.unit.id, unreportedContributors=", ".join(unreportedContribingItemQnames) or "none") del unreportedContribingItemQnames[:] boundSummationItems.clear() # dereference facts in list elif arcrole == XbrlConst.essenceAlias: for modelRel in relsSet.modelRelationships: essenceConcept = modelRel.fromModelObject aliasConcept = modelRel.toModelObject essenceBindingKeys = self.esAlConceptBindKeys[essenceConcept] aliasBindingKeys = self.esAlConceptBindKeys[aliasConcept] for esAlBindKey in essenceBindingKeys & aliasBindingKeys: ancestor, contextHash = esAlBindKey essenceFactsKey = (essenceConcept, ancestor, contextHash) aliasFactsKey = (aliasConcept, ancestor, contextHash) if essenceFactsKey in self.esAlFacts and aliasFactsKey in self.esAlFacts: for eF in self.esAlFacts[essenceFactsKey]: for aF in self.esAlFacts[aliasFactsKey]: essenceUnit = self.mapUnit.get(eF.unit,eF.unit) aliasUnit = self.mapUnit.get(aF.unit,aF.unit) if essenceUnit != aliasUnit: self.modelXbrl.log('INCONSISTENCY', "xbrl.5.2.6.2.2:essenceAliasUnitsInconsistency", _("Essence-Alias inconsistent units from %(essenceConcept)s to %(aliasConcept)s in link role %(linkrole)s context %(contextID)s"), modelObject=(modelRel, eF, aF), essenceConcept=essenceConcept.qname, aliasConcept=aliasConcept.qname, linkrole=ELR, linkroleDefinition=self.modelXbrl.roleTypeDefinition(ELR), contextID=eF.context.id) if not XbrlUtil.vEqual(eF, aF): self.modelXbrl.log('INCONSISTENCY', "xbrl.5.2.6.2.2:essenceAliasUnitsInconsistency", _("Essence-Alias inconsistent value from %(essenceConcept)s to %(aliasConcept)s in link role %(linkrole)s context %(contextID)s"), modelObject=(modelRel, eF, aF), essenceConcept=essenceConcept.qname, aliasConcept=aliasConcept.qname, linkrole=ELR, linkroleDefinition=self.modelXbrl.roleTypeDefinition(ELR), contextID=eF.context.id) elif arcrole == XbrlConst.requiresElement: for modelRel in relsSet.modelRelationships: sourceConcept = modelRel.fromModelObject requiredConcept = modelRel.toModelObject if sourceConcept in self.requiresElementFacts and \ not requiredConcept in self.requiresElementFacts: self.modelXbrl.log('INCONSISTENCY', "xbrl.5.2.6.2.4:requiresElementInconsistency", _("Requires-Element %(requiringConcept)s missing required fact for %(requiredConcept)s in link role %(linkrole)s"), modelObject=sourceConcept, requiringConcept=sourceConcept.qname, requiredConcept=requiredConcept.qname, linkrole=ELR, linkroleDefinition=self.modelXbrl.roleTypeDefinition(ELR)) self.modelXbrl.profileActivity("... find inconsistencies", minTimeToShow=1.0) self.modelXbrl.profileActivity() # reset
def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]): global ArchiveFileIOError if ArchiveFileIOError is None: from arelle.FileSource import ArchiveFileIOError unNamedCounter = 1 txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1", "http://xbrl.org/PWD/2014-01-15/taxonomy-package", "http://xbrl.org/PWD/2015-01-14/taxonomy-package", "http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package") catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog",) pkg = {} currentLang = Locale.getLanguageCode() _file = filesource.file(metadataFile)[0] # URL in zip, plain file in file system or web try: tree = etree.parse(_file) except etree.XMLSyntaxError as err: cntlr.addToLog(_("Package catalog syntax error %(error)s"), messageArgs={"error": str(err)}, messageCode="tpe:invalidMetaDataFile", file=os.path.basename(metadataFile), level=logging.ERROR) errors.append("tpe:invalidMetaDataFile") raise # reraise error root = tree.getroot() ns = root.tag.partition("}")[0][1:] nsPrefix = "{{{}}}".format(ns) if ns in txmyPkgNSes: # package file for eltName in ("identifier", "version", "license", "publisher", "publisherURL", "publisherCountry", "publicationDate"): pkg[eltName] = '' for m in root.iterchildren(tag=nsPrefix + eltName): if eltName == "license": pkg[eltName] = m.get("name") else: pkg[eltName] = (m.text or "").strip() break # take first entry if several for eltName in ("name", "description"): closest = '' closestLen = 0 for m in root.iterchildren(tag=nsPrefix + eltName): s = (m.text or "").strip() l = langCloseness(xmlLang(m), currentLang) if l > closestLen: closestLen = l closest = s pkg[eltName] = closest for eltName in ("supersededTaxonomyPackages", "versioningReports"): pkg[eltName] = [] for m in root.iterchildren(tag=nsPrefix + "supersededTaxonomyPackages"): pkg['supersededTaxonomyPackages'] = [ r.text.strip() for r in m.iterchildren(tag=nsPrefix + "taxonomyPackageRef")] for m in root.iterchildren(tag=nsPrefix + "versioningReports"): pkg['versioningReports'] = [ r.get("href") for r in m.iterchildren(tag=nsPrefix + "versioningReport")] # check for duplicate multi-lingual elements (among children of nodes) langElts = defaultdict(list) for n in root.iter(tag=nsPrefix + "*"): for eltName in ("name", "description"): langElts.clear() for m in n.iterchildren(tag=nsPrefix + eltName): langElts[xmlLang(m)].append(m) for lang, elts in langElts.items(): if not lang: cntlr.addToLog(_("Multi-lingual element %(element)s has no in-scope xml:lang attribute"), messageArgs={"element": eltName}, messageCode="tpe:missingLanguageAttribute", refs=[{"href":os.path.basename(metadataFile), "sourceLine":m.sourceline} for m in elts], level=logging.ERROR) errors.append("tpe:missingLanguageAttribute") elif len(elts) > 1: cntlr.addToLog(_("Multi-lingual element %(element)s has multiple (%(count)s) in-scope xml:lang %(lang)s elements"), messageArgs={"element": eltName, "lang": lang, "count": len(elts)}, messageCode="tpe:duplicateLanguagesForElement", refs=[{"href":os.path.basename(metadataFile), "sourceLine":m.sourceline} for m in elts], level=logging.ERROR) errors.append("tpe:duplicateLanguagesForElement") del langElts # dereference else: # oasis catalog, use dirname as the package name # metadataFile may be a File object (with name) or string filename fileName = getattr(metadataFile, 'fileName', # for FileSource named objects getattr(metadataFile, 'name', # for io.file named objects metadataFile)) # for string pkg["name"] = os.path.basename(os.path.dirname(fileName)) pkg["description"] = "oasis catalog" pkg["version"] = "(none)" remappings = {} rewriteTree = tree catalogFile = metadataFile if ns in ("http://xbrl.org/PWD/2015-01-14/taxonomy-package", "http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package"): catalogFile = metadataFile.replace('taxonomyPackage.xml','catalog.xml') try: rewriteTree = etree.parse(filesource.file(catalogFile)[0]) except ArchiveFileIOError: pass for tag, prefixAttr, replaceAttr in ( (nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem", "systemIdStartString", "rewritePrefix"), ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI", "uriStartString", "rewritePrefix")): # oasis catalog for m in rewriteTree.iter(tag=tag): prefixValue = m.get(prefixAttr) replaceValue = m.get(replaceAttr) if prefixValue and replaceValue is not None: if prefixValue not in remappings: base = baseForElement(m) if base: replaceValue = os.path.join(base, replaceValue) if replaceValue: # neither None nor '' if not isAbsolute(replaceValue): if not os.path.isabs(replaceValue): replaceValue = fileBase + replaceValue replaceValue = replaceValue.replace("/", os.sep) _normedValue = os.path.normpath(replaceValue) if replaceValue.endswith(os.sep) and not _normedValue.endswith(os.sep): _normedValue += os.sep remappings[prefixValue] = _normedValue else: cntlr.addToLog(_("Package catalog duplicate rewrite start string %(rewriteStartString)s"), messageArgs={"rewriteStartString": prefixValue}, messageCode="tpe:multipleRewriteURIsForStartString", file=os.path.basename(catalogFile), level=logging.ERROR) errors.append("tpe:multipleRewriteURIsForStartString") pkg["remappings"] = remappings nameToUrls = {} pkg["nameToUrls"] = nameToUrls for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"): name = None closestLen = 0 # find closest match name node given xml:lang match to current language or no xml:lang for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"): s = (nameNode.text or "").strip() l = langCloseness(xmlLang(nameNode), currentLang) if l > closestLen: closestLen = l name = s if not name: name = _("<unnamed {0}>").format(unNamedCounter) unNamedCounter += 1 epDocCount = 0 for epDoc in entryPointSpec.iterchildren(nsPrefix + "entryPointDocument"): epUrl = epDoc.get('href') base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base') # cope with xml:base if base: resolvedUrl = urljoin(base, epUrl) else: resolvedUrl = epUrl if epDocCount: cntlr.addToLog(_("Skipping multiple-document entry point (not supported) %(href)s"), messageArgs={"href": epUrl}, messageCode="arelle.packageMultipleDocumentEntryPoints", file=os.path.basename(metadataFile), level=logging.WARNING) errors.append("arelle.packageMultipleDocumentEntryPoints") continue epDocCount += 1 #perform prefix remappings remappedUrl = resolvedUrl longestPrefix = 0 for mapFrom, mapTo in remappings.items(): if remappedUrl.startswith(mapFrom): prefixLength = len(mapFrom) if prefixLength > longestPrefix: _remappedUrl = remappedUrl[prefixLength:] if not (_remappedUrl[0] in (os.sep, '/') or mapTo[-1] in (os.sep, '/')): _remappedUrl = mapTo + os.sep + _remappedUrl else: _remappedUrl = mapTo + _remappedUrl longestPrefix = prefixLength if longestPrefix: remappedUrl = _remappedUrl.replace(os.sep, "/") # always used as FileSource select nameToUrls[name] = (remappedUrl, resolvedUrl) return pkg
def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]): global ArchiveFileIOError if ArchiveFileIOError is None: from arelle.FileSource import ArchiveFileIOError unNamedCounter = 1 txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1", "http://xbrl.org/PWD/2014-01-15/taxonomy-package", "http://xbrl.org/PWD/2015-01-14/taxonomy-package", "http://xbrl.org/PR/2015-12-09/taxonomy-package", "http://xbrl.org/2016/taxonomy-package", "http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package") catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog", ) pkg = {} currentLang = Locale.getLanguageCode() _file = filesource.file(metadataFile)[ 0] # URL in zip, plain file in file system or web parser = lxmlResolvingParser(cntlr) try: tree = etree.parse(_file, parser=parser) # schema validate tp xml xsdTree = etree.parse(TP_XSD, parser=parser) etree.XMLSchema(xsdTree).assertValid(tree) except (etree.XMLSyntaxError, etree.DocumentInvalid) as err: cntlr.addToLog(_("Taxonomy package file syntax error %(error)s"), messageArgs={"error": str(err)}, messageCode="tpe:invalidMetaDataFile", file=os.path.basename(metadataFile), level=logging.ERROR) errors.append("tpe:invalidMetaDataFile") return pkg root = tree.getroot() ns = root.tag.partition("}")[0][1:] nsPrefix = "{{{}}}".format(ns) if ns in txmyPkgNSes: # package file for eltName in ("identifier", "version", "license", "publisher", "publisherURL", "publisherCountry", "publicationDate"): pkg[eltName] = '' for m in root.iterchildren(tag=nsPrefix + eltName): if eltName == "license": pkg[eltName] = m.get("name") else: pkg[eltName] = (m.text or "").strip() break # take first entry if several for eltName in ("name", "description"): closest = '' closestLen = 0 for m in root.iterchildren(tag=nsPrefix + eltName): s = (m.text or "").strip() eltLang = xmlLang(m) l = langCloseness(eltLang, currentLang) if l > closestLen: closestLen = l closest = s elif closestLen == 0 and eltLang.startswith("en"): closest = s # pick english if nothing better if not closest and eltName == "name": # assign default name when none in taxonomy package closest = os.path.splitext(os.path.basename( filesource.baseurl))[0] pkg[eltName] = closest for eltName in ("supersededTaxonomyPackages", "versioningReports"): pkg[eltName] = [] for m in root.iterchildren(tag=nsPrefix + "supersededTaxonomyPackages"): pkg['supersededTaxonomyPackages'] = [ r.text.strip() for r in m.iterchildren(tag=nsPrefix + "taxonomyPackageRef") ] for m in root.iterchildren(tag=nsPrefix + "versioningReports"): pkg['versioningReports'] = [ r.get("href") for r in m.iterchildren(tag=nsPrefix + "versioningReport") ] # check for duplicate multi-lingual elements (among children of nodes) langElts = defaultdict(list) for n in root.iter(tag=nsPrefix + "*"): for eltName in ("name", "description", "publisher"): langElts.clear() for m in n.iterchildren(tag=nsPrefix + eltName): langElts[xmlLang(m)].append(m) for lang, elts in langElts.items(): if not lang: cntlr.addToLog( _("Multi-lingual element %(element)s has no in-scope xml:lang attribute" ), messageArgs={"element": eltName}, messageCode="tpe:missingLanguageAttribute", refs=[{ "href": os.path.basename(metadataFile), "sourceLine": m.sourceline } for m in elts], level=logging.ERROR) errors.append("tpe:missingLanguageAttribute") elif len(elts) > 1: cntlr.addToLog( _("Multi-lingual element %(element)s has multiple (%(count)s) in-scope xml:lang %(lang)s elements" ), messageArgs={ "element": eltName, "lang": lang, "count": len(elts) }, messageCode="tpe:duplicateLanguagesForElement", refs=[{ "href": os.path.basename(metadataFile), "sourceLine": m.sourceline } for m in elts], level=logging.ERROR) errors.append("tpe:duplicateLanguagesForElement") del langElts # dereference else: # oasis catalog, use dirname as the package name # metadataFile may be a File object (with name) or string filename fileName = getattr( metadataFile, 'fileName', # for FileSource named objects getattr( metadataFile, 'name', # for io.file named objects metadataFile)) # for string pkg["name"] = os.path.basename(os.path.dirname(fileName)) pkg["description"] = "oasis catalog" pkg["version"] = "(none)" remappings = {} rewriteTree = tree catalogFile = metadataFile if ns in ("http://xbrl.org/PWD/2015-01-14/taxonomy-package", "http://xbrl.org/PR/2015-12-09/taxonomy-package", "http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package", "http://xbrl.org/2016/taxonomy-package", "http://xbrl.org/REC/2016-04-19/taxonomy-package"): catalogFile = metadataFile.replace('taxonomyPackage.xml', 'catalog.xml') try: rewriteTree = etree.parse(filesource.file(catalogFile)[0], parser=parser) # schema validate tp xml xsdTree = etree.parse(CAT_XSD, parser=parser) etree.XMLSchema(xsdTree).assertValid(rewriteTree) except (etree.XMLSyntaxError, etree.DocumentInvalid) as err: cntlr.addToLog(_("Catalog file syntax error %(error)s"), messageArgs={"error": str(err)}, messageCode="tpe:invalidCatalogFile", file=os.path.basename(metadataFile), level=logging.ERROR) errors.append("tpe:invalidCatalogFile") except ArchiveFileIOError: pass for tag, prefixAttr, replaceAttr in ( (nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem", "systemIdStartString", "rewritePrefix"), ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI", "uriStartString", "rewritePrefix")): # oasis catalog for m in rewriteTree.iter(tag=tag): prefixValue = m.get(prefixAttr) replaceValue = m.get(replaceAttr) if prefixValue and replaceValue is not None: if prefixValue not in remappings: base = baseForElement(m) if base: replaceValue = os.path.join(base, replaceValue) if replaceValue: # neither None nor '' if not isAbsolute(replaceValue): if not os.path.isabs(replaceValue): replaceValue = fileBase + replaceValue replaceValue = replaceValue.replace("/", os.sep) _normedValue = cntlr.webCache.normalizeUrl(replaceValue) if replaceValue.endswith( os.sep) and not _normedValue.endswith(os.sep): _normedValue += os.sep remappings[prefixValue] = _normedValue else: cntlr.addToLog( _("Package catalog duplicate rewrite start string %(rewriteStartString)s" ), messageArgs={"rewriteStartString": prefixValue}, messageCode="tpe:multipleRewriteURIsForStartString", file=os.path.basename(catalogFile), level=logging.ERROR) errors.append("tpe:multipleRewriteURIsForStartString") pkg["remappings"] = remappings entryPoints = defaultdict(list) pkg["entryPoints"] = entryPoints for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"): name = None closestLen = 0 # find closest match name node given xml:lang match to current language or no xml:lang for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"): s = (nameNode.text or "").strip() nameLang = xmlLang(nameNode) l = langCloseness(nameLang, currentLang) if l > closestLen: closestLen = l name = s elif closestLen == 0 and nameLang.startswith("en"): name = s # pick english if nothing better if not name: name = _("<unnamed {0}>").format(unNamedCounter) unNamedCounter += 1 epDocCount = 0 for epDoc in entryPointSpec.iterchildren(nsPrefix + "entryPointDocument"): epUrl = epDoc.get('href') base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base' ) # cope with xml:base if base: resolvedUrl = urljoin(base, epUrl) else: resolvedUrl = epUrl epDocCount += 1 #perform prefix remappings remappedUrl = resolvedUrl longestPrefix = 0 for mapFrom, mapTo in remappings.items(): if remappedUrl.startswith(mapFrom): prefixLength = len(mapFrom) if prefixLength > longestPrefix: _remappedUrl = remappedUrl[prefixLength:] if not (_remappedUrl[0] in (os.sep, '/') or mapTo[-1] in (os.sep, '/')): _remappedUrl = mapTo + os.sep + _remappedUrl else: _remappedUrl = mapTo + _remappedUrl longestPrefix = prefixLength if longestPrefix: remappedUrl = _remappedUrl.replace( os.sep, "/") # always used as FileSource select # find closest language description closest = '' closestLen = 0 for m in entryPointSpec.iterchildren(tag=nsPrefix + "description"): s = (m.text or "").strip() eltLang = xmlLang(m) l = langCloseness(eltLang, currentLang) if l > closestLen: closestLen = l closest = s elif closestLen == 0 and eltLang.startswith("en"): closest = s # pick english if nothing better if not closest and name: # assign default name when none in taxonomy package closest = name entryPoints[name].append((remappedUrl, resolvedUrl, closest)) return pkg
def parsePackage(mainWin, metadataFile): unNamedCounter = 1 txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1", "http://xbrl.org/PWD/2014-01-15/taxonomy-package") catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog", ) pkg = {} currentLang = Locale.getLanguageCode() tree = etree.parse(metadataFile) root = tree.getroot() ns = root.tag.partition("}")[0][1:] nsPrefix = "{{{}}}".format(ns) if ns in txmyPkgNSes: # package file for eltName in ("name", "description", "version"): pkg[eltName] = '' for m in root.iterchildren(tag=nsPrefix + eltName): pkg[eltName] = m.text.strip() break # take first entry if several else: # oasis catalog, use dirname as the package name # metadataFile may be a File object (with name) or string filename fileName = getattr( metadataFile, 'fileName', # for FileSource named objects getattr( metadataFile, 'name', # for io.file named objects metadataFile)) # for string pkg["name"] = os.path.basename(os.path.dirname(fileName)) pkg["description"] = "oasis catalog" pkg["version"] = "(none)" remappings = {} for tag, prefixAttr, replaceAttr in ( (nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package (nsPrefix + "rewriteSystem", "systemIdStartString", "rewritePrefix")): # oasis catalog for m in tree.iter(tag=tag): prefixValue = m.get(prefixAttr) replaceValue = m.get(replaceAttr) if prefixValue and replaceValue is not None: remappings[prefixValue] = replaceValue pkg["remappings"] = remappings nameToUrls = {} pkg["nameToUrls"] = nameToUrls for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"): name = None # find closest match name node given xml:lang match to current language or no xml:lang for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"): xmlLang = nameNode.get( '{http://www.w3.org/XML/1998/namespace}lang') if name is None or not xmlLang or currentLang == xmlLang: name = nameNode.text if currentLang == xmlLang: # most prefer one with the current locale's language break if not name: name = _("<unnamed {0}>").format(unNamedCounter) unNamedCounter += 1 epDocCount = 0 for epDoc in entryPointSpec.iterchildren(nsPrefix + "entryPointDocument"): if epDocCount: mainWin.addToLog( _("WARNING: skipping multiple-document entry point (not supported)" )) continue epDocCount += 1 epUrl = epDoc.get('href') base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base' ) # cope with xml:base if base: resolvedUrl = urljoin(base, epUrl) else: resolvedUrl = epUrl #perform prefix remappings remappedUrl = resolvedUrl for prefix, replace in remappings.items(): remappedUrl = remappedUrl.replace(prefix, replace, 1) nameToUrls[name] = (remappedUrl, resolvedUrl) return pkg
def backgroundSaveInstance(self, newFilename=None): cntlr = self.modelXbrl.modelManager.cntlr if newFilename and self.modelXbrl.modelDocument.type != ModelDocument.Type.INSTANCE: self.modelXbrl.modelManager.showStatus(_("creating new instance {0}").format(os.path.basename(newFilename))) self.modelXbrl.modelManager.cntlr.waitForUiThreadQueue() # force status update self.modelXbrl.createInstance(newFilename) # creates an instance as this modelXbrl's entrypoing instance = self.modelXbrl cntlr.showStatus(_("Saving {0}").format(instance.modelDocument.basename)) cntlr.waitForUiThreadQueue() # force status update newCntx = ModelXbrl.AUTO_LOCATE_ELEMENT newUnit = ModelXbrl.AUTO_LOCATE_ELEMENT # check user keyed changes for bodyCell in self.gridBody.winfo_children(): if isinstance(bodyCell, gridCell) and bodyCell.isChanged: value = bodyCell.value objId = bodyCell.objectId if objId: if objId[0] == "f": factPrototypeIndex = int(objId[1:]) factPrototype = self.factPrototypes[factPrototypeIndex] concept = factPrototype.concept entityIdentScheme = self.newFactItemOptions.entityIdentScheme entityIdentValue = self.newFactItemOptions.entityIdentValue periodType = factPrototype.concept.periodType periodStart = self.newFactItemOptions.startDateDate if periodType == "duration" else None periodEndInstant = self.newFactItemOptions.endDateDate qnameDims = factPrototype.context.qnameDims prevCntx = instance.matchContext( entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, qnameDims, [], []) if prevCntx is not None: cntxId = prevCntx.id else: # need new context newCntx = instance.createContext(entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, concept.qname, qnameDims, [], [], afterSibling=newCntx) cntxId = newCntx.id # new context if concept.isNumeric: if concept.isMonetary: unitMeasure = qname(XbrlConst.iso4217, self.newFactItemOptions.monetaryUnit) unitMeasure.prefix = "iso4217" # want to save with a recommended prefix decimals = self.newFactItemOptions.monetaryDecimals elif concept.isShares: unitMeasure = XbrlConst.qnXbrliShares decimals = self.newFactItemOptions.nonMonetaryDecimals else: unitMeasure = XbrlConst.qnXbrliPure decimals = self.newFactItemOptions.nonMonetaryDecimals prevUnit = instance.matchUnit([unitMeasure],[]) if prevUnit is not None: unitId = prevUnit.id else: newUnit = instance.createUnit([unitMeasure],[], afterSibling=newUnit) unitId = newUnit.id attrs = [("contextRef", cntxId)] if concept.isNumeric: attrs.append(("unitRef", unitId)) attrs.append(("decimals", decimals)) value = Locale.atof(self.modelXbrl.locale, value, str.strip) newFact = instance.createFact(concept.qname, attributes=attrs, text=value) bodyCell.objectId = newFact.objectId() # switch cell to now use fact ID if self.factPrototypes[factPrototypeIndex] is not None: self.factPrototypes[factPrototypeIndex].clear() self.factPrototypes[factPrototypeIndex] = None #dereference fact prototype else: # instance fact, not prototype fact = self.modelXbrl.modelObject(objId) if fact.concept.isNumeric: value = Locale.atof(self.modelXbrl.locale, value, str.strip) if fact.value != value: if fact.concept.isNumeric and fact.isNil != (not value): fact.isNil = not value if value: # had been nil, now it needs decimals fact.decimals = (self.newFactItemOptions.monetaryDecimals if fact.concept.isMonetary else self.newFactItemOptions.nonMonetaryDecimals) fact.text = value XmlValidate.validate(instance, fact) bodyCell.isChanged = False # clear change flag instance.saveInstance(newFilename) # may override prior filename for instance from main menu cntlr.showStatus(_("Saved {0}").format(instance.modelDocument.basename), clearAfter=3000)