def __init__(self, modelManager):
self.modelManager = modelManager
self.urls = [os.path.join(modelManager.cntlr.configDir, "disclosuresystems.xml")]
# get custom config xml file url
for pluginXbrlMethod in pluginClassMethods("DisclosureSystem.ConfigURL"):
self.urls.append(pluginXbrlMethod(self))
# get custom disclosure system (type name, test variable)
self.pluginDisclosureTypes = {} # dict of type name, test variable name
for pluginXbrlMethod in pluginClassMethods("DisclosureSystem.Types"):
for typeName, typeTestVariable in pluginXbrlMethod(self):
self.pluginDisclosureTypes[typeName] = typeTestVariable
self.clear()
def saveTargetDocument(filing, modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs,
outputZip=None, filingFiles=None,
suffix=DEFAULT_DISTINGUISHING_SUFFIX, iext=DEFAULT_INSTANCE_EXT):
sourceDir = os.path.dirname(modelXbrl.modelDocument.filepath)
def addLocallyReferencedFile(elt,filingFiles):
if elt.tag in ("a", "img"):
for attrTag, attrValue in elt.items():
if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrValue):
attrValue = attrValue.partition('#')[0] # remove anchor
if attrValue: # ignore anchor references to base document
attrValue = os.path.normpath(attrValue) # change url path separators to host separators
file = os.path.join(sourceDir,attrValue)
if modelXbrl.fileSource.isInArchive(file, checkExistence=True) or modelXbrl.fileSource.exists(file):
filingFiles.add(file)
targetUrlParts = targetDocumentFilename.rpartition(".")
targetUrl = targetUrlParts[0] + suffix + targetUrlParts[2]
modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl))
for pluginXbrlMethod in pluginClassMethods("InlineDocumentSet.CreateTargetInstance"):
targetInstance = pluginXbrlMethod(modelXbrl, targetUrl, targetDocumentSchemaRefs, filingFiles,
# no lang on xbrl:xbrl, specific xml:lang on elements which aren't en-US
baseXmlLang=None, defaultXmlLang="en-US")
if outputZip:
targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip, updateFileHistory=False, xmlcharrefreplace=True)
else:
fh = io.StringIO();
targetInstance.saveInstance(overrideFilepath=targetUrl, outputFile=fh, updateFileHistory=False, xmlcharrefreplace=True)
fh.seek(0)
filing.writeFile(targetUrl, fh.read())
fh.close()
if getattr(modelXbrl, "isTestcaseVariation", False):
modelXbrl.extractedInlineInstance = True # for validation comparison
modelXbrl.modelManager.showStatus(_("Saved extracted instance"), clearAfter=5000)
return # there can only be one "InlineDocumentSet.CreateTargetInstance" but just to be sure
cntlr.logTrace(_("Unable to save extracted document, missing plugin class \"InlineDocumentSet.CreateTargetInstance\"."))
def load(self, filesource, nextaction=None):
"""Load an entry point modelDocument object(s), which in turn load documents they discover
(for the case of instance, taxonomies, and versioning reports), but defer loading instances
for test case and RSS feeds.
The modelXbrl that is loaded is 'stacked', by this class, so that any modelXbrl operations such as validate,
and close, operate on the most recently loaded modelXbrl, and compareDTSes operates on the two
most recently loaded modelXbrl's.
:param filesource: may be a FileSource object, with the entry point selected, or string file name (or web URL).
:type filesource: FileSource or str
:param nextAction: status line text string, if any, to show upon completion
:type nextAction: str
"""
try:
if filesource.url.startswith("urn:uuid:"): # request for an open modelXbrl
for modelXbrl in self.loadedModelXbrls:
if not modelXbrl.isClosed and modelXbrl.uuid == filesource.url:
return modelXbrl
raise IOError(_("Open file handle is not open: {0}").format(filesource.url))
except AttributeError:
pass # filesource may be a string, which has no url attribute
self.filesource = filesource
modelXbrl = None # loaded modelXbrl
for customLoader in pluginClassMethods("ModelManager.Load"):
modelXbrl = customLoader(self, filesource)
if modelXbrl is not None:
break # custom loader did the loading
if modelXbrl is None: # use default xbrl loader
modelXbrl = ModelXbrl.load(self, filesource, nextaction)
self.modelXbrl = modelXbrl
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def initUI(testObject):
testObject.saveReferences = False # <- set this to create new references
testObject.referencesDir = None
testObject.cntlrWinMain = None
testObject.process = psutil.Process(os.getpid())
testObject.entryPointInfos = None
testObject.maxSize = 0
testObject.prevMemConsumed = 0
testObject.testContext = None
testObject.viewHelper = None
testObject.application = Tk()
testObject.cntlrWinMain = CntlrWinMain(testObject.application)
testObject.application.protocol("WM_DELETE_WINDOW", testObject.cntlrWinMain.quit)
testObject.testDir = getTestDir()
testObject.referencesDir = testObject.testDir + "/references/"
testObject.cntlrWinMain.setTestMode(True)
# make sure we use a plugin loaded by the plugin manager!!
for pluginMethod in pluginClassMethods("DevTesting.GetTestContext"):
testObject.testContext = pluginMethod()
break
testObject.testContext.recordTableLayout = True
testObject.testContext.saveFilePath = testObject.testDir + "/tmp/a1.xbrl"
def resetProxies(self, httpProxyTuple):
# for ntlm user and password are required
self.hasNTLM = False
if isinstance(httpProxyTuple,(tuple,list)) and len(httpProxyTuple) == 5:
useOsProxy, _urlAddr, _urlPort, user, password = httpProxyTuple
_proxyDirFmt = proxyDirFmt(httpProxyTuple)
# only try ntlm if user and password are provided because passman is needed
if user and not useOsProxy:
for pluginXbrlMethod in pluginClassMethods("Proxy.HTTPNtlmAuthHandler"):
HTTPNtlmAuthHandler = pluginXbrlMethod()
if HTTPNtlmAuthHandler is not None:
self.hasNTLM = True
if not self.hasNTLM: # try for python site-packages ntlm
try:
from ntlm import HTTPNtlmAuthHandler
self.hasNTLM = True
except ImportError:
pass
if self.hasNTLM:
pwrdmgr = proxyhandlers.HTTPPasswordMgrWithDefaultRealm()
pwrdmgr.add_password(None, _proxyDirFmt["http"], user, password)
self.proxy_handler = proxyhandlers.ProxyHandler({})
self.proxy_auth_handler = proxyhandlers.ProxyBasicAuthHandler(pwrdmgr)
self.http_auth_handler = proxyhandlers.HTTPBasicAuthHandler(pwrdmgr)
self.ntlm_auth_handler = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(pwrdmgr)
self.opener = proxyhandlers.build_opener(self.proxy_handler, self.ntlm_auth_handler, self.proxy_auth_handler, self.http_auth_handler)
if not self.hasNTLM:
self.proxy_handler = proxyhandlers.ProxyHandler(proxyDirFmt(httpProxyTuple))
self.proxy_auth_handler = proxyhandlers.ProxyBasicAuthHandler()
self.http_auth_handler = proxyhandlers.HTTPBasicAuthHandler()
self.opener = proxyhandlers.build_opener(self.proxy_handler, self.proxy_auth_handler, self.http_auth_handler)
def testcaseVariationXbrlLoaded(testcaseModelXbrl, instanceModelXbrl, modelTestcaseVariation, *args, **kwargs):
# Validate of RSS feed item or testcase variation (simulates filing & cmd line load events
modelManager = instanceModelXbrl.modelManager
if (hasattr(testcaseModelXbrl, "efmOptions") and
modelManager.validateDisclosureSystem and getattr(modelManager.disclosureSystem, "EFMplugin", False) and
(instanceModelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE or
instanceModelXbrl.modelDocument.type == ModelDocument.Type.INLINEXBRL)):
cntlr = modelManager.cntlr
options = testcaseModelXbrl.efmOptions
entrypointFiles = [{"file":instanceModelXbrl.modelDocument.uri}]
if not hasattr(modelManager, "efmFiling"): # first instance of filing
modelManager.efmFiling = Filing(cntlr, options, instanceModelXbrl.fileSource, entrypointFiles, None, None, instanceModelXbrl.errorCaptureLevel)
# this event is called for filings (of instances) as well as test cases, for test case it just keeps options accessible
for pluginXbrlMethod in pluginClassMethods("EdgarRenderer.Filing.Start"):
pluginXbrlMethod(cntlr, options, entrypointFiles, modelManager.efmFiling)
modelManager.efmFiling.addReport(instanceModelXbrl)
_report = modelManager.efmFiling.reports[-1]
_report.entryPoint = entrypointFiles[0]
modelManager.efmFiling.arelleUnitTests = instanceModelXbrl.arelleUnitTests.copy() # allow unit tests to be used after instance processing finished
# check for parameters on instance
for _instanceElt in XmlUtil.descendants(modelTestcaseVariation, "*", "instance", "readMeFirst", "true", False):
if instanceModelXbrl.modelDocument.uri.endswith(_instanceElt.text):
if _instanceElt.get("exhibitType"):
_report.entryPoint["exhibitType"] = _report.exhibitType = _instanceElt.get("exhibitType")
break
def test(self):
testDir = os.path.dirname(os.path.abspath(sys.modules[__name__].__file__))
testFileSmall = testDir + "/solvency/2.0/random/spv_20_instance.xbrl"
application = Tk()
cntlrWinMain = CntlrWinMain(application)
application.protocol("WM_DELETE_WINDOW", cntlrWinMain.quit)
cntlrWinMain.setTestMode(True)
for pluginMethod in pluginClassMethods("DevTesting.GetTestContext"):
testContext = pluginMethod()
break
testContext.checkMemoryOnClose = True
testContext.dumpFilePrefix = testDir + "/tmp/dump_"
tr = tracker.SummaryTracker()
for idx in range(4):
print("\nIteration " + str(idx))
cntlrWinMain.fileOpenFile(testFileSmall)
cntlrWinMain.logClear()
cntlrWinMain.fileClose()
tr.print_diff()
if idx > 1:
assert testContext.diffNumObjects < 8000, "Check for new objects leak"
def validateRssFeed(self):
self.modelXbrl.info("info", "RSS Feed", modelDocument=self.modelXbrl)
from arelle.FileSource import openFileSource
for rssItem in self.modelXbrl.modelDocument.rssItems:
self.modelXbrl.info("info", _("RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"),
modelObject=rssItem, accessionNumber=rssItem.accessionNumber, formType=rssItem.formType, companyName=rssItem.companyName, period=rssItem.period)
modelXbrl = None
try:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
openFileSource(rssItem.zippedUrl, self.modelXbrl.modelManager.cntlr),
_("validating"))
self.instValidator.validate(modelXbrl, self.modelXbrl.modelManager.formulaOptions.typedParameters())
self.instValidator.close()
rssItem.setResults(modelXbrl)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, rssItem.objectId())
for pluginXbrlMethod in pluginClassMethods("Validate.RssItem"):
pluginXbrlMethod(self, modelXbrl, rssItem)
modelXbrl.close()
except Exception as err:
self.modelXbrl.error("exception",
_("RSS item validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=(self.modelXbrl, modelXbrl),
instance=rssItem.zippedUrl, error=err,
exc_info=True)
try:
self.instValidator.close()
if modelXbrl is not None:
modelXbrl.close()
except Exception as err:
pass
del modelXbrl # completely dereference
def clear(self):
self.selection = None
self.standardTaxonomiesDict = {}
self.familyHrefs = {}
self.standardLocalHrefs = set()
self.standardAuthorities = set()
self.baseTaxonomyNamespaces = set()
self.standardPrefixes = {}
self.names = []
self.name = None
self.validationType = None
self.exclusiveTypesPattern = None # regex of type matches exclusive with validationType
# previoulsy built-in types (intent to replace with plugin defined types)
self.EFM = False
self.GFM = False
self.EFMorGFM = False
self.HMRC = False
self.SBRNL = False
self.pluginTypes = set()
for pluginXbrlMethod in pluginClassMethods("DisclosureSystem.Types"):
for typeName, typeTestVariable in pluginXbrlMethod(self):
setattr(self, typeTestVariable, False)
self.pluginTypes.add(typeName)
self.validateFileText = False
self.allowedExternalHrefPattern = None
self.schemaValidateSchema = None
self.blockDisallowedReferences = False
self.maxSubmissionSubdirectoryEntryNesting = 0
self.defaultXmlLang = None
self.defaultXmlEncoding = "utf-8"
self.xmlLangPattern = None
self.defaultLanguage = None
self.language = None
self.standardTaxonomiesUrl = None
self.mappingsUrl = os.path.join(self.modelManager.cntlr.configDir, "mappings.xml")
self.mappedFiles = {}
self.mappedPaths = []
self.utrUrl = "http://www.xbrl.org/utr/utr.xml"
self.utrTypeEntries = None
self.identifierSchemePattern = None
self.identifierValuePattern = None
self.identifierValueName = None
self.contextElement = None
self.roleDefinitionPattern = None
self.labelCheckPattern = None
self.labelTrimPattern = None
self.deiNamespacePattern = None
self.deiAmendmentFlagElement = None
self.deiCurrentFiscalYearEndDateElement = None
self.deiDocumentFiscalYearFocusElement = None
self.deiDocumentPeriodEndDateElement = None
self.deiFilerIdentifierElement = None
self.deiFilerNameElement = None
self.logLevelFilter = None
self.logCodeFilter = None
self.standardTaxonomyDatabase = None
self.standardTaxonomyUrlPattern = None
self.version = (0,0,0)
def testcaseVariationXbrlValidated(testcaseModelXbrl, instanceModelXbrl, *args, **kwargs):
modelManager = instanceModelXbrl.modelManager
if (hasattr(modelManager, "efmFiling") and
(instanceModelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE or
instanceModelXbrl.modelDocument.type == ModelDocument.Type.INLINEXBRL)):
efmFiling = modelManager.efmFiling
_report = modelManager.efmFiling.reports[-1]
for pluginXbrlMethod in pluginClassMethods("EdgarRenderer.Xbrl.Run"):
pluginXbrlMethod(modelManager.cntlr, efmFiling.options, instanceModelXbrl, efmFiling, _report)
def readMeFirstUris(self):
try:
return self._readMeFirstUris
except AttributeError:
self._readMeFirstUris = []
# first look if any plugin method to get readme first URIs
if not any(
pluginXbrlMethod(self)
for pluginXbrlMethod in pluginClassMethods(
"ModelTestcaseVariation.ReadMeFirstUris")):
if self.localName == "testGroup": #w3c testcase
instanceTestElement = XmlUtil.descendant(
self, None, "instanceTest")
if instanceTestElement is not None: # take instance first
self._readMeFirstUris.append(
XmlUtil.descendantAttr(
instanceTestElement, None, "instanceDocument",
"{http://www.w3.org/1999/xlink}href"))
else:
schemaTestElement = XmlUtil.descendant(
self, None, "schemaTest")
if schemaTestElement is not None:
self._readMeFirstUris.append(
XmlUtil.descendantAttr(
schemaTestElement, None, "schemaDocument",
"{http://www.w3.org/1999/xlink}href"))
elif self.localName == "test-case": #xpath testcase
inputFileElement = XmlUtil.descendant(
self, None, "input-file")
if inputFileElement is not None: # take instance first
self._readMeFirstUris.append(
"TestSources/" + inputFileElement.text + ".xml")
else:
# default built-in method for readme first uris
for anElement in self.iterdescendants():
if isinstance(anElement,
ModelObject) and anElement.get(
"readMeFirst") == "true":
if anElement.get(
"{http://www.w3.org/1999/xlink}href"):
uri = anElement.get(
"{http://www.w3.org/1999/xlink}href")
else:
uri = XmlUtil.innerText(anElement)
if anElement.get("name"):
self._readMeFirstUris.append((ModelValue.qname(
anElement, anElement.get("name")), uri))
elif anElement.get("dts"):
self._readMeFirstUris.append(
(anElement.get("dts"), uri))
else:
self._readMeFirstUris.append(uri)
if not self._readMeFirstUris: # provide a dummy empty instance document
self._readMeFirstUris.append(
os.path.join(self.modelXbrl.modelManager.cntlr.configDir,
"empty-instance.xml"))
return self._readMeFirstUris
def filingStart(cntlr, options, filesource, entrypointFiles, sourceZipStream=None, responseZipStream=None, *args, **kwargs):
modelManager = cntlr.modelManager
# cntlr.addToLog("TRACE EFM filing start val={} plugin={}".format(modelManager.validateDisclosureSystem, getattr(modelManager.disclosureSystem, "EFMplugin", False)))
if modelManager.validateDisclosureSystem and getattr(modelManager.disclosureSystem, "EFMplugin", False):
# cntlr.addToLog("TRACE EFM filing start 2 classes={} moduleInfos={}".format(pluginMethodsForClasses, modulePluginInfos))
modelManager.efmFiling = Filing(cntlr, options, filesource, entrypointFiles, sourceZipStream, responseZipStream)
# this event is called for filings (of instances) as well as test cases, for test case it just keeps options accessible
for pluginXbrlMethod in pluginClassMethods("EdgarRenderer.Filing.Start"):
pluginXbrlMethod(cntlr, options, entrypointFiles, modelManager.efmFiling)
def resultXbrlInstanceUri(self):
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ResultXbrlInstanceUri"):
resultInstanceUri = pluginXbrlMethod(self)
if resultInstanceUri is not None:
return resultInstanceUri or None # (empty string returns None)
resultInstance = XmlUtil.descendant(XmlUtil.descendant(self, None, "result"), None, "instance")
if resultInstance is not None:
return XmlUtil.text(resultInstance)
return None
def severityLevel(self):
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ExpectedSeverity"):
severityLevelName = pluginXbrlMethod(self)
if severityLevelName: # ignore plug in if not a plug-in-recognized test case
return logging._checkLevel(severityLevelName)
# default behavior without plugins
# SEC error cases have <assert severity={err|wrn}>...
if XmlUtil.descendant(self, None, "assert", attrName="severity", attrValue="wrn") is not None:
return logging._checkLevel("WARNING")
return logging._checkLevel("INCONSISTENCY")
def xbrlRun(cntlr, options, modelXbrl, *args, **kwargs):
# cntlr.addToLog("TRACE EFM xbrl run")
modelManager = cntlr.modelManager
if (hasattr(modelManager, "efmFiling") and
(modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE or
modelXbrl.modelDocument.type == ModelDocument.Type.INLINEXBRL)):
efmFiling = modelManager.efmFiling
_report = efmFiling.reports[-1]
if True: # HF TESTING: not (options.abortOnMajorError and len(modelXbrl.errors) > 0):
for pluginXbrlMethod in pluginClassMethods("EdgarRenderer.Xbrl.Run"):
pluginXbrlMethod(cntlr, options, modelXbrl, modelManager.efmFiling, _report)
def validateRssFeed(self):
self.modelXbrl.info("info", "RSS Feed", modelDocument=self.modelXbrl)
from arelle.FileSource import openFileSource
reloadCache = getattr(self.modelXbrl, "reloadCache", False)
for rssItem in self.modelXbrl.modelDocument.rssItems:
if getattr(rssItem, "skipRssItem", False):
self.modelXbrl.info("info", _("skipping RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"),
modelObject=rssItem, accessionNumber=rssItem.accessionNumber, formType=rssItem.formType, companyName=rssItem.companyName, period=rssItem.period)
continue
self.modelXbrl.info("info", _("RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"),
modelObject=rssItem, accessionNumber=rssItem.accessionNumber, formType=rssItem.formType, companyName=rssItem.companyName, period=rssItem.period)
modelXbrl = None
try:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
openFileSource(rssItem.zippedUrl, self.modelXbrl.modelManager.cntlr, reloadCache=reloadCache),
_("validating"), rssItem=rssItem)
for pluginXbrlMethod in pluginClassMethods("RssItem.Xbrl.Loaded"):
pluginXbrlMethod(modelXbrl, {}, rssItem)
if getattr(rssItem, "doNotProcessRSSitem", False) or modelXbrl.modelDocument is None:
modelXbrl.close()
continue # skip entry based on processing criteria
self.instValidator.validate(modelXbrl, self.modelXbrl.modelManager.formulaOptions.typedParameters(self.modelXbrl.prefixedNamespaces))
self.instValidator.close()
rssItem.setResults(modelXbrl)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, rssItem.objectId())
for pluginXbrlMethod in pluginClassMethods("Validate.RssItem"):
pluginXbrlMethod(self, modelXbrl, rssItem)
modelXbrl.close()
except Exception as err:
self.modelXbrl.error("exception:" + type(err).__name__,
_("RSS item validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=(self.modelXbrl, modelXbrl),
instance=rssItem.zippedUrl, error=err,
exc_info=True)
try:
self.instValidator.close()
if modelXbrl is not None:
modelXbrl.close()
except Exception as err:
pass
del modelXbrl # completely dereference
def filingEnd(cntlr, options, filesource, entrypointFiles, sourceZipStream=None, responseZipStream=None, *args, **kwargs):
#cntlr.addToLog("TRACE EFM filing end")
modelManager = cntlr.modelManager
if hasattr(modelManager, "efmFiling"):
for pluginXbrlMethod in pluginClassMethods("EdgarRenderer.Filing.End"):
pluginXbrlMethod(cntlr, options, filesource, modelManager.efmFiling)
#cntlr.addToLog("TRACE EdgarRenderer end")
# save JSON file of instances and referenced documents
filingReferences = dict((report.url, report)
for report in modelManager.efmFiling.reports)
modelManager.efmFiling.close()
del modelManager.efmFiling
def commandLineXbrlRun(cntlr, options, modelXbrl, *args, **kwargs):
# skip if another class handles saving (e.g., EdgarRenderer)
for pluginXbrlMethod in pluginClassMethods('InlineDocumentSet.SavesTargetInstance'):
if pluginXbrlMethod():
return # saving of target instance is handled by another class
# extend XBRL-loaded run processing for this option
if getattr(options, "saveTargetInstance", False) or getattr(options, "saveTargetFiling", False):
if cntlr.modelManager is None or cntlr.modelManager.modelXbrl is None or (
cntlr.modelManager.modelXbrl.modelDocument.type not in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET)):
cntlr.addToLog("No inline XBRL document or manifest loaded.")
return
runSaveTargetDocumentMenuCommand(cntlr,
runInBackground=False,
saveTargetFiling=getattr(options, "saveTargetFiling", False))
def expected(self):
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ExpectedResult"):
expected = pluginXbrlMethod(self)
if expected:
return expected
# default behavior without plugins
if self.localName == "testcase":
return self.document.basename[:4] #starts with PASS or FAIL
elif self.localName == "testGroup": #w3c testcase
instanceTestElement = XmlUtil.descendant(self, None, "instanceTest")
if instanceTestElement is not None: # take instance first
return XmlUtil.descendantAttr(instanceTestElement, None, "expected", "validity")
else:
schemaTestElement = XmlUtil.descendant(self, None, "schemaTest")
if schemaTestElement is not None:
return XmlUtil.descendantAttr(schemaTestElement, None, "expected", "validity")
resultElement = XmlUtil.descendant(self, None, "result")
if resultElement is not None:
expected = resultElement.get("expected")
if expected and resultElement.get("nonStandardErrorCodes") == "true":
# if @expected and @nonStandardErrorCodes then use expected instead of error codes
return expected
errorElement = XmlUtil.descendant(self, None, "error")
resultElement = XmlUtil.descendant(self, None, "result")
if errorElement is not None and not errorElement.get("nonStandardErrorCodes"):
_errorText = XmlUtil.text(errorElement)
if ' ' in _errorText: # list of tokens
return _errorText
return ModelValue.qname(errorElement, _errorText) # turn into a QName
if resultElement is not None:
if expected:
return expected
for assertElement in XmlUtil.children(resultElement, None, "assert"):
num = assertElement.get("num")
if num == "99999": # inline test, use name as expected
return assertElement.get("name")
if len(num) == 5:
return "EFM.{0}.{1}.{2}".format(num[0],num[1:3],num[3:6])
asserTests = {}
for atElt in XmlUtil.children(resultElement, None, "assertionTests"):
try:
asserTests[atElt.get("assertionID")] = (_INT(atElt.get("countSatisfied")),_INT(atElt.get("countNotSatisfied")))
except ValueError:
pass
if asserTests:
return asserTests
elif self.get("result"):
return self.get("result")
return None
def runSaveTargetDocumentMenuCommand(cntlr, runInBackground=False, saveTargetFiling=False):
# skip if another class handles saving (e.g., EdgarRenderer)
for pluginXbrlMethod in pluginClassMethods('InlineDocumentSet.SavesTargetInstance'):
if pluginXbrlMethod():
return # saving of target instance is handled by another class
# save DTS menu item has been invoked
if (cntlr.modelManager is None or
cntlr.modelManager.modelXbrl is None or
cntlr.modelManager.modelXbrl.modelDocument.type not in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET)):
cntlr.addToLog("No inline XBRL document set loaded.")
return
modelDocument = cntlr.modelManager.modelXbrl.modelDocument
if modelDocument.type == Type.INLINEXBRLDOCUMENTSET:
targetFilename = modelDocument.targetDocumentPreferredFilename
targetSchemaRefs = modelDocument.targetDocumentSchemaRefs
else:
filepath, fileext = os.path.splitext(modelDocument.filepath)
if fileext not in (".xml", ".xbrl"):
fileext = ".xbrl"
targetFilename = filepath + fileext
targetSchemaRefs = set(modelDocument.relativeUri(referencedDoc.uri)
for referencedDoc in modelDocument.referencesDocument.keys()
if referencedDoc.type == Type.SCHEMA)
if runInBackground:
import threading
thread = threading.Thread(target=lambda _x = modelDocument.modelXbrl, _f = targetFilename, _s = targetSchemaRefs:
saveTargetDocument(_x, _f, _s))
thread.daemon = True
thread.start()
else:
if saveTargetFiling:
targetFilename = os.path.basename(targetFilename)
filingZip = zipfile.ZipFile(saveTargetFiling, 'w', zipfile.ZIP_DEFLATED, True)
filingFiles = set()
# copy referencedDocs to two levels
def addRefDocs(doc):
for refDoc in doc.referencesDocument.keys():
if refDoc.uri not in filingFiles:
filingFiles.add(refDoc.uri)
addRefDocs(refDoc)
addRefDocs(modelDocument)
else:
filingZip = None
filingFiles = None
saveTargetDocument(modelDocument.modelXbrl, targetFilename, targetSchemaRefs, filingZip, filingFiles)
if saveTargetFiling:
instDir = os.path.dirname(modelDocument.uri.split(IXDS_DOC_SEPARATOR)[0])
for refFile in filingFiles:
if refFile.startswith(instDir):
filingZip.write(refFile, modelDocument.relativeUri(refFile))
def expected(self):
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ExpectedResult"):
expected = pluginXbrlMethod(self)
if expected:
return expected
# default behavior without plugins
if self.localName == "testcase":
return self.document.basename[:4] #starts with PASS or FAIL
elif self.localName == "testGroup": #w3c testcase
instanceTestElement = XmlUtil.descendant(self, None, "instanceTest")
if instanceTestElement is not None: # take instance first
return XmlUtil.descendantAttr(instanceTestElement, None, "expected", "validity")
else:
schemaTestElement = XmlUtil.descendant(self, None, "schemaTest")
if schemaTestElement is not None:
return XmlUtil.descendantAttr(schemaTestElement, None, "expected", "validity")
resultElement = XmlUtil.descendant(self, None, "result")
if resultElement is not None:
expected = resultElement.get("expected")
if expected and resultElement.get("nonStandardErrorCodes") == "true":
# if @expected and @nonStandardErrorCodes then use expected instead of error codes
return expected
errorElement = XmlUtil.descendant(self, None, "error")
if errorElement is not None:
_errorText = XmlUtil.text(errorElement)
if ' ' in _errorText: # list of tokens
return _errorText
return ModelValue.qname(errorElement, _errorText) # turn into a QName
if resultElement is not None:
if expected:
return expected
for assertElement in XmlUtil.children(resultElement, None, "assert"):
num = assertElement.get("num")
if num == "99999": # inline test, use name as expected
return assertElement.get("name")
if len(num) == 5:
return "EFM.{0}.{1}.{2}".format(num[0],num[1:3],num[3:6])
asserTests = {}
for atElt in XmlUtil.children(resultElement, None, "assertionTests"):
try:
asserTests[atElt.get("assertionID")] = (_INT(atElt.get("countSatisfied")),_INT(atElt.get("countNotSatisfied")))
except ValueError:
pass
if asserTests:
return asserTests
elif self.get("result"):
return self.get("result")
return None
def severityLevel(self):
for pluginXbrlMethod in pluginClassMethods(
"ModelTestcaseVariation.ExpectedSeverity"):
severityLevelName = pluginXbrlMethod(self)
if severityLevelName: # ignore plug in if not a plug-in-recognized test case
return logging._checkLevel(severityLevelName)
# default behavior without plugins
# SEC error cases have <assert severity={err|wrn}>...
if XmlUtil.descendant(self,
None,
"assert",
attrName="severity",
attrValue="wrn") is not None:
return logging._checkLevel("WARNING")
return logging._checkLevel("INCONSISTENCY")
def commandLineXbrlRun(cntlr, options, modelXbrl, *args, **kwargs):
# skip if another class handles saving (e.g., EdgarRenderer)
for pluginXbrlMethod in pluginClassMethods('InlineDocumentSet.SavesTargetInstance'):
if pluginXbrlMethod():
return # saving of target instance is handled by another class
# extend XBRL-loaded run processing for this option
if getattr(options, "saveTargetInstance", False) or getattr(options, "saveTargetFiling", False):
if cntlr.modelManager is None or cntlr.modelManager.modelXbrl is None or (
cntlr.modelManager.modelXbrl.modelDocument.type not in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET)):
cntlr.addToLog("No inline XBRL document or manifest loaded.")
return
runSaveTargetDocumentMenuCommand(cntlr,
runInBackground=False,
saveTargetFiling=getattr(options, "saveTargetFiling", False),
encodeSavedXmlChars=getattr(options, "encodeSavedXmlChars", False))
def load(self,
filesource,
nextaction=None,
taxonomyPackages=None,
**kwargs):
"""Load an entry point modelDocument object(s), which in turn load documents they discover
(for the case of instance, taxonomies, and versioning reports), but defer loading instances
for test case and RSS feeds.
The modelXbrl that is loaded is 'stacked', by this class, so that any modelXbrl operations such as validate,
and close, operate on the most recently loaded modelXbrl, and compareDTSes operates on the two
most recently loaded modelXbrl's.
:param filesource: may be a FileSource object, with the entry point selected, or string file name (or web URL).
:type filesource: FileSource or str
:param nextAction: status line text string, if any, to show upon completion
:type nextAction: str
:param taxonomyPackages: array of URLs of taxonomy packages required for load operation
"""
if taxonomyPackages:
resetPackageMappings = False
for pkgUrl in taxonomyPackages:
if PackageManager.addPackage(self.cntlr, pkgUrl):
resetPackageMappings = True
if resetPackageMappings:
PackageManager.rebuildRemappings(self.cntlr)
try:
if filesource.url.startswith(
"urn:uuid:"): # request for an open modelXbrl
for modelXbrl in self.loadedModelXbrls:
if not modelXbrl.isClosed and modelXbrl.uuid == filesource.url:
return modelXbrl
raise IOError(
_("Open file handle is not open: {0}").format(
filesource.url))
except AttributeError:
pass # filesource may be a string, which has no url attribute
self.filesource = filesource
modelXbrl = None # loaded modelXbrl
for customLoader in pluginClassMethods("ModelManager.Load"):
modelXbrl = customLoader(self, filesource)
if modelXbrl is not None:
break # custom loader did the loading
if modelXbrl is None: # use default xbrl loader
modelXbrl = ModelXbrl.load(self, filesource, nextaction, **kwargs)
self.modelXbrl = modelXbrl
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def exists(self, filepath):
archiveFileSource = self.fileSourceContainingFilepath(filepath)
if archiveFileSource is not None:
if filepath.startswith(archiveFileSource.basefile):
archiveFileName = filepath[len(archiveFileSource.basefile) + 1:]
else: # filepath.startswith(self.baseurl)
archiveFileName = filepath[len(archiveFileSource.baseurl) + 1:]
if (archiveFileSource.isZip or archiveFileSource.isTarGz or
archiveFileSource.isEis or archiveFileSource.isXfd or
archiveFileSource.isRss or self.isInstalledTaxonomyPackage):
return archiveFileName.replace("\\","/") in archiveFileSource.dir
for pluginMethod in pluginClassMethods("FileSource.Exists"): #custom overrides for decription, etc
existsResult = pluginMethod(self.cntlr, filepath)
if existsResult is not None:
return existsResult
# assume it may be a plain ordinary file path
return os.path.exists(filepath)
def testcaseVariationXbrlLoaded(testcaseModelXbrl, instanceModelXbrl):
# Validate of RSS feed item or testcase variation (simulates filing & cmd line load events
modelManager = instanceModelXbrl.modelManager
if (hasattr(testcaseModelXbrl, "efmOptions") and
modelManager.validateDisclosureSystem and getattr(modelManager.disclosureSystem, "EFMplugin", False) and
(instanceModelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE or
instanceModelXbrl.modelDocument.type == ModelDocument.Type.INLINEXBRL)):
cntlr = modelManager.cntlr
options = testcaseModelXbrl.efmOptions
entrypointFiles = [{"file":instanceModelXbrl.modelDocument.uri}]
modelManager.efmFiling = Filing(cntlr, options, instanceModelXbrl.fileSource, entrypointFiles, None, None)
# this event is called for filings (of instances) as well as test cases, for test case it just keeps options accessible
for pluginXbrlMethod in pluginClassMethods("EdgarRenderer.Filing.Start"):
pluginXbrlMethod(cntlr, options, entrypointFiles, modelManager.efmFiling)
modelManager.efmFiling.addReport(instanceModelXbrl)
_report = modelManager.efmFiling.reports[-1]
_report.entryPoint = entrypointFiles[0]
def exists(self, filepath):
archiveFileSource = self.fileSourceContainingFilepath(filepath)
if archiveFileSource is not None:
if filepath.startswith(archiveFileSource.basefile):
archiveFileName = filepath[len(archiveFileSource.basefile) + 1:]
else: # filepath.startswith(self.baseurl)
archiveFileName = filepath[len(archiveFileSource.baseurl) + 1:]
if (archiveFileSource.isZip or archiveFileSource.isTarGz or
archiveFileSource.isEis or archiveFileSource.isXfd or
archiveFileSource.isRss or self.isInstalledTaxonomyPackage):
return archiveFileName.replace("\\","/") in archiveFileSource.dir
for pluginMethod in pluginClassMethods("FileSource.Exists"): #custom overrides for decription, etc
existsResult = pluginMethod(self.cntlr, filepath)
if existsResult is not None:
return existsResult
# assume it may be a plain ordinary file path
return os.path.exists(filepath)
def saveTargetDocument(filing,
modelXbrl,
targetDocumentFilename,
targetDocumentSchemaRefs,
outputZip=None,
filingFiles=None,
suffix=DEFAULT_DISTINGUISHING_SUFFIX,
iext=DEFAULT_INSTANCE_EXT):
sourceDir = os.path.dirname(modelXbrl.modelDocument.filepath)
targetUrlParts = targetDocumentFilename.rpartition(".")
targetUrl = targetUrlParts[0] + suffix + targetUrlParts[2]
modelXbrl.modelManager.showStatus(
_("Extracting instance ") + os.path.basename(targetUrl))
for pluginXbrlMethod in pluginClassMethods(
"InlineDocumentSet.CreateTargetInstance"):
targetInstance = pluginXbrlMethod(
modelXbrl,
targetUrl,
targetDocumentSchemaRefs,
filingFiles,
# no lang on xbrl:xbrl, specific xml:lang on elements which aren't en-US
baseXmlLang=None,
defaultXmlLang="en-US")
if outputZip:
targetInstance.saveInstance(overrideFilepath=targetUrl,
outputZip=outputZip,
updateFileHistory=False,
xmlcharrefreplace=True)
else:
fh = io.StringIO()
targetInstance.saveInstance(overrideFilepath=targetUrl,
outputFile=fh,
updateFileHistory=False,
xmlcharrefreplace=True)
fh.seek(0)
filing.writeFile(targetUrl, fh.read())
fh.close()
if getattr(modelXbrl, "isTestcaseVariation", False):
modelXbrl.extractedInlineInstance = True # for validation comparison
modelXbrl.modelManager.showStatus(_("Saved extracted instance"),
clearAfter=5000)
return # there can only be one "InlineDocumentSet.CreateTargetInstance" but just to be sure
cntlr.logTrace(
_("Unable to save extracted document, missing plugin class \"InlineDocumentSet.CreateTargetInstance\"."
))
def filingStart(cntlr,
options,
filesource,
entrypointFiles,
sourceZipStream=None,
responseZipStream=None):
modelManager = cntlr.modelManager
# cntlr.addToLog("TRACE EFM filing start val={} plugin={}".format(modelManager.validateDisclosureSystem, getattr(modelManager.disclosureSystem, "EFMplugin", False)))
if modelManager.validateDisclosureSystem and getattr(
modelManager.disclosureSystem, "EFMplugin", False):
# cntlr.addToLog("TRACE EFM filing start 2 classes={} moduleInfos={}".format(pluginMethodsForClasses, modulePluginInfos))
modelManager.efmFiling = Filing(cntlr, options, filesource,
entrypointFiles, sourceZipStream,
responseZipStream)
# this event is called for filings (of instances) as well as test cases, for test case it just keeps options accessible
for pluginXbrlMethod in pluginClassMethods(
"EdgarRenderer.Filing.Start"):
pluginXbrlMethod(cntlr, options, entrypointFiles,
modelManager.efmFiling)
def filingEnd(cntlr,
options,
filesource,
entrypointFiles,
sourceZipStream=None,
responseZipStream=None):
#cntlr.addToLog("TRACE EFM filing end")
modelManager = cntlr.modelManager
if hasattr(modelManager, "efmFiling"):
for pluginXbrlMethod in pluginClassMethods("EdgarRenderer.Filing.End"):
pluginXbrlMethod(cntlr, options, filesource,
modelManager.efmFiling)
#cntlr.addToLog("TRACE EdgarRenderer end")
# save JSON file of instances and referenced documents
filingReferences = dict(
(report.url, report) for report in modelManager.efmFiling.reports)
modelManager.efmFiling.close()
del modelManager.efmFiling
def readMeFirstUris(self):
try:
return self._readMeFirstUris
except AttributeError:
self._readMeFirstUris = []
# first look if any plugin method to get readme first URIs
if not any(pluginXbrlMethod(self)
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReadMeFirstUris")):
if self.localName == "testGroup": #w3c testcase
instanceTestElement = XmlUtil.descendant(self, None, "instanceTest")
if instanceTestElement is not None: # take instance first
self._readMeFirstUris.append(XmlUtil.descendantAttr(instanceTestElement, None,
"instanceDocument",
"{http://www.w3.org/1999/xlink}href"))
else:
schemaTestElement = XmlUtil.descendant(self, None, "schemaTest")
if schemaTestElement is not None:
self._readMeFirstUris.append(XmlUtil.descendantAttr(schemaTestElement, None,
"schemaDocument",
"{http://www.w3.org/1999/xlink}href"))
elif self.localName == "test-case": #xpath testcase
inputFileElement = XmlUtil.descendant(self, None, "input-file")
if inputFileElement is not None: # take instance first
self._readMeFirstUris.append("TestSources/" + inputFileElement.text + ".xml")
elif self.resultIsTaxonomyPackage:
self._readMeFirstUris.append( os.path.join(self.modelDocument.filepathdir, "tests", self.get("name") + ".zip") )
else:
# default built-in method for readme first uris
for anElement in self.iterdescendants():
if isinstance(anElement,ModelObject) and anElement.get("readMeFirst") == "true":
if anElement.get("{http://www.w3.org/1999/xlink}href"):
uri = anElement.get("{http://www.w3.org/1999/xlink}href")
else:
uri = XmlUtil.innerText(anElement)
if anElement.get("name"):
self._readMeFirstUris.append( (ModelValue.qname(anElement, anElement.get("name")), uri) )
elif anElement.get("dts"):
self._readMeFirstUris.append( (anElement.get("dts"), uri) )
else:
self._readMeFirstUris.append(uri)
if not self._readMeFirstUris: # provide a dummy empty instance document
self._readMeFirstUris.append(os.path.join(self.modelXbrl.modelManager.cntlr.configDir, "empty-instance.xml"))
return self._readMeFirstUris
def resetProxies(self, httpProxyTuple):
# for ntlm user and password are required
self.hasNTLM = False
if isinstance(httpProxyTuple,
(tuple, list)) and len(httpProxyTuple) == 5:
useOsProxy, _urlAddr, _urlPort, user, password = httpProxyTuple
_proxyDirFmt = proxyDirFmt(httpProxyTuple)
# only try ntlm if user and password are provided because passman is needed
if user and not useOsProxy:
for pluginXbrlMethod in pluginClassMethods(
"Proxy.HTTPNtlmAuthHandler"):
HTTPNtlmAuthHandler = pluginXbrlMethod()
if HTTPNtlmAuthHandler is not None:
self.hasNTLM = True
if not self.hasNTLM: # try for python site-packages ntlm
try:
from ntlm import HTTPNtlmAuthHandler
self.hasNTLM = True
except ImportError:
pass
if self.hasNTLM:
pwrdmgr = proxyhandlers.HTTPPasswordMgrWithDefaultRealm()
pwrdmgr.add_password(None, _proxyDirFmt["http"], user,
password)
self.proxy_handler = proxyhandlers.ProxyHandler({})
self.proxy_auth_handler = proxyhandlers.ProxyBasicAuthHandler(
pwrdmgr)
self.http_auth_handler = proxyhandlers.HTTPBasicAuthHandler(
pwrdmgr)
self.ntlm_auth_handler = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(
pwrdmgr)
self.opener = proxyhandlers.build_opener(
self.proxy_handler, self.ntlm_auth_handler,
self.proxy_auth_handler, self.http_auth_handler)
if not self.hasNTLM:
self.proxy_handler = proxyhandlers.ProxyHandler(
proxyDirFmt(httpProxyTuple))
self.proxy_auth_handler = proxyhandlers.ProxyBasicAuthHandler()
self.http_auth_handler = proxyhandlers.HTTPBasicAuthHandler()
self.opener = proxyhandlers.build_opener(self.proxy_handler,
self.proxy_auth_handler,
self.http_auth_handler)
def __init__(self, modelXbrl, inputXbrlInstance, sourceElement, inScopeVars=None):
self.modelXbrl = modelXbrl
self.isRunTimeExceeded = False
self.inputXbrlInstance = inputXbrlInstance
self.outputLastContext = {} # last context element output per output instance
self.outputLastUnit = {}
self.outputLastFact = {}
self.outputFirstFact = {}
self.sourceElement = sourceElement
self.contextItem = self.inputXbrlInstance.targetXbrlRootElement
self.progHeader = None
self.traceType = None
self.variableSet = None
self.inScopeVars = {} if inScopeVars is None else inScopeVars
self.cachedFilterResults = {}
if inputXbrlInstance:
self.inScopeVars[XbrlConst.qnStandardInputInstance] = inputXbrlInstance.modelXbrl
self.customFunctions = {}
for pluginXbrlMethod in pluginClassMethods("Formula.CustomFunctions"):
self.customFunctions.update(pluginXbrlMethod())
def __init__(self, modelXbrl, inputXbrlInstance, sourceElement, inScopeVars=None):
self.modelXbrl = modelXbrl
self.isRunTimeExceeded = False
self.inputXbrlInstance = inputXbrlInstance
self.outputLastContext = {} # last context element output per output instance
self.outputLastUnit = {}
self.outputLastFact = {}
self.outputFirstFact = {}
self.sourceElement = sourceElement
self.contextItem = self.inputXbrlInstance.xmlRootElement
self.progHeader = None
self.traceType = None
self.variableSet = None
self.inScopeVars = {} if inScopeVars is None else inScopeVars
self.cachedFilterResults = {}
if inputXbrlInstance:
self.inScopeVars[XbrlConst.qnStandardInputInstance] = inputXbrlInstance.modelXbrl
self.customFunctions = {}
for pluginXbrlMethod in pluginClassMethods("Formula.CustomFunctions"):
self.customFunctions.update(pluginXbrlMethod())
def __init__(self,
cntlr,
options=None,
filesource=None,
entrypointfiles=None,
sourceZipStream=None,
responseZipStream=None,
errorCaptureLevel=None):
self.cntlr = cntlr
self.options = options
self.filesource = filesource
self.entrypointfiles = entrypointfiles
self.sourceZipStream = sourceZipStream
self.responseZipStream = responseZipStream
self.submissionType = None
self.reports = []
self.renderedFiles = set() # filing-level rendered files
self.reportZip = None
if responseZipStream:
self.setReportZipStreamMode('w')
else:
try: #zipOutputFile only present with EdgarRenderer plugin options
if options and options.zipOutputFile:
if not os.path.isabs(options.zipOutputFile):
zipOutDir = os.path.dirname(filesource.basefile)
zipOutFile = os.path.join(zipOutDir,
options.zipOutputFile)
else:
zipOutFile = options.zipOutputFile
self.reportZip = zipfile.ZipFile(zipOutFile, 'w',
zipfile.ZIP_DEFLATED,
True)
except AttributeError:
self.reportZip = None
self.errorCaptureLevel = errorCaptureLevel or logging._checkLevel(
"INCONSISTENCY")
self.errors = []
self.arelleUnitTests = {} # copied from each instance loaded
for pluginXbrlMethod in pluginClassMethods("Security.Crypt.Init"):
pluginXbrlMethod(self, options, filesource, entrypointfiles,
sourceZipStream)
def start(self, tag, attrib, nsmap=None):
modelXbrl.streamingParentModelObject = self.currentMdlObj # pass parent to makeelement for ModelObjectFactory
mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
mdlObj.sourceline = 1
if self.newTree:
self.newTree = False
self.currentMdlObj = mdlObj
modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
modelXbrl.modelDocument = modelDocument # needed for incremental validation
mdlObj.init(modelDocument)
modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement
modelDocument.parserLookupName = _parserLookupName
modelDocument.parserLookupClass = _parserLookupClass
modelDocument.xmlRootElement = mdlObj
modelDocument.schemaLocationElements.add(mdlObj)
modelDocument.documentEncoding = _encoding
modelDocument._creationSoftwareComment = creationSoftwareComment
modelXbrl.info("streamingExtensions:streaming",
_("Stream processing this instance."),
modelObject = modelDocument)
else:
self.currentMdlObj.append(mdlObj)
self.currentMdlObj = mdlObj
mdlObj._init()
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if (self.beforeInstanceStream and (
(ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
(ns == XbrlConst.xbrli and ln in ("context", "unit")) or
(ns not in (XbrlConst.link, XbrlConst.xbrli)))):
self.beforeInstanceStream = False
if _streamingExtensionsValidate:
instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters())
else: # need default dimensions
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
elif not self.beforeInstanceStream and self.beforeStartStreamingPlugin:
for pluginMethod in pluginClassMethods("Streaming.Start"):
pluginMethod(modelXbrl)
self.beforeStartStreamingPlugin = False
return mdlObj
def expected(self):
for pluginXbrlMethod in pluginClassMethods(u"ModelTestcaseVariation.ExpectedResult"):
expected = pluginXbrlMethod(self)
if expected:
return expected
# default behavior without plugins
if self.localName == u"testcase":
return self.document.basename[:4] #starts with PASS or FAIL
elif self.localName == u"testGroup": #w3c testcase
instanceTestElement = XmlUtil.descendant(self, None, u"instanceTest")
if instanceTestElement is not None: # take instance first
return XmlUtil.descendantAttr(instanceTestElement, None, u"expected", u"validity")
else:
schemaTestElement = XmlUtil.descendant(self, None, u"schemaTest")
if schemaTestElement is not None:
return XmlUtil.descendantAttr(schemaTestElement, None, u"expected", u"validity")
errorElement = XmlUtil.descendant(self, None, u"error")
if errorElement is not None:
return ModelValue.qname(errorElement, XmlUtil.text(errorElement))
resultElement = XmlUtil.descendant(self, None, u"result")
if resultElement is not None:
expected = resultElement.get(u"expected")
if expected:
return expected
for assertElement in XmlUtil.children(resultElement, None, u"assert"):
num = assertElement.get(u"num")
if len(num) == 5:
return u"EFM.{0}.{1}.{2}".format(num[0],num[1:3],num[3:6])
asserTests = {}
for atElt in XmlUtil.children(resultElement, None, u"assertionTests"):
try:
asserTests[atElt.get(u"assertionID")] = (_INT(atElt.get(u"countSatisfied")),_INT(atElt.get(u"countNotSatisfied")))
except ValueError:
pass
if asserTests:
return asserTests
elif self.get(u"result"):
return self.get(u"result")
return None
def resetProxies(self, httpProxyTuple):
# for ntlm user and password are required
self.hasNTLM = False
self._httpProxyTuple = httpProxyTuple # save for resetting in noCertificateCheck setter
if isinstance(httpProxyTuple,(tuple,list)) and len(httpProxyTuple) == 5:
useOsProxy, _urlAddr, _urlPort, user, password = httpProxyTuple
_proxyDirFmt = proxyDirFmt(httpProxyTuple)
# only try ntlm if user and password are provided because passman is needed
if user and not useOsProxy:
for pluginXbrlMethod in pluginClassMethods("Proxy.HTTPNtlmAuthHandler"):
HTTPNtlmAuthHandler = pluginXbrlMethod()
if HTTPNtlmAuthHandler is not None:
self.hasNTLM = True
if not self.hasNTLM: # try for python site-packages ntlm
try:
from ntlm import HTTPNtlmAuthHandler
self.hasNTLM = True
except ImportError:
pass
if self.hasNTLM:
pwrdmgr = proxyhandlers.HTTPPasswordMgrWithDefaultRealm()
pwrdmgr.add_password(None, _proxyDirFmt["http"], user, password)
self.proxy_handler = proxyhandlers.ProxyHandler({})
self.proxy_auth_handler = proxyhandlers.ProxyBasicAuthHandler(pwrdmgr)
self.http_auth_handler = proxyhandlers.HTTPBasicAuthHandler(pwrdmgr)
self.ntlm_auth_handler = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(pwrdmgr)
proxyHandlers = [self.proxy_handler, self.ntlm_auth_handler, self.proxy_auth_handler, self.http_auth_handler]
if not self.hasNTLM:
self.proxy_handler = proxyhandlers.ProxyHandler(proxyDirFmt(httpProxyTuple))
self.proxy_auth_handler = proxyhandlers.ProxyBasicAuthHandler()
self.http_auth_handler = proxyhandlers.HTTPBasicAuthHandler()
proxyHandlers = [self.proxy_handler, self.proxy_auth_handler, self.http_auth_handler]
if ssl and self.noCertificateCheck:
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
proxyHandlers.append(proxyhandlers.HTTPSHandler(context=context))
self.opener = proxyhandlers.build_opener(*proxyHandlers)
def testcaseVariationXbrlLoaded(testcaseModelXbrl, instanceModelXbrl):
# Validate of RSS feed item or testcase variation (simulates filing & cmd line load events
modelManager = instanceModelXbrl.modelManager
if (hasattr(testcaseModelXbrl, "efmOptions")
and modelManager.validateDisclosureSystem
and getattr(modelManager.disclosureSystem, "EFMplugin", False) and
(instanceModelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE
or instanceModelXbrl.modelDocument.type
== ModelDocument.Type.INLINEXBRL)):
cntlr = modelManager.cntlr
options = testcaseModelXbrl.efmOptions
entrypointFiles = [{"file": instanceModelXbrl.modelDocument.uri}]
modelManager.efmFiling = Filing(cntlr, options,
instanceModelXbrl.fileSource,
entrypointFiles, None, None)
# this event is called for filings (of instances) as well as test cases, for test case it just keeps options accessible
for pluginXbrlMethod in pluginClassMethods(
"EdgarRenderer.Filing.Start"):
pluginXbrlMethod(cntlr, options, entrypointFiles,
modelManager.efmFiling)
modelManager.efmFiling.addReport(instanceModelXbrl)
_report = modelManager.efmFiling.reports[-1]
_report.entryPoint = entrypointFiles[0]
def testcaseVariationXbrlLoaded(testcaseModelXbrl, instanceModelXbrl,
modelTestcaseVariation, *args, **kwargs):
# Validate of RSS feed item or testcase variation (simulates filing & cmd line load events
modelManager = instanceModelXbrl.modelManager
if (hasattr(testcaseModelXbrl, "efmOptions")
and modelManager.validateDisclosureSystem
and getattr(modelManager.disclosureSystem, "EFMplugin", False) and
(instanceModelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE
or instanceModelXbrl.modelDocument.type
== ModelDocument.Type.INLINEXBRL)):
cntlr = modelManager.cntlr
options = testcaseModelXbrl.efmOptions
entrypointFiles = [{"file": instanceModelXbrl.modelDocument.uri}]
if not hasattr(modelManager, "efmFiling"): # first instance of filing
modelManager.efmFiling = Filing(
cntlr, options, instanceModelXbrl.fileSource, entrypointFiles,
None, None, instanceModelXbrl.errorCaptureLevel)
# this event is called for filings (of instances) as well as test cases, for test case it just keeps options accessible
for pluginXbrlMethod in pluginClassMethods(
"EdgarRenderer.Filing.Start"):
pluginXbrlMethod(cntlr, options, entrypointFiles,
modelManager.efmFiling)
modelManager.efmFiling.addReport(instanceModelXbrl)
_report = modelManager.efmFiling.reports[-1]
_report.entryPoint = entrypointFiles[0]
modelManager.efmFiling.arelleUnitTests = instanceModelXbrl.arelleUnitTests.copy(
) # allow unit tests to be used after instance processing finished
# check for parameters on instance
for _instanceElt in XmlUtil.descendants(modelTestcaseVariation, "*",
"instance", "readMeFirst",
"true", False):
if instanceModelXbrl.modelDocument.uri.endswith(_instanceElt.text):
if _instanceElt.get("exhibitType"):
_report.entryPoint[
"exhibitType"] = _report.exhibitType = _instanceElt.get(
"exhibitType")
break
def test(self):
process = psutil.Process(os.getpid())
largeEntryPoint = "http://www.eba.europa.eu/eu/fr/xbrl/crr/fws/corep/its-2014-05/2015-02-16/mod/corep_ind.xsd"
smallEntryPoint = "http://eiopa.europa.eu/eu/xbrl/s2md/fws/solvency/solvency2/2015-05-31/mod/d1s.xsd"
application = Tk()
cntlrWinMain = CntlrWinMain(application)
application.protocol("WM_DELETE_WINDOW", cntlrWinMain.quit)
cntlrWinMain.setTestMode(True)
for pluginMethod in pluginClassMethods("DevTesting.GetTestContext"):
testContext = pluginMethod()
break
testContext.checkMemoryOnClose = True
testDir = os.path.dirname(os.path.abspath(sys.modules[__name__].__file__))
testContext.saveFilePath = testDir + "/tmp/a1.xbrl"
testContext.dumpFilePrefix = testDir + "/tmp/dump_"
tr = tracker.SummaryTracker()
prevMemoryConsumed = 0
for idx in range(4):
print("\nIteration " + str(idx))
cntlrWinMain.fileOpenFile(largeEntryPoint)
maxSize = DevTesting.humanizeSize(process.memory_info()[0])
cntlrWinMain.logClear()
cntlrWinMain.fileClose()
tr.print_diff()
memoryConsumed = process.memory_info()[0]
print("Memory consumed: " + DevTesting.humanizeSize(memoryConsumed) + " max= " + maxSize)
diffMemeory = memoryConsumed - prevMemoryConsumed
print(str(diffMemeory))
if idx > 1:
assert diffMemeory < 60000000, "Check memory leaks regression"
prevMemoryConsumed = memoryConsumed
def parse(modelObject, xpathExpression, element, name, traceType):
from arelle.ModelFormulaObject import Trace
global modelXbrl, pluginCustomFunctions
modelXbrl = modelObject.modelXbrl
global exprStack
exprStack = []
global xmlElement
xmlElement = element
returnProg = None
pluginCustomFunctions = {}
for pluginXbrlMethod in pluginClassMethods("Formula.CustomFunctions"):
pluginCustomFunctions.update(pluginXbrlMethod())
# throws ParseException
if xpathExpression and len(xpathExpression) > 0:
# normalize End of Line
try:
formulaOptions = modelXbrl.modelManager.formulaOptions
normalizedExpr = normalizeExpr( xpathExpression )
# for debugging parser looping or stack recursion, uncomment this:
# modelObject.modelXbrl.modelManager.showStatus(_("Parsing file {0} line {1} expr {2}").format(element.modelDocument.basename,element.sourceline,normalizedExpr))
# should be option "compiled code"
if ((formulaOptions.traceVariableSetExpressionSource and traceType == Trace.VARIABLE_SET) or
(formulaOptions.traceVariableExpressionSource and traceType == Trace.VARIABLE) or
(formulaOptions.traceCallExpressionSource and traceType == Trace.CALL)):
modelXbrl.info("formula:trace", "Source %(name)s %(source)s",
modelObject=element,
name=name,
source=normalizedExpr)
exprStack.append( ProgHeader(modelObject,name,element,normalizedExpr,traceType) )
L = xpathExpr.parseString( normalizedExpr, parseAll=True )
#modelXbrl.error( _("AST {0} {1}").format(name, L),
# "info", "formula:trace")
# should be option "compiled code"
if ((formulaOptions.traceVariableSetExpressionCode and traceType == Trace.VARIABLE_SET) or
(formulaOptions.traceVariableExpressionCode and traceType == Trace.VARIABLE) or
(formulaOptions.traceCallExpressionCode and traceType == Trace.CALL)):
modelXbrl.info("formula:trace", _("Code %(name)s %(source)s"),
modelObject=element,
name=name,
source=exprStack)
except (ParseException, ParseSyntaxException) as err:
modelXbrl.error("err:XPST0003",
_("Parse error in %(name)s error: %(error)s \n%(source)s"),
modelObject=element,
name=name,
error=err,
source=exceptionErrorIndication(err))
# insert after ProgHeader before ordinary executable expression that may have successfully compiled
exprStack.insert(1, OperationDef(normalizedExpr, 0,
QNameDef(0, "fn", XbrlConst.fn, "error"),
(OperationDef(normalizedExpr, 0,
QNameDef(0, "fn", XbrlConst.fn, "QName"),
(XbrlConst.xpath2err, "err:XPST0003"),False),
str(err)), False))
except (ValueError) as err:
modelXbrl.error("parser:unableToParse",
_("Parsing terminated in %(name)s due to error: %(error)s \n%(source)s"),
modelObject=element,
name=name,
error=err,
source=normalizedExpr)
modelXbrl.debug("debug", str(traceback.format_exception(*sys.exc_info())))
'''
code = []
compile(exprStack, code)
pyCode = ''.join(code)
val.modelXbrl.error(
_("PyCode {0} {1}").format(
name,
pyCode),
"info", "formula:trace")
return pyCode
'''
returnProg = exprStack
exprStack = [] # dereference
xmlElement = None
modelXbrl = None
return returnProg
def select(self, name):
self.clear()
if not name:
return True # nothing to load
result = False
status = _("loading disclosure system and mappings")
try:
if name:
isSelected = False
for url in self.urls: # urls in revese order, last plugin first
xmldoc = etree.parse(url)
for dsElt in xmldoc.iter(tag="DisclosureSystem"):
namesStr = dsElt.get("names")
if namesStr:
names = namesStr.split("|")
if name in names:
self.names = names
self.name = self.names[0]
self.validationType = dsElt.get(
"validationType")
self.exclusiveTypesPattern = compileAttrPattern(
dsElt,
"exclusiveTypesPattern",
patternIfNoAttr=None)
if self.validationType not in self.pluginTypes:
self.EFM = self.validationType == "EFM"
self.GFM = self.validationType == "GFM"
self.EFMorGFM = self.EFM or self.GFM
self.HMRC = self.validationType == "HMRC"
self.SBRNL = self.validationType == "SBR.NL"
for pluginXbrlMethod in pluginClassMethods(
"DisclosureSystem.Types"):
for typeName, typeTestVariable in pluginXbrlMethod(
self):
setattr(
self, typeTestVariable,
self.validationType == typeName)
self.validateFileText = dsElt.get(
"validateFileText") == "true"
if dsElt.get("allowedExternalHrefPattern"):
self.allowedExternalHrefPattern = re.compile(
dsElt.get(
"allowedExternalHrefPattern"))
self.blockDisallowedReferences = dsElt.get(
"blockDisallowedReferences") == "true"
try:
self.maxSubmissionSubdirectoryEntryNesting = int(
dsElt.get(
"maxSubmissionSubdirectoryEntryNesting"
))
except (ValueError, TypeError):
self.maxSubmissionSubdirectoryEntryNesting = 0
self.defaultXmlLang = dsElt.get(
"defaultXmlLang")
if dsElt.get(
"defaultXmlEncoding", default=None
) is not None: # don't reset from utf-8 unless supplied with a value
self.defaultXmlEncoding = dsElt.get(
"defaultXmlEncoding"
) # may be an empty string
self.xmlLangPattern = compileAttrPattern(
dsElt, "xmlLangPattern")
self.xmlLangIsInheritable = dsElt.get(
"xmlLangIsInheritable", "true") == "true"
self.defaultLanguage = dsElt.get(
"defaultLanguage")
if dsElt.get("standardTaxonomiesUrl"):
self.standardTaxonomiesUrl = self.modelManager.cntlr.webCache.normalizeUrl(
dsElt.get("standardTaxonomiesUrl"),
url)
if dsElt.get("mappingsUrl"):
self.mappingsUrl = self.modelManager.cntlr.webCache.normalizeUrl(
dsElt.get("mappingsUrl"), url)
if dsElt.get(
"utrUrl"
): # may be mapped by mappingsUrl entries, see below
self.utrUrl = self.modelManager.cntlr.webCache.normalizeUrl(
dsElt.get("utrUrl"), url)
self.identifierSchemePattern = compileAttrPattern(
dsElt, "identifierSchemePattern")
self.identifierValuePattern = compileAttrPattern(
dsElt, "identifierValuePattern")
self.identifierValueName = dsElt.get(
"identifierValueName")
self.contextElement = dsElt.get(
"contextElement")
self.roleDefinitionPattern = compileAttrPattern(
dsElt, "roleDefinitionPattern")
self.labelCheckPattern = compileAttrPattern(
dsElt, "labelCheckPattern", re.DOTALL)
self.labelTrimPattern = compileAttrPattern(
dsElt, "labelTrimPattern", re.DOTALL)
self.deiNamespacePattern = compileAttrPattern(
dsElt, "deiNamespacePattern")
self.deiAmendmentFlagElement = dsElt.get(
"deiAmendmentFlagElement")
self.deiCurrentFiscalYearEndDateElement = dsElt.get(
"deiCurrentFiscalYearEndDateElement")
self.deiDocumentFiscalYearFocusElement = dsElt.get(
"deiDocumentFiscalYearFocusElement")
self.deiDocumentPeriodEndDateElement = dsElt.get(
"deiDocumentPeriodEndDateElement")
self.deiFilerIdentifierElement = dsElt.get(
"deiFilerIdentifierElement")
self.deiFilerNameElement = dsElt.get(
"deiFilerNameElement")
self.logLevelFilter = dsElt.get(
"logLevelFilter")
self.logCodeFilter = dsElt.get("logCodeFilter")
self.standardTaxonomyDatabase = dsElt.get(
"standardTaxonomyDatabase")
self.standardTaxonomyUrlPattern = compileAttrPattern(
dsElt, "standardTaxonomyUrlPattern")
self.selection = self.name
isSelected = True
result = True
break
if isSelected:
break
self.loadMappings()
self.utrUrl = self.mappedUrl(
self.utrUrl) # utr may be mapped, change to its mapped entry
self.loadStandardTaxonomiesDict()
self.utrTypeEntries = None # clear any prior loaded entries
# set log level filters (including resetting prior disclosure systems values if no such filter)
self.modelManager.cntlr.setLogLevelFilter(
self.logLevelFilter
) # None or "" clears out prior filter if any
self.modelManager.cntlr.setLogCodeFilter(self.logCodeFilter)
if result:
status = _("loaded")
else:
status = _("unable to load disclosure system {}").format(name)
self.modelManager.cntlr.addToLog(
_("Disclosure System \"%(name)s\" not recognized (a plug-in may be needed)."
),
messageCode="arelle:disclosureSystemName",
messageArgs={"name": name},
level=logging.ERROR)
except (EnvironmentError, etree.LxmlError) as err:
status = _("exception during loading")
result = False
self.modelManager.cntlr.addToLog(
_("Disclosure System \"%(name)s\" loading error: %(error)s"),
messageCode="arelle:disclosureSystemLoadingError",
messageArgs={
"error": str(err),
"name": name
},
level=logging.ERROR)
etree.clear_error_log()
self.modelManager.cntlr.showStatus(
_("Disclosure system and mappings {0}: {1}").format(status, name),
3500)
return result
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
else: # need own file source, may need instance discovery
filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
if filesource and not filesource.selection and filesource.isArchive:
for _archiveFile in filesource.dir: # find instance document in archive
filesource.select(_archiveFile)
if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
break # use this selection
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel)
if modelXbrl.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelXbrl.close()
self.determineNotLoadedTestStatus(modelTestcaseVariation)
elif resultIsVersioningReport:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
self.modelXbrl.error("exception",
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
RenderingEvaluator.init(modelXbrl)
if modelXbrl.hasFormulae:
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
self.modelXbrl.error("exception",
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("formula:resultFactCounts",
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts))
else:
for fact in expectedInstance.facts:
unmatchedFactsStack = []
if formulaOutputInstance.matchFact(fact, unmatchedFactsStack, deemP0inf=True) is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = fact
formulaOutputInstance.error("formula:expectedFactMissing",
_("Formula output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname)
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
elif resultIsTaxonomyPackage:
from arelle import PackageManager, PrototypeInstanceObject
dtsName = readMeFirstUri
modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
else: # need own file source, may need instance discovery
filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
if filesource and not filesource.selection and filesource.isArchive:
for _archiveFile in filesource.dir: # find instance document in archive
filesource.select(_archiveFile)
if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
break # use this selection
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel)
modelXbrl.isTestcaseVariation = True
if modelXbrl.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif resultIsVersioningReport or resultIsTaxonomyPackage:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif resultIsTaxonomyPackage:
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
RenderingEvaluator.init(modelXbrl)
if modelXbrl.hasFormulae:
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
compareIxResultInstance = getattr(modelXbrl, "extractedInlineInstance", False) and modelTestcaseVariation.resultXbrlInstanceUri
if compareIxResultInstance:
formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
errMsgPrefix = "ix"
else: # delete input instances before formula output comparision
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
errMsgPrefix = "formula"
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts),
messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
else:
formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
def factFootnotes(fact, footnotesRelSet):
footnotes = []
footnoteRels = footnotesRelSet.fromModelObject(fact)
if footnoteRels:
# most process rels in same order between two instances, use labels to sort
for i, footnoteRel in enumerate(sorted(footnoteRels,
key=lambda r: (r.fromLabel,r.toLabel))):
modelObject = footnoteRel.toModelObject
if isinstance(modelObject, ModelResource):
xml = modelObject.viewText().strip()
footnotes.append("Footnote {}: {}".format(
i+1, # compare footnote with HTML serialized
xml,
#re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))
))
elif isinstance(modelObject, ModelFact):
footnotes.append("Footnoted fact {}: {} context: {} value: {}".format(
i+1,
modelObject.qname,
modelObject.contextID,
collapseWhitespace(modelObject.value)))
return footnotes
for expectedInstanceFact in expectedInstance.facts:
unmatchedFactsStack = []
formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True)
if formulaOutputFact is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = expectedInstanceFact
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else: # compare footnotes
expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
if expectedInstanceFactFootnotes != formulaOutputFactFootnotes:
formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
_("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=expectedInstanceFactFootnotes, footnotes2=formulaOutputFactFootnotes,
messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
if compareIxResultInstance:
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def runSaveTargetDocumentMenuCommand(cntlr,
runInBackground=False,
saveTargetFiling=False,
encodeSavedXmlChars=False):
# skip if another class handles saving (e.g., EdgarRenderer)
for pluginXbrlMethod in pluginClassMethods(
'InlineDocumentSet.SavesTargetInstance'):
if pluginXbrlMethod():
return # saving of target instance is handled by another class
# save DTS menu item has been invoked
if (cntlr.modelManager is None or cntlr.modelManager.modelXbrl is None
or cntlr.modelManager.modelXbrl.modelDocument.type
not in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET)):
cntlr.addToLog("No inline XBRL document set loaded.")
return
modelDocument = cntlr.modelManager.modelXbrl.modelDocument
if modelDocument.type == Type.INLINEXBRLDOCUMENTSET:
targetFilename = modelDocument.targetDocumentPreferredFilename
targetSchemaRefs = modelDocument.targetDocumentSchemaRefs
else:
filepath, fileext = os.path.splitext(modelDocument.filepath)
if fileext not in (".xml", ".xbrl"):
fileext = ".xbrl"
targetFilename = filepath + fileext
targetSchemaRefs = set(
modelDocument.relativeUri(referencedDoc.uri)
for referencedDoc in modelDocument.referencesDocument.keys()
if referencedDoc.type == Type.SCHEMA)
if runInBackground:
import threading
thread = threading.Thread(
target=lambda _x=modelDocument.modelXbrl, _f=targetFilename, _s=
targetSchemaRefs: saveTargetDocument(_x, _f, _s))
thread.daemon = True
thread.start()
else:
if saveTargetFiling:
targetFilename = os.path.basename(targetFilename)
filingZip = zipfile.ZipFile(saveTargetFiling, 'w',
zipfile.ZIP_DEFLATED, True)
filingFiles = set()
# copy referencedDocs to two levels
def addRefDocs(doc):
for refDoc in doc.referencesDocument.keys():
if refDoc.uri not in filingFiles:
filingFiles.add(refDoc.uri)
addRefDocs(refDoc)
addRefDocs(modelDocument)
else:
filingZip = None
filingFiles = None
saveTargetDocument(modelDocument.modelXbrl,
targetFilename,
targetSchemaRefs,
filingZip,
filingFiles,
encodeSavedXmlChars=encodeSavedXmlChars)
if saveTargetFiling:
instDir = os.path.dirname(
modelDocument.uri.split(IXDS_DOC_SEPARATOR)[0])
for refFile in filingFiles:
if refFile.startswith(instDir):
filingZip.write(refFile,
modelDocument.relativeUri(refFile))
def parentMenuCommand(cntl):
for i in range(1, 100):
for pluginMethod in pluginClassMethods(
"Import.Unpackaged.Entry{}".format(i)):
pluginMethod()
def streamingExtensionsLoader(modelXbrl, mappedUri, filepath, *args, **kwargs):
# check if big instance and has header with an initial incomplete tree walk (just 2 elements
if not _streamingExtensionsCheck:
return None
# track whether modelXbrl has been validated by this streaming extension
modelXbrl._streamingExtensionValidated = False
def logSyntaxErrors(parsercontext):
for error in parsercontext.error_log:
modelXbrl.error(
"xmlSchema:syntax",
_("%(error)s, %(fileName)s, line %(line)s, column %(column)s, %(sourceAction)s source element"
),
modelObject=modelXbrl,
fileName=os.path.basename(filepath),
error=error.message,
line=error.line,
column=error.column,
sourceAction="streaming")
#### note: written for iterparse of lxml prior to version 3.3, otherwise rewrite to use XmlPullParser ###
#### note: iterparse wants a binary file, but file is text mode
_file, = modelXbrl.fileSource.file(filepath, binary=True)
startedAt = time.time()
modelXbrl.profileActivity()
''' this seems twice as slow as iterparse
class instInfoTarget():
def __init__(self, element_factory=None, parser=None):
self.newTree = True
self.streamingAspects = None
self.foundInstance = False
self.creationSoftwareComment = ''
self.currentEltTag = "(before xbrli:xbrl)"
self.numRootFacts = 0
def start(self, tag, attrib, nsmap=None):
if self.newTree:
if tag == "{http://www.xbrl.org/2003/instance}xbrl":
self.foundInstance = True
self.newTree = False
else: # break
raise NotInstanceDocumentException()
elif not tag.startswith("{http://www.xbrl.org/"):
self.numRootFacts += 1
if self.numRootFacts % 1000 == 0:
modelXbrl.profileActivity("... streaming tree check", minTimeToShow=20.0)
self.currentEltTag = tag
def end(self, tag):
pass
def data(self, data):
pass
def comment(self, text):
if not self.foundInstance: # accumulate comments before xbrli:xbrl
self.creationSoftwareComment += ('\n' if self.creationSoftwareComment else '') + text
elif not self.creationSoftwareComment:
self.creationSoftwareComment = text # or first comment after xbrli:xbrl
def pi(self, target, data):
if target == "xbrl-streamable-instance":
if self.currentEltTag == "{http://www.xbrl.org/2003/instance}xbrl":
self.streamingAspects = dict(etree.PI(target,data).attrib.copy()) # dereference target results
else:
modelXbrl.error("streamingExtensions:headerMisplaced",
_("Header is misplaced: %(target)s, must follow xbrli:xbrl element but was found at %(element)s"),
modelObject=modelXbrl, target=target, element=self.currentEltTag)
def close(self):
if not self.creationSoftwareComment:
self.creationSoftwareComment = None
return True
instInfo = instInfoTarget()
infoParser = etree.XMLParser(recover=True, huge_tree=True, target=instInfo)
try:
etree.parse(_file, parser=infoParser, base_url=filepath)
except NotInstanceDocumentException:
pass
'''
foundErrors = False
foundInstance = False
streamingAspects = None
creationSoftwareComment = None
instInfoNumRootFacts = 0
numElts = 0
elt = None
instInfoContext = etree.iterparse(_file,
events=("start", "end"),
huge_tree=True)
try:
for event, elt in instInfoContext:
if event == "start":
if elt.getparent() is not None:
if elt.getparent(
).tag == "{http://www.xbrl.org/2003/instance}xbrl":
if not foundInstance:
foundInstance = True
pi = precedingProcessingInstruction(
elt, "xbrl-streamable-instance")
if pi is None:
break
else:
streamingAspects = dict(pi.attrib.copy())
if creationSoftwareComment is None:
creationSoftwareComment = precedingComment(
elt)
if not elt.tag.startswith("{http://www.xbrl.org/"):
instInfoNumRootFacts += 1
if instInfoNumRootFacts % 1000 == 0:
modelXbrl.profileActivity(
"... streaming tree check",
minTimeToShow=20.0)
elif not foundInstance:
break
elif elt.tag == "{http://www.xbrl.org/2003/instance}xbrl":
creationSoftwareComment = precedingComment(elt)
if precedingProcessingInstruction(
elt, "xbrl-streamable-instance") is not None:
modelXbrl.error(
"streamingExtensions:headerMisplaced",
_("Header is misplaced: %(error)s, must follow xbrli:xbrl element"
),
modelObject=elt)
elif event == "end":
elt.clear()
numElts += 1
if numElts % 1000 == 0 and elt.getparent() is not None:
while elt.getprevious() is not None and elt.getparent(
) is not None:
del elt.getparent()[0]
except etree.XMLSyntaxError as err:
modelXbrl.error("xmlSchema:syntax",
_("Unrecoverable error: %(error)s"),
error=err)
_file.close()
return err
_file.seek(0, io.SEEK_SET) # allow reparsing
if not foundInstance or streamingAspects is None:
del elt
_file.close()
return None
modelXbrl.profileStat(_("streaming tree check"), time.time() - startedAt)
startedAt = time.time()
try:
version = Decimal(streamingAspects.get("version"))
if int(version) != 1:
modelXbrl.error(
"streamingExtensions:unsupportedVersion",
_("Streaming version %(version)s, major version number must be 1"
),
modelObject=elt,
version=version)
foundErrors = True
except (InvalidOperation, OverflowError):
modelXbrl.error("streamingExtensions:versionError",
_("Version %(version)s, number must be 1.n"),
modelObject=elt,
version=streamingAspects.get("version", "(none)"))
foundErrors = True
for bufAspect in ("contextBuffer", "unitBuffer", "footnoteBuffer"):
try:
bufLimit = Decimal(streamingAspects.get(bufAspect, "INF"))
if bufLimit < 1 or (bufLimit.is_finite() and bufLimit % 1 != 0):
raise InvalidOperation
elif bufAspect == "contextBuffer":
contextBufferLimit = bufLimit
elif bufAspect == "unitBuffer":
unitBufferLimit = bufLimit
elif bufAspect == "footnoteBuffer":
footnoteBufferLimit = bufLimit
except InvalidOperation:
modelXbrl.error(
"streamingExtensions:valueError",
_("Streaming %(attrib)s %(value)s, number must be a positive integer or INF"
),
modelObject=elt,
attrib=bufAspect,
value=streamingAspects.get(bufAspect))
foundErrors = True
if _streamingExtensionsValidate:
incompatibleValidations = []
_validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
_disclosureSystem = modelXbrl.modelManager.disclosureSystem
if _validateDisclosureSystem and _disclosureSystem.validationType == "EFM":
incompatibleValidations.append("EFM")
if _validateDisclosureSystem and _disclosureSystem.validationType == "GFM":
incompatibleValidations.append("GFM")
if _validateDisclosureSystem and _disclosureSystem.validationType == "HMRC":
incompatibleValidations.append("HMRC")
if modelXbrl.modelManager.validateCalcLB:
incompatibleValidations.append("calculation LB")
if incompatibleValidations:
modelXbrl.error(
"streamingExtensions:incompatibleValidation",
_("Streaming instance validation does not support %(incompatibleValidations)s validation"
),
modelObject=modelXbrl,
incompatibleValidations=', '.join(incompatibleValidations))
foundErrors = True
if instInfoContext.error_log:
foundErrors = True
logSyntaxErrors(instInfoContext)
del instInfoContext # dereference
for pluginMethod in pluginClassMethods("Streaming.BlockStreaming"):
_blockingPluginName = pluginMethod(modelXbrl)
if _blockingPluginName: # name of blocking plugin is returned
modelXbrl.error(
"streamingExtensions:incompatiblePlugIn",
_("Streaming instance not supported by plugin %(blockingPlugin)s"
),
modelObject=modelXbrl,
blockingPlugin=_blockingPluginName)
foundErrors = True
if foundErrors:
_file.close()
return None
_encoding = XmlUtil.encoding(_file.read(512))
_file.seek(0, io.SEEK_SET) # allow reparsing
if _streamingExtensionsValidate:
validator = Validate(modelXbrl)
instValidator = validator.instValidator
contextBuffer = []
contextsToDrop = []
unitBuffer = []
unitsToDrop = []
footnoteBuffer = []
footnoteLinksToDrop = []
_streamingFactsPlugin = any(
True for pluginMethod in pluginClassMethods("Streaming.Facts"))
_streamingValidateFactsPlugin = (_streamingExtensionsValidate and any(
True
for pluginMethod in pluginClassMethods("Streaming.ValidateFacts")))
''' this is very much slower than iterparse
class modelLoaderTarget():
def __init__(self, element_factory=None, parser=None):
self.newTree = True
self.currentMdlObj = None
self.beforeInstanceStream = True
self.beforeStartStreamingPlugin = True
self.numRootFacts = 1
modelXbrl.makeelementParentModelObject = None
modelXbrl.isStreamingMode = True
self.factsCheckVersion = None
self.factsCheckMd5s = Md5Sum()
def start(self, tag, attrib, nsmap=None):
modelXbrl.makeelementParentModelObject = self.currentMdlObj # pass parent to makeelement for ModelObjectFactory
mdlObj = _parser.makeelement(tag, attrib=attrib, nsmap=nsmap)
mdlObj.sourceline = 1
if self.newTree:
self.newTree = False
self.currentMdlObj = mdlObj
modelDocument = ModelDocument(modelXbrl, Type.INSTANCE, mappedUri, filepath, mdlObj.getroottree())
modelXbrl.modelDocument = modelDocument # needed for incremental validation
mdlObj.init(modelDocument)
modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement
modelDocument.parserLookupName = _parserLookupName
modelDocument.parserLookupClass = _parserLookupClass
modelDocument.xmlRootElement = mdlObj
modelDocument.schemaLocationElements.add(mdlObj)
modelDocument.documentEncoding = _encoding
modelDocument._creationSoftwareComment = creationSoftwareComment
modelXbrl.info("streamingExtensions:streaming",
_("Stream processing this instance."),
modelObject = modelDocument)
else:
self.currentMdlObj.append(mdlObj)
self.currentMdlObj = mdlObj
mdlObj._init()
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if (self.beforeInstanceStream and (
(ns == XbrlConst.link and ln not in ("schemaRef", "linkbaseRef")) or
(ns == XbrlConst.xbrli and ln in ("context", "unit")) or
(ns not in (XbrlConst.link, XbrlConst.xbrli)))):
self.beforeInstanceStream = False
if _streamingExtensionsValidate:
instValidator.validate(modelXbrl, modelXbrl.modelManager.formulaOptions.typedParameters(modelXbrl.prefixedNamespaces))
else: # need default dimensions
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
elif not self.beforeInstanceStream and self.beforeStartStreamingPlugin:
for pluginMethod in pluginClassMethods("Streaming.Start"):
pluginMethod(modelXbrl)
self.beforeStartStreamingPlugin = False
return mdlObj
def end(self, tag):
modelDocument = modelXbrl.modelDocument
mdlObj = self.currentMdlObj
parentMdlObj = mdlObj.getparent()
self.currentMdlObj = parentMdlObj
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
else:
if _streamingExtensionsValidate and len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
if _streamingValidateFactsPlugin:
contextsToDrop.append(cntx)
else:
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
#>>XmlValidate.validate(modelXbrl, mdlObj)
#>>modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj,)
instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
instValidator.checkContextsDimensions(contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if _streamingExtensionsValidate and len(unitBuffer) >= unitBufferLimit:
# drop before adding as dropped may have same id as added
unit = unitBuffer.pop(0)
if _streamingValidateFactsPlugin:
unitsToDrop.append(unit)
else:
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
unit = None
#>>XmlValidate.validate(modelXbrl, mdlObj)
#>>modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkUnits( (mdlObj,) )
elif ln == "xbrl": # end of document
# check remaining batched facts if any
if _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
# finish any final batch of facts
if len(modelXbrl.facts) > 0:
factsToCheck = modelXbrl.facts.copy()
factsHaveBeenProcessed = True
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
if not pluginMethod(modelXbrl, factsToCheck):
factsHaveBeenProcessed = False
if factsHaveBeenProcessed:
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(fact)]
for cntx in contextsToDrop:
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
for unit in unitsToDrop:
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
for footnoteLink in footnoteLinksToDrop:
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
fact = cntx = unit = footnoteLink = None
del contextsToDrop[:]
del unitsToDrop[:]
del footnoteLinksToDrop[:]
del factsToCheck
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
for pluginMethod in pluginClassMethods("Streaming.Finish"):
pluginMethod(modelXbrl)
elif ns == XbrlConst.link:
if ln == "footnoteLink":
XmlValidate.validate(modelXbrl, mdlObj)
footnoteLinks = (mdlObj,)
modelDocument.linkbaseDiscover(footnoteLinks, inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
if _streamingValidateFactsPlugin:
footnoteLinksToDrop.append(footnoteLink)
else:
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elif ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(mdlObj)
elif not modelXbrl.skipDTS:
if ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj,), inInstance=True)
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl:
self.numRootFacts += 1
#>>XmlValidate.validate(modelXbrl, mdlObj)
#>>modelDocument.factDiscover(mdlObj, modelXbrl.facts)
if self.factsCheckVersion:
self.factCheckFact(mdlObj)
if _streamingExtensionsValidate or _streamingValidateFactsPlugin:
factsToCheck = (mdlObj,) # validate current fact by itself
if _streamingExtensionsValidate:
instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
instValidator.checkFactsDimensions(factsToCheck)
if _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
# use batches of 1000 facts
if len(modelXbrl.facts) > 1000:
factsToCheck = modelXbrl.facts.copy()
factsHaveBeenProcessed = True
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
for pluginMethod in pluginClassMethods("Streaming.ValidateFacts"):
if not pluginMethod(modelXbrl, factsToCheck):
factsHaveBeenProcessed = False
if factsHaveBeenProcessed:
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
del parentMdlObj[parentMdlObj.index(fact)]
for cntx in contextsToDrop:
dropContext(modelXbrl, cntx)
del parentMdlObj[parentMdlObj.index(cntx)]
for unit in unitsToDrop:
dropUnit(modelXbrl, unit)
del parentMdlObj[parentMdlObj.index(unit)]
for footnoteLink in footnoteLinksToDrop:
dropFootnoteLink(modelXbrl, footnoteLink)
del parentMdlObj[parentMdlObj.index(footnoteLink)]
fact = cntx = unit = footnoteLink = None
del contextsToDrop[:]
del unitsToDrop[:]
del footnoteLinksToDrop[:]
del factsToCheck # dereference fact or batch of facts
else:
dropFact(modelXbrl, mdlObj, modelXbrl.facts) # single fact has been processed
del parentMdlObj[parentMdlObj.index(mdlObj)]
if self.numRootFacts % 1000 == 0:
pass
#modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts,
# 100.0 * self.numRootFacts / instInfoNumRootFacts),
# minTimeToShow=20.0)
gc.collect()
sys.stdout.write ("\rAt fact {} of {} mem {}".format(self.numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed))
return mdlObj
def data(self, data):
self.currentMdlObj.text = data
def comment(self, text):
pass
def pi(self, target, data):
if target == "xbrl-facts-check":
_match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", data)
if _match:
_matchGroups = _match.groups()
if len(_matchGroups) == 2:
if _matchGroups[0] == "version":
self.factsCheckVersion = _matchGroups[1]
elif _matchGroups[0] == "sum-of-fact-md5s":
try:
expectedMd5 = Md5Sum(_matchGroups[1])
if self.factsCheckMd5s != expectedMd5:
modelXbrl.warning("streamingExtensions:xbrlFactsCheckWarning",
_("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"),
modelObject=modelXbrl, expectedMd5=expectedMd5, actualMd5Sum=self.factsCheckMd5s)
else:
modelXbrl.info("info",
_("Successful XBRL facts sum of md5s."),
modelObject=modelXbrl)
except ValueError:
modelXbrl.error("streamingExtensions:xbrlFactsCheckError",
_("Invalid sum-of-md5s %(sumOfMd5)s"),
modelObject=modelXbrl, sumOfMd5=_matchGroups[1])
def close(self):
del modelXbrl.makeelementParentModelObject
return None
def factCheckFact(self, fact):
self.factsCheckMd5s += fact.md5sum
for _tupleFact in fact.modelTupleFacts:
self.factCheckFact(_tupleFact)
_parser, _parserLookupName, _parserLookupClass = parser(modelXbrl, filepath, target=modelLoaderTarget())
etree.parse(_file, parser=_parser, base_url=filepath)
logSyntaxErrors(_parser)
'''
# replace modelLoaderTarget with iterparse (as it now supports CustomElementClassLookup)
streamingParserContext = etree.iterparse(_file,
events=("start", "end"),
huge_tree=True)
from arelle.ModelObjectFactory import setParserElementClassLookup
modelXbrl.isStreamingMode = True # must be set before setting element class lookup
(_parser, _parserLookupName,
_parserLookupClass) = setParserElementClassLookup(streamingParserContext,
modelXbrl)
foundInstance = False
beforeInstanceStream = beforeStartStreamingPlugin = True
numRootFacts = 0
factsCheckVersion = None
def factCheckFact(fact):
modelDocument._factsCheckMd5s += fact.md5sum
for _tupleFact in fact.modelTupleFacts:
factCheckFact(_tupleFact)
for event, mdlObj in streamingParserContext:
if event == "start":
if mdlObj.tag == "{http://www.xbrl.org/2003/instance}xbrl":
modelDocument = ModelDocument(modelXbrl, Type.INSTANCE,
mappedUri, filepath,
mdlObj.getroottree())
modelXbrl.modelDocument = modelDocument # needed for incremental validation
mdlObj.init(modelDocument)
modelDocument.parser = _parser # needed for XmlUtil addChild's makeelement
modelDocument.parserLookupName = _parserLookupName
modelDocument.parserLookupClass = _parserLookupClass
modelDocument.xmlRootElement = mdlObj
modelDocument.schemaLocationElements.add(mdlObj)
modelDocument.documentEncoding = _encoding
modelDocument._creationSoftwareComment = precedingComment(
mdlObj)
modelDocument._factsCheckMd5s = Md5Sum()
modelXbrl.info("streamingExtensions:streaming",
_("Stream processing this instance."),
modelObject=modelDocument)
elif mdlObj.getparent() is not None:
mdlObj._init() # requires discovery as part of start elements
if mdlObj.getparent(
).tag == "{http://www.xbrl.org/2003/instance}xbrl":
if not foundInstance:
foundInstance = True
pi = precedingProcessingInstruction(
mdlObj, "xbrl-facts-check")
if pi is not None:
factsCheckVersion = pi.attrib.get("version", None)
elif not foundInstance:
break
ns = mdlObj.qname.namespaceURI
ln = mdlObj.qname.localName
if beforeInstanceStream:
if ((ns == XbrlConst.link
and ln not in ("schemaRef", "linkbaseRef")) or
(ns == XbrlConst.xbrli and ln in ("context", "unit"))
or (ns not in (XbrlConst.link, XbrlConst.xbrli))):
beforeInstanceStream = False
if _streamingExtensionsValidate:
instValidator.validate(
modelXbrl,
modelXbrl.modelManager.formulaOptions.
typedParameters(modelXbrl.prefixedNamespaces))
else: # need default dimensions
ValidateXbrlDimensions.loadDimensionDefaults(
modelXbrl)
elif not beforeInstanceStream and beforeStartStreamingPlugin:
for pluginMethod in pluginClassMethods("Streaming.Start"):
pluginMethod(modelXbrl)
beforeStartStreamingPlugin = False
elif event == "end":
parentMdlObj = mdlObj.getparent()
ns = mdlObj.namespaceURI
ln = mdlObj.localName
if ns == XbrlConst.xbrli:
if ln == "context":
if mdlObj.get("sticky"):
del mdlObj.attrib["sticky"]
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
else:
if len(contextBuffer) >= contextBufferLimit:
# drop before adding as dropped may have same id as added
cntx = contextBuffer.pop(0)
if _streamingFactsPlugin or _streamingValidateFactsPlugin:
contextsToDrop.append(cntx)
else:
dropContext(modelXbrl, cntx)
#>>del parentMdlObj[parentMdlObj.index(cntx)]
cntx = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.contextDiscover(mdlObj)
if contextBufferLimit.is_finite():
contextBuffer.append(mdlObj)
if _streamingExtensionsValidate:
contextsToCheck = (mdlObj, )
instValidator.checkContexts(contextsToCheck)
if modelXbrl.hasXDT:
instValidator.checkContextsDimensions(
contextsToCheck)
del contextsToCheck # dereference
elif ln == "unit":
if len(unitBuffer) >= unitBufferLimit:
# drop before additing as dropped may have same id as added
unit = unitBuffer.pop(0)
if _streamingFactsPlugin or _streamingValidateFactsPlugin:
unitsToDrop.append(unit)
else:
dropUnit(modelXbrl, unit)
#>>del parentMdlObj[parentMdlObj.index(unit)]
unit = None
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.unitDiscover(mdlObj)
if unitBufferLimit.is_finite():
unitBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkUnits((mdlObj, ))
elif ln == "xbrl": # end of document
# check remaining batched facts if any
if _streamingFactsPlugin or _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
# finish any final batch of facts
if len(modelXbrl.facts) > 0:
factsToCheck = modelXbrl.facts.copy()
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
if _streamingValidateFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.ValidateFacts"):
pluginMethod(instValidator, factsToCheck)
if _streamingFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.Facts"):
pluginMethod(modelXbrl, factsToCheck)
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
#>>del parentMdlObj[parentMdlObj.index(fact)]
for cntx in contextsToDrop:
dropContext(modelXbrl, cntx)
#>>del parentMdlObj[parentMdlObj.index(cntx)]
for unit in unitsToDrop:
dropUnit(modelXbrl, unit)
#>>del parentMdlObj[parentMdlObj.index(unit)]
for footnoteLink in footnoteLinksToDrop:
dropFootnoteLink(modelXbrl, footnoteLink)
#>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
fact = cntx = unit = footnoteLink = None
del contextsToDrop[:]
del unitsToDrop[:]
del footnoteLinksToDrop[:]
del factsToCheck
# check remaining footnote refs
for footnoteLink in footnoteBuffer:
checkFootnoteHrefs(modelXbrl, footnoteLink)
pi = childProcessingInstruction(mdlObj,
"xbrl-facts-check",
reversed=True)
if pi is not None: # attrib is in .text, not attrib, no idea why!!!
_match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']",
pi.text)
if _match:
_matchGroups = _match.groups()
if len(_matchGroups) == 2:
if _matchGroups[0] == "sum-of-fact-md5s":
try:
expectedMd5 = Md5Sum(_matchGroups[1])
if modelDocument._factsCheckMd5s != expectedMd5:
modelXbrl.warning(
"streamingExtensions:xbrlFactsCheckWarning",
_("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"
),
modelObject=modelXbrl,
expectedMd5=expectedMd5,
actualMd5Sum=modelDocument.
_factsCheckMd5s)
else:
modelXbrl.info(
"info",
_("Successful XBRL facts sum of md5s."
),
modelObject=modelXbrl)
except ValueError:
modelXbrl.error(
"streamingExtensions:xbrlFactsCheckError",
_("Invalid sum-of-md5s %(sumOfMd5)s"
),
modelObject=modelXbrl,
sumOfMd5=_matchGroups[1])
if _streamingValidateFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.ValidateFinish"):
pluginMethod(instValidator)
if _streamingFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.Finish"):
pluginMethod(modelXbrl)
elif ns == XbrlConst.link:
if ln in ("schemaRef", "linkbaseRef"):
modelDocument.discoverHref(
mdlObj,
urlRewritePluginClass=
"ModelDocument.InstanceSchemaRefRewriter")
elif ln in ("roleRef", "arcroleRef"):
modelDocument.linkbaseDiscover((mdlObj, ), inInstance=True)
elif ln == "footnoteLink":
XmlValidate.validate(modelXbrl, mdlObj)
footnoteLinks = (mdlObj, )
modelDocument.linkbaseDiscover(footnoteLinks,
inInstance=True)
if footnoteBufferLimit.is_finite():
footnoteBuffer.append(mdlObj)
if _streamingExtensionsValidate:
instValidator.checkLinks(footnoteLinks)
if len(footnoteBuffer) > footnoteBufferLimit:
# check that hrefObjects for locators were all satisfied
# drop before addition as dropped may have same id as added
footnoteLink = footnoteBuffer.pop(0)
checkFootnoteHrefs(modelXbrl, footnoteLink)
if _streamingValidateFactsPlugin:
footnoteLinksToDrop.append(footnoteLink)
else:
dropFootnoteLink(modelXbrl, footnoteLink)
#>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
footnoteLink = None
footnoteLinks = None
elif parentMdlObj.qname == XbrlConst.qnXbrliXbrl and isinstance(
mdlObj, ModelFact):
numRootFacts += 1
XmlValidate.validate(modelXbrl, mdlObj)
modelDocument.factDiscover(mdlObj, modelXbrl.facts)
if factsCheckVersion:
factCheckFact(mdlObj)
if _streamingExtensionsValidate or _streamingFactsPlugin or _streamingValidateFactsPlugin:
factsToCheck = (mdlObj,
) # validate current fact by itself
if _streamingExtensionsValidate:
instValidator.checkFacts(factsToCheck)
if modelXbrl.hasXDT:
instValidator.checkFactsDimensions(factsToCheck)
if _streamingFactsPlugin or _streamingValidateFactsPlugin:
# plugin attempts to process batch of all root facts not yet processed (not just current one)
# use batches of 1000 facts
if len(modelXbrl.facts) > 1000:
factsToCheck = modelXbrl.facts.copy()
# can block facts deletion if required data not yet available, such as numeric unit for DpmDB
if _streamingValidateFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.ValidateFacts"):
pluginMethod(instValidator, factsToCheck)
if _streamingFactsPlugin:
for pluginMethod in pluginClassMethods(
"Streaming.Facts"):
pluginMethod(modelXbrl, factsToCheck)
for fact in factsToCheck:
dropFact(modelXbrl, fact, modelXbrl.facts)
#>>del parentMdlObj[parentMdlObj.index(fact)]
for cntx in contextsToDrop:
dropContext(modelXbrl, cntx)
#>>del parentMdlObj[parentMdlObj.index(cntx)]
for unit in unitsToDrop:
dropUnit(modelXbrl, unit)
#>>del parentMdlObj[parentMdlObj.index(unit)]
for footnoteLink in footnoteLinksToDrop:
dropFootnoteLink(modelXbrl, footnoteLink)
#>>del parentMdlObj[parentMdlObj.index(footnoteLink)]
fact = cntx = unit = footnoteLink = None
del contextsToDrop[:]
del unitsToDrop[:]
del footnoteLinksToDrop[:]
del factsToCheck # dereference fact or batch of facts
else:
dropFact(
modelXbrl, mdlObj,
modelXbrl.facts) # single fact has been processed
#>>del parentMdlObj[parentMdlObj.index(mdlObj)]
if numRootFacts % 1000 == 0:
pass
#modelXbrl.profileActivity("... streaming fact {0} of {1} {2:.2f}%".format(self.numRootFacts, instInfoNumRootFacts,
# 100.0 * self.numRootFacts / instInfoNumRootFacts),
# minTimeToShow=20.0)
#gc.collect()
#sys.stdout.write ("\rAt fact {} of {} mem {}".format(numRootFacts, instInfoNumRootFacts, modelXbrl.modelManager.cntlr.memoryUsed))
if mdlObj is not None:
mdlObj.clear()
del _parser, _parserLookupName, _parserLookupClass
if _streamingExtensionsValidate and validator is not None:
_file.close()
del instValidator
validator.close()
# track that modelXbrl has been validated by this streaming extension
modelXbrl._streamingExtensionValidated = True
modelXbrl.profileStat(_("streaming complete"), time.time() - startedAt)
return modelXbrl.modelDocument
def urls(self):
_urls = [os.path.join(self.modelManager.cntlr.configDir, "disclosuresystems.xml")]
# get custom config xml file url, insert before main url in reverse order
for pluginXbrlMethod in pluginClassMethods("DisclosureSystem.ConfigURL"):
_urls.insert(0, pluginXbrlMethod(self))
return _urls
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY):
for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri):
self.validateTestcase(doc) # testcases doc's are sorted by their uri (file names), e.g., for formula
elif hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcaseVariationsByTarget(testcase.testcaseVariations):
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s%(name)s%(target)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=(" {}".format(modelTestcaseVariation.name) if modelTestcaseVariation.name else ""),
target=(" target {}".format(modelTestcaseVariation.ixdsTarget) if modelTestcaseVariation.ixdsTarget else ""),
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings:
errorCaptureLevel = logging._checkLevel("WARNING")
else:
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
elif resultIsTaxonomyPackage:
from arelle import PackageManager, PrototypeInstanceObject
dtsName = readMeFirstUri
modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel,
ixdsTarget=modelTestcaseVariation.ixdsTarget)
else: # need own file source, may need instance discovery
filesource = FileSource.openFileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr, base=baseForElement)
if filesource and not filesource.selection and filesource.isArchive:
try:
if filesource.isTaxonomyPackage:
_rptPkgIxdsOptions = {}
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxdsOptions"):
pluginXbrlMethod(self, _rptPkgIxdsOptions)
filesource.loadTaxonomyPackageMappings()
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxds"):
filesource.select(pluginXbrlMethod(filesource, **_rptPkgIxdsOptions))
else:
from arelle.CntlrCmdLine import filesourceEntrypointFiles
entrypoints = filesourceEntrypointFiles(filesource)
if entrypoints:
# resolve an IXDS in entrypoints
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ArchiveIxds"):
pluginXbrlMethod(self, filesource,entrypoints)
filesource.select(entrypoints[0].get("file", None) )
except Exception as err:
self.modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, entry URL: %(instance)s"),
modelXbrl=self.modelXbrl, instance=readMeFirstUri, error=err)
continue # don't try to load this entry URL
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel,
ixdsTarget=modelTestcaseVariation.ixdsTarget)
modelXbrl.isTestcaseVariation = True
if modelXbrl.modelDocument is None:
modelXbrl.info("arelle:notLoaded",
_("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif resultIsVersioningReport or resultIsTaxonomyPackage:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError))
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif resultIsTaxonomyPackage:
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement)
for d in modelTestcaseVariation.dataUris.values() for uri in d
if not UrlUtil.isAbsolute(uri))
foundDataFiles = set()
variationBase = os.path.dirname(baseForElement)
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
for _inputDTS in inputDTS:
for docUrl, doc in _inputDTS.urlDocs.items():
if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET:
if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file
if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles:
docUrl = docUrl.replace("-formula.xml", ".xf")
foundDataFiles.add(docUrl)
if expectedDataFiles - foundDataFiles:
modelXbrl.info("arelle:testcaseDataNotUsed",
_("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"),
modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles)))
if foundDataFiles - expectedDataFiles:
modelXbrl.info("arelle:testcaseDataUnexpected",
_("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"),
modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles)))
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
try:
RenderingEvaluator.init(modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrlHasFormulae = modelXbrl.hasFormulae
if modelXbrlHasFormulae and self.modelXbrl.modelManager.formulaOptions.formulaAction != "none":
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError))
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
try:
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and
modelTestcaseVariation.resultXbrlInstanceUri is not None)
if compareIxResultInstance:
formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
errMsgPrefix = "ix"
else: # delete input instances before formula output comparision
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
errMsgPrefix = "formula"
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
_matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
_("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.ExpectedInstance.Loaded"):
pluginXbrlMethod(expectedInstance, formulaOutputInstance)
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts),
messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
else:
formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
def factFootnotes(fact, footnotesRelSet):
footnotes = {}
footnoteRels = footnotesRelSet.fromModelObject(fact)
if footnoteRels:
# most process rels in same order between two instances, use labels to sort
for i, footnoteRel in enumerate(sorted(footnoteRels,
key=lambda r: (r.fromLabel,r.toLabel))):
modelObject = footnoteRel.toModelObject
if isinstance(modelObject, ModelResource):
xml = collapseWhitespace(modelObject.viewText().strip())
footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))
elif isinstance(modelObject, ModelFact):
footnotes["Footnoted fact {}".format(i+1)] = \
"{} context: {} value: {}".format(
modelObject.qname,
modelObject.contextID,
collapseWhitespace(modelObject.value))
return footnotes
for expectedInstanceFact in expectedInstance.facts:
unmatchedFactsStack = []
formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False)
#formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True)
if formulaOutputFact is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = expectedInstanceFact
# is it possible to show value mismatches?
expectedFacts = formulaOutputInstance.factsByQname.get(missingFact.qname)
if len(expectedFacts) == 1:
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s, extracted value \"%(value1)s\", expected value \"%(value2)s\""),
modelXbrl=missingFact, fact=missingFact.qname, value1=missingFact.xValue, value2=next(iter(expectedFacts)).xValue,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else:
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else: # compare footnotes
expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or
set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())):
formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
_("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()),
messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
if compareIxResultInstance:
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
_statusCounts = OrderedDict((("pass",0),("fail",0)))
for tv in getattr(testcase, "testcaseVariations", ()):
_statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1
self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k))
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def expectedCount(self):
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ExpectedCount"):
_count = pluginXbrlMethod(self)
if _count is not None: # ignore plug in if not a plug-in-recognized test case
return _count
return None
def viewConcept(self, concept, modelObject, labelPrefix, preferredLabel,
parentnode, n, relationshipSet, visited):
if concept is None:
return
try:
isRelation = isinstance(modelObject,
ModelDtsObject.ModelRelationship)
isModelTable = False
filingIndicatorCode = ""
if isinstance(concept, ModelDtsObject.ModelConcept):
text = labelPrefix + concept.label(
preferredLabel,
lang=self.lang,
linkroleHint=relationshipSet.linkrole)
if (self.arcrole
in ("XBRL-dimensions", XbrlConst.hypercubeDimension)
and concept.isTypedDimension
and concept.typedDomainElement is not None):
text += " (typedDomain={0})".format(
concept.typedDomainElement.qname)
elif isinstance(concept, ModelInstanceObject.ModelFact):
if concept.concept is not None:
text = labelPrefix + concept.concept.label(
preferredLabel,
lang=self.lang,
linkroleHint=relationshipSet.linkrole)
else:
text = str(concept.qname)
if concept.contextID:
text += " [" + concept.contextID + "] = " + concept.effectiveValue
elif self.arcrole == "Table-rendering":
text = concept.localName
elif isinstance(concept, ModelRenderingObject.ModelTable):
text = (concept.genLabel(lang=self.lang, strip=True)
or concept.localName)
isModelTable = True
elif isinstance(concept, ModelDtsObject.ModelResource):
if self.showReferences:
text = (concept.viewText() or concept.localName)
else:
text = (Locale.rtlString(concept.textValue.strip(),
lang=concept.xmlLang)
or concept.localName)
else: # just a resource
text = concept.localName
childnode = self.treeView.insert(parentnode,
"end",
modelObject.objectId(self.id),
text=text,
tags=("odd" if n
& 1 else "even", ))
# Check if we need special rendering of this item
for pluginXbrlMethod in pluginClassMethods(
"CntlrWinMain.Rendering.RenderConcept"):
stopPlugin = pluginXbrlMethod(isModelTable, concept, text,
self, self.modelXbrl, childnode)
if stopPlugin == True:
break
childRelationshipSet = relationshipSet
if self.arcrole == XbrlConst.parentChild: # extra columns
if isRelation:
preferredLabel = modelObject.preferredLabel
if preferredLabel and preferredLabel.startswith(
"http://www.xbrl.org/2003/role/"):
preferredLabel = os.path.basename(preferredLabel)
self.treeView.set(childnode, "preferredLabel",
preferredLabel)
self.treeView.set(childnode, "type", concept.niceType)
self.treeView.set(childnode, "references",
viewReferences(concept))
elif self.arcrole == XbrlConst.summationItem:
if isRelation:
self.treeView.set(childnode, "weight",
"{:+0g} ".format(modelObject.weight))
self.treeView.set(childnode, "balance", concept.balance)
elif self.arcrole == "XBRL-dimensions" and isRelation: # extra columns
relArcrole = modelObject.arcrole
self.treeView.set(childnode, "arcrole",
os.path.basename(relArcrole))
if relArcrole in (XbrlConst.all, XbrlConst.notAll):
self.treeView.set(childnode, "contextElement",
modelObject.contextElement)
self.treeView.set(childnode, "closed", modelObject.closed)
elif relArcrole in (XbrlConst.dimensionDomain,
XbrlConst.domainMember):
self.treeView.set(childnode, "usable", modelObject.usable)
childRelationshipSet = self.modelXbrl.relationshipSet(
XbrlConst.consecutiveArcrole.get(relArcrole,
"XBRL-dimensions"),
modelObject.consecutiveLinkrole)
elif self.arcrole == "Table-rendering": # extra columns
try:
header = concept.header(lang=self.lang,
strip=True,
evaluate=False)
except AttributeError:
header = None # could be a filter
if isRelation and header is None:
header = "{0} {1}".format(
os.path.basename(modelObject.arcrole),
concept.xlinkLabel)
self.treeView.set(childnode, "header", header)
if concept.get("abstract") == "true":
self.treeView.set(childnode, "abstract",
'\u2713') # checkmark unicode character
if concept.get("merge") == "true":
self.treeView.set(childnode, "merge",
'\u2713') # checkmark unicode character
if isRelation:
self.treeView.set(childnode, "axis",
modelObject.axisDisposition)
if isinstance(concept,
(ModelEuAxisCoord, ModelRuleDefinitionNode)):
self.treeView.set(
childnode, "priItem",
concept.aspectValue(None, Aspect.CONCEPT))
self.treeView.set(
childnode, "dims", ' '.join(
("{0},{1}".format(
dim, concept.aspectValue(None, dim))
for dim in (concept.aspectValue(
None, Aspect.DIMENSIONS, inherit=False)
or []))))
elif self.isResourceArcrole: # resource columns
if isRelation:
self.treeView.set(childnode, "arcrole",
os.path.basename(modelObject.arcrole))
if isinstance(concept, ModelDtsObject.ModelResource):
self.treeView.set(childnode, "resource", concept.localName)
self.treeView.set(childnode, "resourcerole",
os.path.basename(concept.role or ''))
self.treeView.set(childnode, "lang", concept.xmlLang)
self.id += 1
self.tag_has[modelObject.objectId()].append(childnode)
if isRelation:
self.tag_has[modelObject.toModelObject.objectId()].append(
childnode)
if concept not in visited:
visited.add(concept)
for modelRel in childRelationshipSet.fromModelObject(concept):
nestedRelationshipSet = childRelationshipSet
targetRole = modelRel.targetRole
if self.arcrole == XbrlConst.summationItem:
childPrefix = "({:0g}) ".format(
modelRel.weight
) # format without .0 on integer weights
elif targetRole is None or len(targetRole) == 0:
targetRole = relationshipSet.linkrole
childPrefix = ""
else:
nestedRelationshipSet = self.modelXbrl.relationshipSet(
childRelationshipSet.arcrole, targetRole)
childPrefix = "(via targetRole) "
toConcept = modelRel.toModelObject
if toConcept in visited:
childPrefix += "(loop)"
labelrole = modelRel.preferredLabel
if not labelrole or self.labelrole == conceptNameLabelRole:
labelrole = self.labelrole
n += 1 # child has opposite row style of parent
self.viewConcept(toConcept, modelRel, childPrefix,
labelrole, childnode, n,
nestedRelationshipSet, visited)
visited.remove(concept)
except AttributeError:
return # bad object, don't try to display
def parseAndRun(args):
"""interface used by Main program and py.test (arelle_test.py)
"""
try:
from arelle import webserver
hasWebServer = True
except ImportError:
hasWebServer = False
cntlr = CntlrCmdLine() # need controller for plug ins to be loaded
usage = "usage: %prog [options]"
parser = OptionParser(usage, version="Arelle(r) {0}".format(Version.version))
parser.add_option("-f", "--file", dest="entrypointFile",
help=_("FILENAME is an entry point, which may be "
"an XBRL instance, schema, linkbase file, "
"inline XBRL instance, testcase file, "
"testcase index file. FILENAME may be "
"a local file or a URI to a web located file."))
parser.add_option("--username", dest="username",
help=_("user name if needed (with password) for web file retrieval"))
parser.add_option("--password", dest="password",
help=_("password if needed (with user name) for web retrieval"))
# special option for web interfaces to suppress closing an opened modelXbrl
parser.add_option("--keepOpen", dest="keepOpen", action="store_true", help=SUPPRESS_HELP)
parser.add_option("-i", "--import", dest="importFiles",
help=_("FILENAME is a list of files to import to the DTS, such as "
"additional formula or label linkbases. "
"Multiple file names are separated by a '|' character. "))
parser.add_option("-d", "--diff", dest="diffFile",
help=_("FILENAME is a second entry point when "
"comparing (diffing) two DTSes producing a versioning report."))
parser.add_option("-r", "--report", dest="versReportFile",
help=_("FILENAME is the filename to save as the versioning report."))
parser.add_option("-v", "--validate",
action="store_true", dest="validate",
help=_("Validate the file according to the entry "
"file type. If an XBRL file, it is validated "
"according to XBRL validation 2.1, calculation linkbase validation "
"if either --calcDecimals or --calcPrecision are specified, and "
"SEC Edgar Filing Manual (if --efm selected) or Global Filer Manual "
"disclosure system validation (if --gfm=XXX selected). "
"If a test suite or testcase, the test case variations "
"are individually so validated. "
"If formulae are present they will be validated and run unless --formula=none is specified. "
))
parser.add_option("--calcDecimals", action="store_true", dest="calcDecimals",
help=_("Specify calculation linkbase validation inferring decimals."))
parser.add_option("--calcdecimals", action="store_true", dest="calcDecimals", help=SUPPRESS_HELP)
parser.add_option("--calcPrecision", action="store_true", dest="calcPrecision",
help=_("Specify calculation linkbase validation inferring precision."))
parser.add_option("--calcprecision", action="store_true", dest="calcPrecision", help=SUPPRESS_HELP)
parser.add_option("--efm", action="store_true", dest="validateEFM",
help=_("Select Edgar Filer Manual (U.S. SEC) disclosure system validation (strict)."))
parser.add_option("--gfm", action="store", dest="disclosureSystemName", help=SUPPRESS_HELP)
parser.add_option("--disclosureSystem", action="store", dest="disclosureSystemName",
help=_("Specify a disclosure system name and"
" select disclosure system validation. "
"Enter --disclosureSystem=help for list of names or help-verbose for list of names and descriptions. "))
parser.add_option("--disclosuresystem", action="store", dest="disclosureSystemName", help=SUPPRESS_HELP)
parser.add_option("--hmrc", action="store_true", dest="validateHMRC",
help=_("Select U.K. HMRC disclosure system validation."))
parser.add_option("--utr", action="store_true", dest="utrValidate",
help=_("Select validation with respect to Unit Type Registry."))
parser.add_option("--utrUrl", action="store", dest="utrUrl",
help=_("Override disclosure systems Unit Type Registry location (URL or file path)."))
parser.add_option("--utrurl", action="store", dest="utrUrl", help=SUPPRESS_HELP)
parser.add_option("--infoset", action="store_true", dest="infosetValidate",
help=_("Select validation with respect testcase infosets."))
parser.add_option("--labelLang", action="store", dest="labelLang",
help=_("Language for labels in following file options (override system settings)"))
parser.add_option("--labellang", action="store", dest="labelLang", help=SUPPRESS_HELP)
parser.add_option("--labelRole", action="store", dest="labelRole",
help=_("Label role for labels in following file options (instead of standard label)"))
parser.add_option("--labelrole", action="store", dest="labelRole", help=SUPPRESS_HELP)
parser.add_option("--DTS", "--csvDTS", action="store", dest="DTSFile",
help=_("Write DTS tree into FILE (may be .csv or .html)"))
parser.add_option("--facts", "--csvFacts", action="store", dest="factsFile",
help=_("Write fact list into FILE"))
parser.add_option("--factListCols", action="store", dest="factListCols",
help=_("Columns for fact list file"))
parser.add_option("--factTable", "--csvFactTable", action="store", dest="factTableFile",
help=_("Write fact table into FILE"))
parser.add_option("--concepts", "--csvConcepts", action="store", dest="conceptsFile",
help=_("Write concepts into FILE"))
parser.add_option("--pre", "--csvPre", action="store", dest="preFile",
help=_("Write presentation linkbase into FILE"))
parser.add_option("--cal", "--csvCal", action="store", dest="calFile",
help=_("Write calculation linkbase into FILE"))
parser.add_option("--dim", "--csvDim", action="store", dest="dimFile",
help=_("Write dimensions (of definition) linkbase into FILE"))
parser.add_option("--formulae", "--htmlFormulae", action="store", dest="formulaeFile",
help=_("Write formulae linkbase into FILE"))
parser.add_option("--viewArcrole", action="store", dest="viewArcrole",
help=_("Write linkbase relationships for viewArcrole into viewFile"))
parser.add_option("--viewarcrole", action="store", dest="viewArcrole", help=SUPPRESS_HELP)
parser.add_option("--viewFile", action="store", dest="viewFile",
help=_("Write linkbase relationships for viewArcrole into viewFile"))
parser.add_option("--viewfile", action="store", dest="viewFile", help=SUPPRESS_HELP)
parser.add_option("--testReport", "--csvTestReport", action="store", dest="testReport",
help=_("Write test report of validation (of test cases) into FILE"))
parser.add_option("--testreport", "--csvtestreport", action="store", dest="testReport", help=SUPPRESS_HELP)
parser.add_option("--testReportCols", action="store", dest="testReportCols",
help=_("Columns for test report file"))
parser.add_option("--testreportcols", action="store", dest="testReportCols", help=SUPPRESS_HELP)
parser.add_option("--rssReport", action="store", dest="rssReport",
help=_("Write RSS report into FILE"))
parser.add_option("--rssreport", action="store", dest="rssReport", help=SUPPRESS_HELP)
parser.add_option("--rssReportCols", action="store", dest="rssReportCols",
help=_("Columns for RSS report file"))
parser.add_option("--rssreportcols", action="store", dest="rssReportCols", help=SUPPRESS_HELP)
parser.add_option("--logFile", action="store", dest="logFile",
help=_("Write log messages into file, otherwise they go to standard output. "
"If file ends in .xml it is xml-formatted, otherwise it is text. "))
parser.add_option("--logfile", action="store", dest="logFile", help=SUPPRESS_HELP)
parser.add_option("--logFormat", action="store", dest="logFormat",
help=_("Logging format for messages capture, otherwise default is \"[%(messageCode)s] %(message)s - %(file)s\"."))
parser.add_option("--logformat", action="store", dest="logFormat", help=SUPPRESS_HELP)
parser.add_option("--logLevel", action="store", dest="logLevel",
help=_("Minimum level for messages capture, otherwise the message is ignored. "
"Current order of levels are debug, info, info-semantic, warning, warning-semantic, warning, assertion-satisfied, inconsistency, error-semantic, assertion-not-satisfied, and error. "))
parser.add_option("--loglevel", action="store", dest="logLevel", help=SUPPRESS_HELP)
parser.add_option("--logLevelFilter", action="store", dest="logLevelFilter",
help=_("Regular expression filter for logLevel. "
"(E.g., to not match *-semantic levels, logLevelFilter=(?!^.*-semantic$)(.+). "))
parser.add_option("--loglevelfilter", action="store", dest="logLevelFilter", help=SUPPRESS_HELP)
parser.add_option("--logCodeFilter", action="store", dest="logCodeFilter",
help=_("Regular expression filter for log message code."))
parser.add_option("--logcodefilter", action="store", dest="logCodeFilter", help=SUPPRESS_HELP)
parser.add_option("--showOptions", action="store_true", dest="showOptions", help=SUPPRESS_HELP)
parser.add_option("--parameters", action="store", dest="parameters", help=_("Specify parameters for formula and validation (name=value[,name=value])."))
parser.add_option("--parameterSeparator", action="store", dest="parameterSeparator", help=_("Specify parameters separator string (if other than comma)."))
parser.add_option("--parameterseparator", action="store", dest="parameterSeparator", help=SUPPRESS_HELP)
parser.add_option("--formula", choices=("validate", "run", "none"), dest="formulaAction",
help=_("Specify formula action: "
"validate - validate only, without running, "
"run - validate and run, or "
"none - prevent formula validation or running when also specifying -v or --validate. "
"if this option is not specified, -v or --validate will validate and run formulas if present"))
parser.add_option("--formulaParamExprResult", action="store_true", dest="formulaParamExprResult", help=_("Specify formula tracing."))
parser.add_option("--formulaParamInputValue", action="store_true", dest="formulaParamInputValue", help=_("Specify formula tracing."))
parser.add_option("--formulaCallExprSource", action="store_true", dest="formulaCallExprSource", help=_("Specify formula tracing."))
parser.add_option("--formulaCallExprCode", action="store_true", dest="formulaCallExprCode", help=_("Specify formula tracing."))
parser.add_option("--formulaCallExprEval", action="store_true", dest="formulaCallExprEval", help=_("Specify formula tracing."))
parser.add_option("--formulaCallExprResult", action="store_true", dest="formulaCallExprResult", help=_("Specify formula tracing."))
parser.add_option("--formulaVarSetExprEval", action="store_true", dest="formulaVarSetExprEval", help=_("Specify formula tracing."))
parser.add_option("--formulaVarSetExprResult", action="store_true", dest="formulaVarSetExprResult", help=_("Specify formula tracing."))
parser.add_option("--formulaVarSetTiming", action="store_true", dest="timeVariableSetEvaluation", help=_("Specify showing times of variable set evaluation."))
parser.add_option("--formulaAsserResultCounts", action="store_true", dest="formulaAsserResultCounts", help=_("Specify formula tracing."))
parser.add_option("--formulaFormulaRules", action="store_true", dest="formulaFormulaRules", help=_("Specify formula tracing."))
parser.add_option("--formulaVarsOrder", action="store_true", dest="formulaVarsOrder", help=_("Specify formula tracing."))
parser.add_option("--formulaVarExpressionSource", action="store_true", dest="formulaVarExpressionSource", help=_("Specify formula tracing."))
parser.add_option("--formulaVarExpressionCode", action="store_true", dest="formulaVarExpressionCode", help=_("Specify formula tracing."))
parser.add_option("--formulaVarExpressionEvaluation", action="store_true", dest="formulaVarExpressionEvaluation", help=_("Specify formula tracing."))
parser.add_option("--formulaVarExpressionResult", action="store_true", dest="formulaVarExpressionResult", help=_("Specify formula tracing."))
parser.add_option("--formulaVarFilterWinnowing", action="store_true", dest="formulaVarFilterWinnowing", help=_("Specify formula tracing."))
parser.add_option("--formulaVarFiltersResult", action="store_true", dest="formulaVarFiltersResult", help=_("Specify formula tracing."))
parser.add_option("--uiLang", action="store", dest="uiLang",
help=_("Language for user interface (override system settings, such as program messages). Does not save setting."))
parser.add_option("--uilang", action="store", dest="uiLang", help=SUPPRESS_HELP)
parser.add_option("--proxy", action="store", dest="proxy",
help=_("Modify and re-save proxy settings configuration. "
"Enter 'system' to use system proxy setting, 'none' to use no proxy, "
"'http://[user[:password]@]host[:port]' "
" (e.g., http://192.168.1.253, http://example.com:8080, http://joe:[email protected]:8080), "
" or 'show' to show current setting, ." ))
parser.add_option("--internetConnectivity", choices=("online", "offline"), dest="internetConnectivity",
help=_("Specify internet connectivity: online or offline"))
parser.add_option("--internetconnectivity", action="store", dest="internetConnectivity", help=SUPPRESS_HELP)
parser.add_option("--internetTimeout", type="int", dest="internetTimeout",
help=_("Specify internet connection timeout in seconds (0 means unlimited)."))
parser.add_option("--internettimeout", type="int", action="store", dest="internetTimeout", help=SUPPRESS_HELP)
parser.add_option("--xdgConfigHome", action="store", dest="xdgConfigHome",
help=_("Specify non-standard location for configuration and cache files (overrides environment parameter XDG_CONFIG_HOME)."))
parser.add_option("--plugins", action="store", dest="plugins",
help=_("Modify plug-in configuration. "
"Re-save unless 'temp' is in the module list. "
"Enter 'show' to show current plug-in configuration. "
"Commands show, and module urls are '|' separated: "
"+url to add plug-in by its url or filename, ~name to reload a plug-in by its name, -name to remove a plug-in by its name, "
"relative URLs are relative to installation plug-in directory, "
" (e.g., '+http://arelle.org/files/hello_web.py', '+C:\Program Files\Arelle\examples\plugin\hello_dolly.py' to load, "
"or +../examples/plugin/hello_dolly.py for relative use of examples directory, "
"~Hello Dolly to reload, -Hello Dolly to remove). "
"If + is omitted from .py file nothing is saved (same as temp). "
"Packaged plug-in urls are their directory's url. " ))
parser.add_option("--abortOnMajorError", action="store_true", dest="abortOnMajorError", help=_("Abort process on major error, such as when load is unable to find an entry or discovered file."))
parser.add_option("--collectProfileStats", action="store_true", dest="collectProfileStats", help=_("Collect profile statistics, such as timing of validation activities and formulae."))
if hasWebServer:
parser.add_option("--webserver", action="store", dest="webserver",
help=_("start web server on host:port[:server] for REST and web access, e.g., --webserver locahost:8080, "
"or specify nondefault a server name, such as cherrypy, --webserver locahost:8080:cherrypy. "
"(It is possible to specify options to be defaults for the web server, such as disclosureSystem and validations, but not including file names.) "))
pluginOptionsIndex = len(parser.option_list)
for optionsExtender in pluginClassMethods("CntlrCmdLine.Options"):
optionsExtender(parser)
pluginLastOptionIndex = len(parser.option_list)
parser.add_option("-a", "--about",
action="store_true", dest="about",
help=_("Show product version, copyright, and license."))
if args is None and cntlr.isGAE:
args = ["--webserver=::gae"]
elif cntlr.isMSW:
# if called from java on Windows any empty-string arguments are lost, see:
# http://bugs.sun.com/view_bug.do?bug_id=6518827
# insert needed arguments
args = []
namedOptions = set()
optionsWithArg = set()
for option in parser.option_list:
names = str(option).split('/')
namedOptions.update(names)
if option.action == "store":
optionsWithArg.update(names)
priorArg = None
for arg in sys.argv[1:]:
if priorArg in optionsWithArg and arg in namedOptions:
# probable java/MSFT interface bug 6518827
args.append('') # add empty string argument
args.append(arg)
priorArg = arg
(options, leftoverArgs) = parser.parse_args(args)
if options.about:
print(_("\narelle(r) {0}\n\n"
"An open source XBRL platform\n"
"(c) 2010-2013 Mark V Systems Limited\n"
"All rights reserved\nhttp://www.arelle.org\[email protected]\n\n"
"Licensed under the Apache License, Version 2.0 (the \"License\"); "
"you may not \nuse this file except in compliance with the License. "
"You may obtain a copy \nof the License at "
"'http://www.apache.org/licenses/LICENSE-2.0'\n\n"
"Unless required by applicable law or agreed to in writing, software \n"
"distributed under the License is distributed on an \"AS IS\" BASIS, \n"
"WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. \n"
"See the License for the specific language governing permissions and \n"
"limitations under the License."
"\n\nIncludes:"
"\n Python(r) (c) 2001-2013 Python Software Foundation"
"\n PyParsing (c) 2003-2013 Paul T. McGuire"
"\n lxml (c) 2004 Infrae, ElementTree (c) 1999-2004 by Fredrik Lundh"
"\n xlrd (c) 2005-2009 Stephen J. Machin, Lingfo Pty Ltd, (c) 2001 D. Giffin, (c) 2000 A. Khan"
"\n xlwt (c) 2007 Stephen J. Machin, Lingfo Pty Ltd, (c) 2005 R. V. Kiseliov"
"{1}"
).format(Version.version,
_("\n Bottle (c) 2011-2013 Marcel Hellkamp") if hasWebServer else ""))
elif options.disclosureSystemName in ("help", "help-verbose"):
text = _("Disclosure system choices: \n{0}").format(' \n'.join(cntlr.modelManager.disclosureSystem.dirlist(options.disclosureSystemName)))
try:
print(text)
except UnicodeEncodeError:
print(text.encode("ascii", "replace").decode("ascii"))
elif len(leftoverArgs) != 0 or (options.entrypointFile is None and
((not options.proxy) and (not options.plugins) and
(not any(pluginOption for pluginOption in parser.option_list[pluginOptionsIndex:pluginLastOptionIndex])) and
(not hasWebServer or options.webserver is None))):
parser.error(_("incorrect arguments, please try\n python CntlrCmdLine.py --help"))
elif hasWebServer and options.webserver:
# webserver incompatible with file operations
if any((options.entrypointFile, options.importFiles, options.diffFile, options.versReportFile,
options.factsFile, options.factListCols, options.factTableFile,
options.conceptsFile, options.preFile, options.calFile, options.dimFile, options.formulaeFile, options.viewArcrole, options.viewFile,
)):
parser.error(_("incorrect arguments with --webserver, please try\n python CntlrCmdLine.py --help"))
else:
cntlr.startLogging(logFileName='logToBuffer')
from arelle import CntlrWebMain
CntlrWebMain.startWebserver(cntlr, options)
else:
# parse and run the FILENAME
cntlr.startLogging(logFileName=(options.logFile or "logToPrint"),
logFormat=(options.logFormat or "[%(messageCode)s] %(message)s - %(file)s"),
logLevel=(options.logLevel or "DEBUG"))
cntlr.run(options)
return cntlr
def file(self,
filepath,
binary=False,
stripDeclaration=False,
encoding=None):
'''
for text, return a tuple of (open file handle, encoding)
for binary, return a tuple of (open file handle, )
'''
archiveFileSource = self.fileSourceContainingFilepath(filepath)
if archiveFileSource is not None:
if filepath.startswith(archiveFileSource.basefile):
archiveFileName = filepath[len(archiveFileSource.basefile) +
1:]
else: # filepath.startswith(self.baseurl)
archiveFileName = filepath[len(archiveFileSource.baseurl) + 1:]
if archiveFileSource.isZip:
try:
if archiveFileSource.isZipBackslashed:
f = archiveFileName.replace("/", "\\")
else:
f = archiveFileName.replace("\\", "/")
b = archiveFileSource.fs.read(f)
if binary:
return (io.BytesIO(b), )
if encoding is None:
encoding = XmlUtil.encoding(b)
if stripDeclaration:
b = stripDeclarationBytes(b)
return (FileNamedTextIOWrapper(filepath,
io.BytesIO(b),
encoding=encoding),
encoding)
except KeyError:
raise ArchiveFileIOError(self, errno.ENOENT,
archiveFileName)
elif archiveFileSource.isTarGz:
try:
fh = archiveFileSource.fs.extractfile(archiveFileName)
b = fh.read()
fh.close(
) # doesn't seem to close properly using a with construct
if binary:
return (io.BytesIO(b), )
if encoding is None:
encoding = XmlUtil.encoding(b)
if stripDeclaration:
b = stripDeclarationBytes(b)
return (FileNamedTextIOWrapper(filepath,
io.BytesIO(b),
encoding=encoding),
encoding)
except KeyError:
raise ArchiveFileIOError(self, archiveFileName)
elif archiveFileSource.isEis:
for docElt in self.eisDocument.iter(
tag="{http://www.sec.gov/edgar/common}document"):
outfn = docElt.findtext(
"{http://www.sec.gov/edgar/common}conformedName")
if outfn == archiveFileName:
b64data = docElt.findtext(
"{http://www.sec.gov/edgar/common}contents")
if b64data:
b = base64.b64decode(b64data.encode("latin-1"))
# remove BOM codes if present
if len(b) > 3 and b[0] == 239 and b[
1] == 187 and b[2] == 191:
start = 3
length = len(b) - 3
b = b[start:start + length]
else:
start = 0
length = len(b)
if binary:
return (io.BytesIO(b), )
if encoding is None:
encoding = XmlUtil.encoding(b,
default="latin-1")
return (io.TextIOWrapper(io.BytesIO(b),
encoding=encoding),
encoding)
raise ArchiveFileIOError(self, errno.ENOENT, archiveFileName)
elif archiveFileSource.isXfd:
for data in archiveFileSource.xfdDocument.iter(tag="data"):
outfn = data.findtext("filename")
if outfn == archiveFileName:
b64data = data.findtext("mimedata")
if b64data:
b = base64.b64decode(b64data.encode("latin-1"))
# remove BOM codes if present
if len(b) > 3 and b[0] == 239 and b[
1] == 187 and b[2] == 191:
start = 3
length = len(b) - 3
b = b[start:start + length]
else:
start = 0
length = len(b)
if binary:
return (io.BytesIO(b), )
if encoding is None:
encoding = XmlUtil.encoding(b,
default="latin-1")
return (io.TextIOWrapper(io.BytesIO(b),
encoding=encoding),
encoding)
raise ArchiveFileIOError(self, errno.ENOENT, archiveFileName)
elif archiveFileSource.isInstalledTaxonomyPackage:
# remove TAXONOMY_PACKAGE_FILE_NAME from file path
if filepath.startswith(archiveFileSource.basefile):
l = len(archiveFileSource.basefile)
for f in TAXONOMY_PACKAGE_FILE_NAMES:
if filepath[l - len(f):l] == f:
filepath = filepath[0:l - len(f) -
1] + filepath[l:]
break
for pluginMethod in pluginClassMethods(
"FileSource.File"): #custom overrides for decription, etc
fileResult = pluginMethod(self.cntlr, filepath, binary,
stripDeclaration)
if fileResult is not None:
return fileResult
if binary:
return (openFileStream(self.cntlr, filepath, 'rb'), )
else:
return openXmlFileStream(self.cntlr, filepath, stripDeclaration)
def startWebserver(_cntlr, options):
"""Called once from main program in CmtlrCmdLine to initiate web server on specified local port.
To test WebServer run from source in IIS, use an entry like this: c:\python33\python.exe c:\\users\\myname\\mySourceFolder\\arelleCmdLine.py %s
:param options: OptionParser options from parse_args of main argv arguments (the argument *webserver* provides hostname and port), port being used to startup the webserver on localhost.
:type options: optparse.Values
"""
global imagesDir, cntlr, optionsPrototype
cntlr = _cntlr
imagesDir = cntlr.imagesDir
optionValuesTypes = _STR_NUM_TYPES + (type(None),)
optionsPrototype = dict((option,value if isinstance(value,_STR_NUM_TYPES) else None)
for option in dir(options)
for value in (getattr(options, option),)
if isinstance(value,optionValuesTypes) and not option.startswith('_'))
host, sep, portServer = options.webserver.partition(":")
port, sep, server = portServer.partition(":")
# start a Bottle application
app = Bottle()
# install REST API interfaces
# if necessary to support CGI hosted servers below root, add <prefix:path> as first part of routes
# and corresponding arguments to the handler methods
# allow plugins to replace or add to default REST API "routes"
pluginResult = None
for pluginMethod in pluginClassMethods("CntlrWebMain.StartWebServer"):
# returns a string containing "skip-routes" to block default routes and/or "skip-run" to block default app startup
pluginResult = pluginMethod(app, cntlr, host, port, server)
break # only provide for a single plugin of this class
if not (isinstance(pluginResult, str) and "skip-routes" in pluginResult):
app.route('/rest/login', GET, login_form)
app.route('/rest/login', POST, login_submit)
app.route('/rest/logout', GET, logout)
app.route('/favicon.ico', GET, arelleIcon)
app.route('/rest/xbrl/<file:path>/open', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/close', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/validation/xbrl', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/DTS', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/concepts', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/pre', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/table', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/cal', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/dim', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/facts', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/factTable', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/roleTypes', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/arcroleTypes', GETorPOST, validation)
app.route('/rest/xbrl/<file:path>/formulae', GETorPOST, validation)
app.route('/rest/xbrl/validation', GETorPOST, validation)
app.route('/rest/xbrl/view', GETorPOST, validation)
app.route('/rest/xbrl/open', GETorPOST, validation)
app.route('/rest/xbrl/close', GETorPOST, validation)
app.route('/images/<imgFile>', GET, image)
app.route('/rest/xbrl/diff', GET, diff)
app.route('/rest/configure', GET, configure)
app.route('/rest/stopWebServer', GET, stopWebServer)
app.route('/quickbooks/server.asmx', POST, quickbooksServer)
app.route('/rest/quickbooks/<qbReport>/xbrl-gl/<file:path>', GET, quickbooksGLrequest)
app.route('/rest/quickbooks/<qbReport>/xbrl-gl/<file:path>/view', GET, quickbooksGLrequest)
app.route('/rest/quickbooks/<qbReport>/xbrl-gl/view', GET, quickbooksGLrequest)
app.route('/rest/quickbooks/response', GET, quickbooksGLresponse)
app.route('/quickbooks/server.html', GET, quickbooksWebPage)
app.route('/quickbooks/localhost.crt', GET, localhostCertificate)
app.route('/localhost.crt', GET, localhostCertificate)
app.route('/rest/test/test', GETorPOST, testTest)
app.route('/help', GET, helpREST)
app.route('/about', GET, about)
app.route('/', GET, indexPageREST)
if server == "cgi":
# catch a non-REST interface by cgi Interface (may be a cgi app exe module, etc)
app.route('<cgiAppPath:path>', GETorPOST, cgiInterface)
if not (isinstance(pluginResult, str) and "skip-run" in pluginResult):
if server == "wsgi":
return app
elif server == "cgi":
if sys.stdin is None:
sys.stdin = open(os.devnull, 'r')
app.run(server=server)
sys.exit(0)
elif server:
sys.path.insert(0,os.path.join(os.path.dirname(__file__),"webserver"))
app.run(host=host, port=port or 80, server=server)
else:
app.run(host=host, port=port or 80)
def watchCycle(self):
while not self.stopRequested:
rssWatchOptions = self.rssModelXbrl.modelManager.rssWatchOptions
# check rss expiration
rssHeaders = self.cntlr.webCache.getheaders(self.rssModelXbrl.modelManager.rssWatchOptions.get("feedSourceUri"))
expires = parseRfcDatetime(rssHeaders.get("expires"))
reloadNow = True # texpires and expires > datetime.datetime.now()
# reload rss feed
self.rssModelXbrl.reload('checking RSS items', reloadCache=reloadNow)
if self.stopRequested: break
# setup validator
postLoadActions = []
if rssWatchOptions.get("validateDisclosureSystemRules"):
self.instValidator = ValidateFiling.ValidateFiling(self.rssModelXbrl)
postLoadActions.append(_("validating"))
elif rssWatchOptions.get("validateXbrlRules") or rssWatchOptions.get("validateFormulaAssertions"):
self.instValidator = ValidateXbrl.ValidateXbrl(self.rssModelXbrl)
postLoadActions.append(_("validating"))
if (rssWatchOptions.get("validateFormulaAssertions")):
postLoadActions.append(_("running formulas"))
else:
self.instValidator = None
matchTextExpr = rssWatchOptions.get("matchTextExpr")
if matchTextExpr:
matchPattern = re.compile(matchTextExpr)
postLoadActions.append(_("matching text"))
else:
matchPattern= None
postLoadAction = ', '.join(postLoadActions)
# anything to check new filings for
if (rssWatchOptions.get("validateDisclosureSystemRules") or
rssWatchOptions.get("validateXbrlRules") or
rssWatchOptions.get("validateCalcLinkbase") or
rssWatchOptions.get("validateFormulaAssertions") or
rssWatchOptions.get("alertMatchedFactText") or
any(pluginXbrlMethod(rssWatchOptions)
for pluginXbrlMethod in pluginClassMethods("RssWatch.HasWatchAction"))
):
# form keys in ascending order of pubdate
pubDateRssItems = []
for rssItem in self.rssModelXbrl.modelDocument.rssItems:
pubDateRssItems.append((rssItem.pubDate,rssItem.objectId()))
for pubDate, rssItemObjectId in sorted(pubDateRssItems):
rssItem = self.rssModelXbrl.modelObject(rssItemObjectId)
# update ui thread via modelManager (running in background here)
self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId())
if self.stopRequested:
break
latestPubDate = XmlUtil.datetimeValue(rssWatchOptions.get("latestPubDate"))
if (latestPubDate and
rssItem.pubDate < latestPubDate):
continue
try:
# try zipped URL if possible, else expanded instance document
modelXbrl = ModelXbrl.load(self.rssModelXbrl.modelManager,
openFileSource(rssItem.zippedUrl, self.cntlr),
postLoadAction)
if self.stopRequested:
modelXbrl.close()
break
emailAlert = False
if modelXbrl.modelDocument is None:
modelXbrl.error("arelle.rssWatch",
_("RSS item %(company)s %(form)s document not loaded: %(date)s"),
modelXbrl=modelXbrl, company=rssItem.companyName,
form=rssItem.formType, date=rssItem.filingDate)
rssItem.status = "not loadable"
else:
# validate schema, linkbase, or instance
if self.stopRequested:
modelXbrl.close()
break
if self.instValidator:
self.instValidator.validate(modelXbrl)
if modelXbrl.errors and rssWatchOptions.get("alertValiditionError"):
emailAlert = True
for pluginXbrlMethod in pluginClassMethods("RssWatch.DoWatchAction"):
pluginXbrlMethod(modelXbrl, rssWatchOptions, rssItem)
# check match expression
if matchPattern:
for fact in modelXbrl.factsInInstance:
v = fact.value
if v is not None:
m = matchPattern.search(v)
if m:
fr, to = m.span()
msg = _("Fact Variable {0}\n context {1}\n matched text: {2}").format(
fact.qname, fact.contextID, v[max(0,fr-20):to+20])
modelXbrl.info("arelle.rssInfo",
msg,
modelXbrl=modelXbrl) # msg as code passes it through to the status
if rssWatchOptions.get("alertMatchedFactText"):
emailAlert = True
if (rssWatchOptions.get("formulaFileUri") and rssWatchOptions.get("validateFormulaAssertions") and
self.instValidator):
# attach formulas
ModelDocument.load(modelXbrl, rssWatchOptions["formulaFileUri"])
ValidateFormula.validate(self.instValidator)
rssItem.setResults(modelXbrl)
modelXbrl.close()
del modelXbrl # completely dereference
self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId())
if rssItem.assertionUnsuccessful and rssWatchOptions.get("alertAssertionUnsuccessful"):
emailAlert = True
msg = _("Filing CIK {0}\n "
"company {1}\n "
"published {2}\n "
"form type {3}\n "
"filing date {4}\n "
"period {5}\n "
"year end {6}\n "
"results: {7}").format(
rssItem.cikNumber,
rssItem.companyName,
rssItem.pubDate,
rssItem.formType,
rssItem.filingDate,
rssItem.period,
rssItem.fiscalYearEnd,
rssItem.status)
self.rssModelXbrl.info("arelle:rssWatch", msg, modelXbrl=self.rssModelXbrl)
emailAddress = rssWatchOptions.get("emailAddress")
if emailAlert and emailAddress:
self.rssModelXbrl.modelManager.showStatus(_("sending e-mail alert"))
import smtplib
from email.mime.text import MIMEText
emailMsg = MIMEText(msg)
emailMsg["Subject"] = _("Arelle RSS Watch alert on {0}").format(rssItem.companyName)
emailMsg["From"] = emailAddress
emailMsg["To"] = emailAddress
smtp = smtplib.SMTP()
smtp.sendmail(emailAddress, [emailAddress], emailMsg.as_string())
smtp.quit()
self.rssModelXbrl.modelManager.showStatus(_("RSS item {0}, {1} completed, status {2}").format(rssItem.companyName, rssItem.formType, rssItem.status), 3500)
self.rssModelXbrl.modelManager.cntlr.rssWatchUpdateOption(rssItem.pubDate.strftime('%Y-%m-%dT%H:%M:%S'))
except Exception as err:
self.rssModelXbrl.error("arelle.rssError",
_("RSS item %(company)s, %(form)s, %(date)s, exception: %(error)s"),
modelXbrl=self.rssModelXbrl, company=rssItem.companyName,
form=rssItem.formType, date=rssItem.filingDate, error=err,
exc_info=True)
if self.stopRequested: break
if self.stopRequested:
self.cntlr.showStatus(_("RSS watch, stop requested"), 10000)
else:
import time
time.sleep(600)
self.thread = None # close thread
self.stopRequested = False
def clear(self):
self.selection = None
self.standardTaxonomiesDict = {}
self.familyHrefs = {}
self.standardLocalHrefs = set()
self.standardAuthorities = set()
self.baseTaxonomyNamespaces = set()
self.standardPrefixes = {}
self.names = []
self.name = None
self.validationType = None
self.exclusiveTypesPattern = None # regex of type matches exclusive with validationType
# previously built-in types (intent to replace with plugin defined types)
self.EFM = False
self.GFM = False
self.EFMorGFM = False
self.HMRC = False
self.SBRNL = False
self.pluginTypes = set()
for pluginXbrlMethod in pluginClassMethods("DisclosureSystem.Types"):
for typeName, typeTestVariable in pluginXbrlMethod(self):
setattr(self, typeTestVariable, False)
self.pluginTypes.add(typeName)
self.validateFileText = False
self.validateEntryText = False
self.allowedExternalHrefPattern = None
self.allowedImageTypes = None
self.schemaValidateSchema = None
self.blockDisallowedReferences = False
self.maxSubmissionSubdirectoryEntryNesting = 0
self.defaultXmlLang = None
self.defaultXmlEncoding = "utf-8"
self.xmlLangPattern = None
self.xmlLangIsInheritable = True # for label and footnote, spec sections 4.11.1.2.1 and 5.2.2.2.1
self.defaultLanguage = None
self.language = None
self.standardTaxonomiesUrl = None
self.mappingsUrl = os.path.join(self.modelManager.cntlr.configDir, "mappings.xml")
self.mappedFiles = {}
self.mappedPaths = []
self.utrUrl = ["http://www.xbrl.org/utr/utr.xml"]
self.utrStatusFilters = None
self.utrTypeEntries = None
self.identifierSchemePattern = None
self.identifierValuePattern = None
self.identifierValueName = None
self.contextElement = None
self.roleDefinitionPattern = None
self.labelCheckPattern = None
self.labelTrimPattern = None
self.deiNamespacePattern = None
self.deiAmendmentFlagElement = None
self.deiCurrentFiscalYearEndDateElement = None
self.deiDocumentFiscalYearFocusElement = None
self.deiDocumentPeriodEndDateElement = None
self.deiFilerIdentifierElement = None
self.deiFilerNameElement = None
self.logLevelFilter = None
self.logCodeFilter = None
self.standardTaxonomyDatabase = None
self.standardTaxonomyUrlPattern = None
self.options = None
self.version = (0,0,0)
def loadCustomTransforms(self):
if self.customTransforms is None:
self.customTransforms = {}
for pluginMethod in pluginClassMethods("ModelManager.LoadCustomTransforms"):
pluginMethod(self.customTransforms)
def parentMenuCommand(cntl):
for i in range(1,100):
for pluginMethod in pluginClassMethods("Import.Unpackaged.Entry{}".format(i)):
pluginMethod()
def validateRssFeed(self):
self.modelXbrl.info("info", "RSS Feed", modelDocument=self.modelXbrl)
from arelle.FileSource import openFileSource
reloadCache = getattr(self.modelXbrl, "reloadCache", False)
for rssItem in self.modelXbrl.modelDocument.rssItems:
if getattr(rssItem, "skipRssItem", False):
self.modelXbrl.info(
"info",
_("skipping RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"
),
modelObject=rssItem,
accessionNumber=rssItem.accessionNumber,
formType=rssItem.formType,
companyName=rssItem.companyName,
period=rssItem.period)
continue
self.modelXbrl.info(
"info",
_("RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"
),
modelObject=rssItem,
accessionNumber=rssItem.accessionNumber,
formType=rssItem.formType,
companyName=rssItem.companyName,
period=rssItem.period)
modelXbrl = None
try:
modelXbrl = ModelXbrl.load(
self.modelXbrl.modelManager,
openFileSource(rssItem.zippedUrl,
self.modelXbrl.modelManager.cntlr,
reloadCache=reloadCache),
_("validating"),
rssItem=rssItem)
for pluginXbrlMethod in pluginClassMethods(
"RssItem.Xbrl.Loaded"):
pluginXbrlMethod(modelXbrl, {}, rssItem)
if getattr(rssItem, "doNotProcessRSSitem",
False) or modelXbrl.modelDocument is None:
modelXbrl.close()
continue # skip entry based on processing criteria
self.instValidator.validate(
modelXbrl,
self.modelXbrl.modelManager.formulaOptions.typedParameters(
))
self.instValidator.close()
rssItem.setResults(modelXbrl)
self.modelXbrl.modelManager.viewModelObject(
self.modelXbrl, rssItem.objectId())
for pluginXbrlMethod in pluginClassMethods("Validate.RssItem"):
pluginXbrlMethod(self, modelXbrl, rssItem)
modelXbrl.close()
except Exception as err:
self.modelXbrl.error(
"exception:" + type(err).__name__,
_("RSS item validation exception: %(error)s, instance: %(instance)s"
),
modelXbrl=(self.modelXbrl, modelXbrl),
instance=rssItem.zippedUrl,
error=err,
exc_info=True)
try:
self.instValidator.close()
if modelXbrl is not None:
modelXbrl.close()
except Exception as err:
pass
del modelXbrl # completely dereference