def backgroundLoadXbrl(self, filesource, importToDTS):
startedAt = time.time()
try:
if importToDTS:
action = _("imported")
modelXbrl = self.modelManager.modelXbrl
if modelXbrl:
ModelDocument.load(modelXbrl, filesource.url)
else:
action = _("loaded")
modelXbrl = self.modelManager.load(filesource, _("views loading"))
except Exception as err:
msg = _("Exception loading {0}: {1}, at {2}").format(
filesource.url,
err,
traceback.format_tb(sys.exc_info()[2]))
# not sure if message box can be shown from background thread
# tkinter.messagebox.showwarning(_("Exception loading"),msg, parent=self.parent)
self.addToLog(msg);
return
if modelXbrl and modelXbrl.modelDocument:
self.addToLog(format_string(self.modelManager.locale,
_("%s in %.2f secs"),
(action, time.time() - startedAt)))
self.showStatus(_("{0}, preparing views").format(action))
self.uiThreadQueue.put((self.showLoadedXbrl, [modelXbrl, importToDTS]))
else:
self.addToLog(format_string(self.modelManager.locale,
_("not successfully %s in %.2f secs"),
(action, time.time() - startedAt)))
def lookup(self, document, proxyElement):
# check if proxyElement's namespace is not known
ns, sep, ln = proxyElement.tag.partition("}")
if sep:
ns = ns[1:]
else:
ln = ns
ns = None
if (ns and
ns not in self.discoveryAttempts and
ns not in self.modelXbrl.namespaceDocs):
# is schema loadable? requires a schemaLocation
from arelle import XmlUtil, ModelDocument
relativeUrl = XmlUtil.schemaLocation(proxyElement, ns)
self.discoveryAttempts.add(ns)
if relativeUrl:
doc = ModelDocument.loadSchemalocatedSchema(self.modelXbrl, proxyElement, relativeUrl, ns, self.baseUrl)
modelObjectClass = self.modelXbrl.matchSubstitutionGroup(
qnameNsLocalName(ns, ln),
elementSubstitutionModelClass)
if modelObjectClass is not None:
return modelObjectClass
else:
xlinkType = proxyElement.get("{http://www.w3.org/1999/xlink}type")
if xlinkType == "extended": return ModelLink
elif xlinkType == "locator": return ModelLocator
elif xlinkType == "resource": return ModelResource
return ModelObject
def doc(xc, p, contextItem, args):
if len(args) != 1: raise XPathContext.FunctionNumArgs()
uri = stringArg(xc, args, 0, "xs:string", emptyFallback=None)
if uri is None:
return ()
if xc.progHeader is None or xc.progHeader.element is None:
raise XPathContext.XPathException(
p, 'err:FODC0005',
_('Function xf:doc no formula resource element for {0}').format(
uri))
if not UrlUtil.isValid(uri):
raise XPathContext.XPathException(
p, 'err:FODC0005',
_('Function xf:doc $uri is not valid {0}').format(uri))
normalizedUri = xc.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
uri,
xc.progHeader.element.modelDocument.baseForElement(
xc.progHeader.element))
if normalizedUri in xc.modelXbrl.urlDocs:
return xc.modelXbrl.urlDocs[normalizedUri].xmlDocument
modelDocument = ModelDocument.load(xc.modelXbrl, normalizedUri)
if modelDocument is None:
raise XPathContext.XPathException(
p, 'err:FODC0005',
_('Function xf:doc $uri not successfully loaded {0}').format(uri))
# assure that document is validated
XmlValidate.validate(xc.modelXbrl, modelDocument.xmlRootElement)
return modelDocument.xmlDocument
def load(modelManager, url, nextaction=None, base=None, useFileSource=None):
if nextaction is None: nextaction = _("loading")
from arelle import (ModelDocument, FileSource)
modelXbrl = create(modelManager)
if useFileSource is not None:
modelXbrl.fileSource = useFileSource
modelXbrl.closeFileSource = False
url = url
elif isinstance(url,FileSource.FileSource):
modelXbrl.fileSource = url
modelXbrl.closeFileSource= True
url = modelXbrl.fileSource.url
else:
modelXbrl.fileSource = FileSource.FileSource(url)
modelXbrl.closeFileSource= True
modelXbrl.modelDocument = ModelDocument.load(modelXbrl, url, base, isEntry=True)
del modelXbrl.entryLoadingUrl
if modelXbrl.modelDocument is not None and modelXbrl.modelDocument.type < ModelDocument.Type.DTSENTRIES:
# at this point DTS is fully discovered but schemaLocated xsd's are not yet loaded
modelDocumentsSchemaLocated = set()
while True: # need this logic because each new pass may add new urlDocs
modelDocuments = set(modelXbrl.urlDocs.values()) - modelDocumentsSchemaLocated
if not modelDocuments:
break
modelDocument = modelDocuments.pop()
modelDocumentsSchemaLocated.add(modelDocument)
modelDocument.loadSchemalocatedSchemas()
#from arelle import XmlValidate
#uncomment for trial use of lxml xml schema validation of entry document
#XmlValidate.xmlValidate(modelXbrl.modelDocument)
modelManager.cntlr.webCache.saveUrlCheckTimes()
modelManager.showStatus(_("xbrl loading finished, {0}...").format(nextaction))
return modelXbrl
def lookup(self, document, proxyElement):
# check if proxyElement's namespace is not known
ns, sep, ln = proxyElement.tag.partition("}")
if sep:
ns = ns[1:]
else:
ln = ns
ns = None
if (ns and
ns not in self.discoveryAttempts and
ns not in self.modelXbrl.namespaceDocs):
# is schema loadable? requires a schemaLocation
from arelle import XmlUtil, ModelDocument
relativeUrl = XmlUtil.schemaLocation(proxyElement, ns)
self.discoveryAttempts.add(ns)
if relativeUrl:
doc = ModelDocument.loadSchemalocatedSchema(self.modelXbrl, proxyElement, relativeUrl, ns, self.baseUrl)
modelObjectClass = self.modelXbrl.matchSubstitutionGroup(
qname(ns, ln),
elementSubstitutionModelClass)
if modelObjectClass is not None:
return modelObjectClass
else:
xlinkType = proxyElement.get("{http://www.w3.org/1999/xlink}type")
if xlinkType == "extended": return ModelLink
elif xlinkType == "locator": return ModelLocator
elif xlinkType == "resource": return ModelResource
return ModelObject
def load(modelManager, url, nextaction, base=None):
from arelle import (ModelDocument, FileSource)
modelXbrl = create(modelManager)
if isinstance(url,FileSource.FileSource):
modelXbrl.fileSource = url
url = modelXbrl.fileSource.url
else:
modelXbrl.fileSource = FileSource.FileSource(url)
modelXbrl.modelDocument = ModelDocument.load(modelXbrl, url, base, isEntry=True)
# at this point DTS is fully discovered but schemaLocated xsd's are not yet loaded
modelDocumentsSchemaLocated = set()
while True: # need this logic because each new pass may add new urlDocs
modelDocuments = set(modelXbrl.urlDocs.values()) - modelDocumentsSchemaLocated
if not modelDocuments:
break
modelDocument = modelDocuments.pop()
modelDocumentsSchemaLocated.add(modelDocument)
modelDocument.loadSchemalocatedSchemas()
#from arelle import XmlValidate
#uncomment for trial use of lxml xml schema validation of entry document
#XmlValidate.xmlValidate(modelXbrl.modelDocument)
modelManager.cntlr.webCache.saveUrlCheckTimes()
modelManager.showStatus(_("xbrl loading finished, {0}...").format(nextaction))
return modelXbrl
def createInstance(self, url=None):
"""Creates an instance document for a DTS which didn't have an instance document, such as
to create a new instance for a DTS which was loaded from a taxonomy or linkbase entry point.
:param url: File name to save the new instance document
:type url: str
"""
from arelle import (ModelDocument, FileSource)
if self.modelDocument.type == ModelDocument.Type.INSTANCE: # entry already is an instance
return self.modelDocument # use existing instance entry point
priorFileSource = self.fileSource
self.fileSource = FileSource.FileSource(url)
if self.uri.startswith("http://"):
schemaRefUri = self.uri
else: # relativize local paths
schemaRefUri = os.path.relpath(self.uri, os.path.dirname(url))
self.modelDocument = ModelDocument.create(self, ModelDocument.Type.INSTANCE, url, schemaRefs=[schemaRefUri], isEntry=True)
if priorFileSource:
priorFileSource.close()
self.closeFileSource= True
del self.entryLoadingUrl
# reload dts views
from arelle import ViewWinDTS
for view in self.views:
if isinstance(view, ViewWinDTS.ViewDTS):
self.modelManager.cntlr.uiThreadQueue.put((view.view, []))
def xfxc_element(xc, p, contextItem, args):
if not 2 <= len(args) <= 4: raise XPathContext.FunctionNumArgs()
qn = qnameArg(xc, p, args, 0, 'QName', emptyFallback=None)
attrArg = args[1] if isinstance(args[1], (list, tuple)) else (args[1], )
# attributes have to be pairs
if attrArg:
if len(attrArg) & 1 or any(
not isinstance(attrArg[i], (QName, _STR_BASE))
for i in range(0, len(attrArg), 2)):
raise XPathContext.FunctionArgType(
1,
"((xs:qname|xs:string),xs:anyAtomicValue)",
errCode="xfxce:AttributesNotNameValuePairs")
else:
attrParam = [
(attrArg[i], attrArg[i + 1]
) # need name-value pairs for XmlUtil function
for i in range(0, len(attrArg), 2)
]
else:
attrParam = None
value = atomicArg(xc, p, args, 2, "xs:anyAtomicType", emptyFallback='')
if not value: # be sure '' is None so no text node is created
value = None
if len(args) < 4:
childElements = None
else:
childElements = xc.flattenSequence(args[3])
# scratchpad instance document emulates fn:doc( ) to hold XML nodes
scratchpadXmlDocUrl = "http://www.xbrl.org/2012/function/creation/xml_scratchpad.xml"
if scratchpadXmlDocUrl in xc.modelXbrl.urlDocs:
modelDocument = xc.modelXbrl.urlDocs[scratchpadXmlDocUrl]
else:
# create scratchpad xml document
# this will get the fake instance document in the list of modelXbrl docs so that it is garbage collected
from arelle import ModelDocument
modelDocument = ModelDocument.create(
xc.modelXbrl,
ModelDocument.Type.UnknownXML,
scratchpadXmlDocUrl,
initialXml=
"<xfc:dummy xmlns:xfc='http://www.xbrl.org/2012/function/creation'/>"
)
newElement = XmlUtil.addChild(modelDocument.xmlRootElement,
qn,
attributes=attrParam,
text=value)
if childElements:
for element in childElements:
if isinstance(element, etree.ElementBase):
newElement.append(element)
# node myst be validated for use in instance creation (typed dimension references)
XmlValidate.validate(xc.modelXbrl, newElement)
return newElement
def create(modelManager, newDocumentType=None, url=None, schemaRefs=None, createModelDocument=True, isEntry=False):
from arelle import (ModelDocument, FileSource)
modelXbrl = ModelXbrl(modelManager)
modelXbrl.locale = modelManager.locale
if newDocumentType:
modelXbrl.fileSource = FileSource.FileSource(url)
if createModelDocument:
modelXbrl.modelDocument = ModelDocument.create(modelXbrl, newDocumentType, url, schemaRefs=schemaRefs, isEntry=isEntry)
return modelXbrl
def reload(self, nextaction, reloadCache=False):
from arelle import ModelDocument
self.init(keepViews=True)
self.modelDocument = ModelDocument.load(self,
self.fileSource.url,
isEntry=True,
reloadCache=reloadCache)
self.modelManager.showStatus(
_("xbrl loading finished, {0}...").format(nextaction), 5000)
self.modelManager.reloadViews(self)
def create(modelManager, newDocumentType=None, url=None, schemaRefs=None, createModelDocument=True, isEntry=False):
from arelle import (ModelDocument, FileSource)
modelXbrl = ModelXbrl(modelManager)
modelXbrl.locale = modelManager.locale
if newDocumentType:
modelXbrl.fileSource = FileSource.FileSource(url) # url may be an open file handle, use str(url) below
modelXbrl.closeFileSource= True
if createModelDocument:
modelXbrl.modelDocument = ModelDocument.create(modelXbrl, newDocumentType, str(url), schemaRefs=schemaRefs, isEntry=isEntry)
if isEntry:
del modelXbrl.entryLoadingUrl
return modelXbrl
def reload(self,nextaction,reloadCache=False):
"""Reloads all model objects from their original entry point URL, preserving any open views (which are reloaded).
:param nextAction: status line text string, if any, to show upon completion
:type nextAction: str
:param reloadCache: True to force clearing and reloading of web cache, if working online.
:param reloadCache: bool
"""
from arelle import ModelDocument
self.init(keepViews=True)
self.modelDocument = ModelDocument.load(self, self.fileSource.url, isEntry=True, reloadCache=reloadCache)
self.modelManager.showStatus(_("xbrl loading finished, {0}...").format(nextaction),5000)
self.modelManager.reloadViews(self)
def load(modelManager, url, nextaction=None, base=None, useFileSource=None, errorCaptureLevel=None):
"""Each loaded instance, DTS, testcase, testsuite, versioning report, or RSS feed, is represented by an
instance of a ModelXbrl object. The ModelXbrl object has a collection of ModelDocument objects, each
representing an XML document (for now, with SQL whenever its time comes). One of the modelDocuments of
the ModelXbrl is the entry point (of discovery or of the test suite).
:param url: may be a filename or FileSource object
:type url: str or FileSource
:param nextaction: text to use as status line prompt on conclusion of loading and discovery
:type nextaction: str
:param base: the base URL if any (such as a versioning report's URL when loading to/from DTS modelXbrl).
:type base: str
:param useFileSource: for internal use (when an entry point is in a FileSource archive and discovered files expected to also be in the entry point's archive.
:type useFileSource: bool
:returns: ModelXbrl -- a new modelXbrl, performing DTS discovery for instance, inline XBRL, schema, linkbase, and versioning report entry urls
"""
if nextaction is None: nextaction = _("loading")
from arelle import (ModelDocument, FileSource)
modelXbrl = create(modelManager, errorCaptureLevel=errorCaptureLevel)
if useFileSource is not None:
modelXbrl.fileSource = useFileSource
modelXbrl.closeFileSource = False
url = url
elif isinstance(url,FileSource.FileSource):
modelXbrl.fileSource = url
modelXbrl.closeFileSource= True
url = modelXbrl.fileSource.url
else:
modelXbrl.fileSource = FileSource.FileSource(url)
modelXbrl.closeFileSource= True
modelXbrl.modelDocument = ModelDocument.load(modelXbrl, url, base, isEntry=True)
del modelXbrl.entryLoadingUrl
if modelXbrl.modelDocument is not None and modelXbrl.modelDocument.type < ModelDocument.Type.DTSENTRIES:
# at this point DTS is fully discovered but schemaLocated xsd's are not yet loaded
modelDocumentsSchemaLocated = set()
while True: # need this logic because each new pass may add new urlDocs
modelDocuments = set(modelXbrl.urlDocs.values()) - modelDocumentsSchemaLocated
if not modelDocuments:
break
modelDocument = modelDocuments.pop()
modelDocumentsSchemaLocated.add(modelDocument)
modelDocument.loadSchemalocatedSchemas()
#from arelle import XmlValidate
#uncomment for trial use of lxml xml schema validation of entry document
#XmlValidate.xmlValidate(modelXbrl.modelDocument)
modelManager.cntlr.webCache.saveUrlCheckTimes()
modelManager.showStatus(_("xbrl loading finished, {0}...").format(nextaction))
return modelXbrl
def xfxc_element(xc, p, contextItem, args):
if not 2 <= len(args) <= 4: raise XPathContext.FunctionNumArgs()
qn = qnameArg(xc, p, args, 0, 'QName', emptyFallback=None)
attrArg = args[1] if isinstance(args[1],(list,tuple)) else (args[1],)
# attributes have to be pairs
if attrArg:
if len(attrArg) & 1 or any(not isinstance(attrArg[i], (QName, _STR_BASE))
for i in range(0, len(attrArg),2)):
raise XPathContext.FunctionArgType(1,"((xs:qname|xs:string),xs:anyAtomicValue)", errCode="xfxce:AttributesNotNameValuePairs")
else:
attrParam = [(attrArg[i],attrArg[i+1]) # need name-value pairs for XmlUtil function
for i in range(0, len(attrArg),2)]
else:
attrParam = None
value = atomicArg(xc, p, args, 2, "xs:anyAtomicType", emptyFallback='')
if not value: # be sure '' is None so no text node is created
value = None
if len(args) < 4:
childElements = None
else:
childElements = xc.flattenSequence(args[3])
# scratchpad instance document emulates fn:doc( ) to hold XML nodes
scratchpadXmlDocUrl = "http://www.xbrl.org/2012/function/creation/xml_scratchpad.xml"
if scratchpadXmlDocUrl in xc.modelXbrl.urlDocs:
modelDocument = xc.modelXbrl.urlDocs[scratchpadXmlDocUrl]
else:
# create scratchpad xml document
# this will get the fake instance document in the list of modelXbrl docs so that it is garbage collected
from arelle import ModelDocument
modelDocument = ModelDocument.create(xc.modelXbrl,
ModelDocument.Type.UnknownXML,
scratchpadXmlDocUrl,
initialXml="<xfc:dummy xmlns:xfc='http://www.xbrl.org/2012/function/creation'/>")
newElement = XmlUtil.addChild(modelDocument.xmlRootElement,
qn,
attributes=attrParam,
text=value)
if childElements:
for element in childElements:
if isinstance(element, etree.ElementBase):
newElement.append(element)
# node myst be validated for use in instance creation (typed dimension references)
XmlValidate.validate(xc.modelXbrl, newElement)
return newElement
def lookup(self, document, proxyElement):
# check if proxyElement's namespace is not known
ns, sep, ln = proxyElement.tag.partition("}")
if sep:
ns = ns[1:]
else:
ln = ns
ns = None
if (ns and ns not in self.discoveryAttempts
and ns not in self.modelXbrl.namespaceDocs):
# is schema loadable? requires a schemaLocation
relativeUrl = XmlUtil.schemaLocation(proxyElement, ns)
self.discoveryAttempts.add(ns)
if relativeUrl:
doc = ModelDocument.loadSchemalocatedSchema(
self.modelXbrl, proxyElement, relativeUrl, ns,
self.baseUrl)
modelObjectClass = self.modelXbrl.matchSubstitutionGroup(
qnameNsLocalName(ns, ln), elementSubstitutionModelClass)
if modelObjectClass is not None:
return modelObjectClass
elif (self.streamingOrSkipDTS
and ns not in (XbrlConst.xbrli, XbrlConst.link)):
# self.makeelementParentModelObject is set in streamingExtensions.py and ModelXbrl.createFact
ancestor = proxyElement.getparent() or getattr(
self.modelXbrl, "makeelementParentModelObject", None)
while ancestor is not None:
tag = ancestor.tag # not a modelObject yet, just parser prototype
if tag.startswith("{http://www.xbrl.org/2003/instance}"
) or tag.startswith(
"{http://www.xbrl.org/2003/linkbase}"):
if tag == "{http://www.xbrl.org/2003/instance}xbrl":
return ModelFact # element not parented by context or footnoteLink
else:
break # cannot be a fact
ancestor = ancestor.getparent()
xlinkType = proxyElement.get("{http://www.w3.org/1999/xlink}type")
if xlinkType == "extended": return ModelLink
elif xlinkType == "locator": return ModelLocator
elif xlinkType == "resource": return ModelResource
return ModelObject
def lookup(self, document, proxyElement):
# check if proxyElement's namespace is not known
ns, sep, ln = proxyElement.tag.partition("}")
if sep:
ns = ns[1:]
else:
ln = ns
ns = None
if (ns and
ns not in self.discoveryAttempts and
ns not in self.modelXbrl.namespaceDocs):
# is schema loadable? requires a schemaLocation
relativeUrl = XmlUtil.schemaLocation(proxyElement, ns)
self.discoveryAttempts.add(ns)
if relativeUrl:
doc = ModelDocument.loadSchemalocatedSchema(self.modelXbrl, proxyElement, relativeUrl, ns, self.baseUrl)
modelObjectClass = self.modelXbrl.matchSubstitutionGroup(
qnameNsLocalName(ns, ln),
elementSubstitutionModelClass)
if modelObjectClass is not None:
return modelObjectClass
elif (self.streamingOrSkipDTS and
ns not in (XbrlConst.xbrli, XbrlConst.link)):
# self.makeelementParentModelObject is set in streamingExtensions.py and ModelXbrl.createFact
ancestor = proxyElement.getparent() or getattr(self.modelXbrl, "makeelementParentModelObject", None)
while ancestor is not None:
tag = ancestor.tag # not a modelObject yet, just parser prototype
if tag.startswith("{http://www.xbrl.org/2003/instance}") or tag.startswith("{http://www.xbrl.org/2003/linkbase}"):
if tag == "{http://www.xbrl.org/2003/instance}xbrl":
return ModelFact # element not parented by context or footnoteLink
else:
break # cannot be a fact
ancestor = ancestor.getparent()
xlinkType = proxyElement.get("{http://www.w3.org/1999/xlink}type")
if xlinkType == "extended": return ModelLink
elif xlinkType == "locator": return ModelLocator
elif xlinkType == "resource": return ModelResource
return ModelObject
def createInstance(self, url=None):
from arelle import (ModelDocument, FileSource)
if self.modelDocument.type == ModelDocument.Type.INSTANCE: # entry already is an instance
return self.modelDocument # use existing instance entry point
priorFileSource = self.fileSource
self.fileSource = FileSource.FileSource(url)
if self.uri.startswith("http://"):
schemaRefUri = self.uri
else: # relativize local paths
schemaRefUri = os.path.relpath(self.uri, os.path.dirname(url))
self.modelDocument = ModelDocument.create(self, ModelDocument.Type.INSTANCE, url, schemaRefs=[schemaRefUri], isEntry=True)
if priorFileSource:
priorFileSource.close()
self.closeFileSource= True
del self.entryLoadingUrl
# reload dts views
from arelle import ViewWinDTS
for view in self.views:
if isinstance(view, ViewWinDTS.ViewDTS):
self.modelManager.cntlr.uiThreadQueue.put((view.view, []))
def doc(xc, p, contextItem, args):
if len(args) != 1: raise XPathContext.FunctionNumArgs()
uri = stringArg(xc, args, 0, "xs:string", emptyFallback=None)
if uri is None:
return ()
if xc.progHeader is None or xc.progHeader.element is None:
raise XPathContext.XPathException(p, 'err:FODC0005', _('Function xf:doc no formula resource element for {0}').format(uri))
if not UrlUtil.isValid(uri):
raise XPathContext.XPathException(p, 'err:FODC0005', _('Function xf:doc $uri is not valid {0}').format(uri))
normalizedUri = xc.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
uri,
xc.progHeader.element.modelDocument.baseForElement(xc.progHeader.element))
if normalizedUri in xc.modelXbrl.urlDocs:
return xc.modelXbrl.urlDocs[normalizedUri].xmlDocument
modelDocument = ModelDocument.load(xc.modelXbrl, normalizedUri)
if modelDocument is None:
raise XPathContext.XPathException(p, 'err:FODC0005', _('Function xf:doc $uri not successfully loaded {0}').format(uri))
# assure that document is validated
XmlValidate.validate(xc.modelXbrl, modelDocument.xmlRootElement)
return modelDocument.xmlDocument
def create(modelManager,
newDocumentType=None,
url=None,
schemaRefs=None,
createModelDocument=True,
isEntry=False):
from arelle import (ModelDocument, FileSource)
modelXbrl = ModelXbrl(modelManager)
modelXbrl.locale = modelManager.locale
if newDocumentType:
modelXbrl.fileSource = FileSource.FileSource(
url) # url may be an open file handle, use str(url) below
modelXbrl.closeFileSource = True
if createModelDocument:
modelXbrl.modelDocument = ModelDocument.create(
modelXbrl,
newDocumentType,
str(url),
schemaRefs=schemaRefs,
isEntry=isEntry)
if isEntry:
del modelXbrl.entryLoadingUrl
return modelXbrl
def createInstance(self, url=None):
from arelle import (ModelDocument, FileSource)
if self.modelDocument.type == ModelDocument.Type.INSTANCE: # entry already is an instance
return self.modelDocument # use existing instance entry point
priorFileSource = self.fileSource
self.fileSource = FileSource.FileSource(url)
if self.uri.startswith("http://"):
schemaRefUri = self.uri
else: # relativize local paths
schemaRefUri = os.path.relpath(self.uri, os.path.dirname(url))
self.modelDocument = ModelDocument.create(self,
ModelDocument.Type.INSTANCE,
url,
schemaRefs=[schemaRefUri],
isEntry=True)
if priorFileSource:
priorFileSource.close()
self.closeFileSource = True
del self.entryLoadingUrl
# reload dts views
from arelle import ViewWinDTS
for view in self.views:
if isinstance(view, ViewWinDTS.ViewDTS):
self.modelManager.cntlr.uiThreadQueue.put((view.view, []))
def runFromExcel(self, options):
testGenFileName = options.excelfilename
#testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls"
testGenDir = os.path.dirname(testGenFileName)
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt'
modelTestcases = ModelXbrl.create(self.modelManager)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow))
# start index file
indexFiles = [testGenDir + os.sep + 'creationTestcasesIndex.xml',
testGenDir + os.sep + 'consumptionTestcasesIndex.xml']
indexDocs = [xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today)
) for purpose in ("Creation","Consumption")]
testcasesElements = [XmlUtil.child(indexDoc, None, "testcases") for indexDoc in indexDocs]
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
row = testcaseIndexSheet.row(iRow)
if row[0].ctype == xlrd.XL_CELL_EMPTY or row[1].ctype == xlrd.XL_CELL_EMPTY or row[2].ctype == xlrd.XL_CELL_EMPTY:
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
intention = row[3].value
if intention is None or len(intention) == 0:
continue # test not ready to run
reason = row[4].value
expectedEvent = row[5].value
base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and reason.lower() not in ("n.a.", "error") and expectedEvent != "N.A.":
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base)
else:
modelDTS = ModelXbrl.create(self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts",
base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS, uri.strip(), base=base)
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom and modelDTSto:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace("from.xsd","report.xml")
reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile,
base)
testcasesDir = os.path.dirname(os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml",
testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"]
for i,testcaseFile in enumerate(testcaseFiles):
XmlUtil.addChild(testcasesElements[i], None, "testcase",
("uri",
testcaseFile[len(testGenDir)+1:].replace("\\","/")) )
# start testcase file
testcaseDocs = [xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'<name>{0}</name>'
'<description>{0}</description>'
'</testcase>'.format(testcaseName,purpose,today)
) for purpose in ("Creation","Consumption")]
testcaseElements = [XmlUtil.child(testcaseDoc, conformanceNS, "testcase") for testcaseDoc in testcaseDocs]
priorTestcasesDir = testcasesDir
variationID = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(modelTestcases)
modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto)
# check for expected elements
if expectedEvent and expectedEvent not in (
"No change", "N.A."):
if len(modelVersReport.xmlDocument.getElementsByTagNameNS('*',expectedEvent)) == 0:
modelTestcases.error(
"Generated test case {0} missing expected event {1}".format(
reportName,
expectedEvent),
"wrn", "missingEvent")
modelVersReport.close([])
for i,testcaseElt in enumerate(testcaseElements):
variationElement = XmlUtil.addChild(testcaseElt, conformanceNS, "variation",
attributes=("id", "_{0:02n}".format(variationID)))
XmlUtil.addChild(variationElement, conformanceNS, "name", text=intention)
dataElement = XmlUtil.addChild(variationElement, conformanceNS, "data")
for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")):
for schemaURI in schemaURIs.split(","):
XmlUtil.addChild(dataElement, conformanceNS, "schema",
attributes=((("dts",dtsAttr),) +
((("readMeFirst","true"),) if i == 0 else ())),
text=os.path.basename(schemaURI.strip()))
resultElement = XmlUtil.addChild(variationElement, conformanceNS, "result")
XmlUtil.addChild(resultElement if i == 0 else dataElement,
conformanceNS, "versioningReport",
attributes=(("readMeFirst","true") if i == 1 else ()),
text="report/" + reportName)
variationID += 1
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i,indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def loadFromExcel(cntlr, excelFile):
from openpyxl import load_workbook
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
startedAt = time.time()
if os.path.isabs(excelFile):
# allow relative filenames to loading directory
priorCWD = os.getcwd()
os.chdir(os.path.dirname(excelFile))
else:
priorCWD = None
importExcelBook = load_workbook(excelFile, read_only=True, data_only=True)
sheetNames = importExcelBook.get_sheet_names()
if "DTS" in sheetNames:
dtsWs = importExcelBook["DTS"]
elif "Sheet2" in sheetNames:
dtsWs = importExcelBook["Sheet2"]
else:
dtsWs = None
imports = {"xbrli": ( ("namespace", XbrlConst.xbrli),
("schemaLocation", "http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd") )} # xml of imports
importXmlns = {}
linkbaseRefs = []
labelLinkbases = []
hasPreLB = hasCalLB = hasDefLB = False
# xxxLB structure [ (elr1, def1, "_ELR_", [roots]), (elr2, def2, "_ELR_", [rootw]) ...]
# roots = (rootHref, None, "_root_", [children])
# children = (childPrefix, childName, arcrole, [grandChildren])
preLB = []
defLB = []
calLB = []
def lbDepthList(lbStruct, depth, parentList=None):
if depth == topDepth:
if len(lbStruct) > 0:
return lbStruct[-1].childStruct
else:
cntlr.addToLog("Depth error, Excel row: {excelRow}"
.format(excelRow=iRow),
messageCode="importExcel:depth")
return None
return lbDepthList(lbStruct[-1].childStruct, depth-1, list)
extensionElements = {}
extensionRoles = {} # key is roleURI, value is role definition
extensionLabels = {} # key = (prefix, name, lang, role), value = label text
importSheetName = None
skipRows = [] # [(from,to),(from,to)] row number starting at 1
def extensionHref(prefix, name):
if prefix == extensionSchemaPrefix:
filename = extensionSchemaFilename
elif prefix in imports:
filename = imports[prefix][1][1]
else:
return None
return "{0}#{1}_{2}".format(filename, prefix, name)
isUSGAAP = False
for iRow, row in enumerate(dtsWs.rows if dtsWs else ()):
try:
if (len(row) < 1 or row[0].value is None): # skip if col 1 is empty
continue
action = filetype = prefix = filename = namespaceURI = None
if len(row) > 0: action = row[0].value
if len(row) > 1: filetype = row[1].value
if len(row) > 2: prefix = row[2].value
if len(row) > 3: filename = row[3].value
if len(row) > 4: namespaceURI = row[4].value
lbType = lang = None
if action == "import":
imports[prefix] = ( ("namespace", namespaceURI), ("schemaLocation", filename) )
importXmlns[prefix] = namespaceURI
if re.match(r"http://[^/]+/us-gaap/", namespaceURI):
isUSGAAP = True
elif action == "extension":
if filetype == "schema":
extensionSchemaPrefix = prefix
extensionSchemaFilename = filename
extensionSchemaNamespaceURI = namespaceURI
elif filetype == "linkbase":
typeLang = prefix.split()
if len(typeLang) > 0:
lbType = typeLang[0]
else:
lbType = "unknown"
if len(typeLang) > 1:
lang = typeLang[1]
else:
lang = "en"
if lbType == "label":
labelLinkbases.append((lang, filename))
elif lbType == "presentation":
hasPreLB = True
elif lbType == "definition":
hasDefLB = True
elif lbType == "calculation":
hasCalLB = True
linkbaseRefs.append( (lbType, filename) )
elif filetype == "role" and namespaceURI:
extensionRoles[namespaceURI] = filename
elif action == "worksheet" and filename:
importSheetName = filename
elif action == "colheader" and filename and namespaceURI:
importColHeaderMap[filename].append(namespaceURI)
elif action == "skip rows":
if filename:
fromRow, _sep, toRow = filename.partition("-")
try:
skipRows.append((int(fromRow), int(toRow) if toRow else int(fromRow)))
except (ValueError, TypeError):
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:skip rows")
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:exception")
if not isUSGAAP: # need extra namespace declaration
importXmlns["iod"] = "http://disclosure.edinet-fsa.go.jp/taxonomy/common/2013-03-31/iod"
# find column headers row
headerCols = {}
hasLinkroleSeparateRow = True
headerRows = set()
topDepth = 999999
if importSheetName and importSheetName in sheetNames:
conceptsWs = importExcelBook[importSheetName]
elif "Concepts" in sheetNames:
conceptsWs = importExcelBook["Concepts"]
elif "Sheet1" in sheetNames:
conceptsWs = importExcelBook["Sheet1"]
else:
conceptsWs = None
def setHeaderCols(row):
headerCols.clear()
for iCol, colCell in enumerate(row):
v = colCell.value
if v in importColHeaderMap:
for hdr in importColHeaderMap[v]:
if hdr in importColumnHeaders:
headerCols[importColumnHeaders[hdr]] = iCol
elif v in importColumnHeaders:
headerCols[importColumnHeaders[v]] = iCol
# find out which rows are header rows
for iRow, row in enumerate(conceptsWs.rows if conceptsWs else ()):
if any(fromRow <= iRow+1 <= toRow for fromRow,toRow in skipRows):
continue
for iCol, colCell in enumerate(row):
setHeaderCols(row)
if all(colName in headerCols
for colName in ("name", "type", "depth")): # must have these to be a header col
# it's a header col
headerRows.add(iRow+1)
if 'linkrole' in headerCols:
hasLinkroleSeparateRow = False
headerCols.clear()
def cellHasValue(row, header, _type):
if header in headerCols:
iCol = headerCols[header]
return iCol < len(row) and isinstance(row[iCol].value, _type)
return False
def cellValue(row, header, strip=False, nameChars=False):
if header in headerCols:
iCol = headerCols[header]
if iCol < len(row):
v = row[iCol].value
if strip and isinstance(v, str):
v = v.strip()
if nameChars and isinstance(v, str):
v = ''.join(c for c in v if c.isalnum() or c in ('.', '_', '-'))
return v
return None
def checkImport(qname):
prefix, sep, localName = qname.partition(":")
if sep:
if prefix not in imports:
if prefix == "xbrldt":
imports["xbrldt"] = ("namespace", XbrlConst.xbrldt), ("schemaLocation", "http://www.xbrl.org/2005/xbrldt-2005.xsd")
elif prefix == "nonnum":
imports["nonnum"] = ("namespace", "http://www.xbrl.org/dtr/type/non-numeric"), ("schemaLocation", "http://www.xbrl.org/dtr/type/nonNumeric-2009-12-16.xsd")
else:
cntlr.addToLog("Warning: prefix schema file is not imported for: {qname}"
.format(qname=qname),
messageCode="importExcel:warning")
# find top depth
for iRow, row in enumerate(conceptsWs.rows if conceptsWs else ()):
if (iRow + 1) in headerRows:
setHeaderCols(row)
elif not (hasLinkroleSeparateRow and (iRow + 1) in headerRows) and 'depth' in headerCols:
depth = cellValue(row, 'depth')
if isinstance(depth, int) and depth < topDepth:
topDepth = depth
# find header rows
currentELR = currentELRdefinition = None
for iRow, row in enumerate(conceptsWs.rows if conceptsWs else ()):
useLabels = False
if any(fromRow <= iRow+1 <= toRow for fromRow,toRow in skipRows):
continue
if (all(col is None for col in row) or
all(isinstance(row[i].value, str) and row[i].value.strip() == "n/a"
for i in (headerCols.get("name"), headerCols.get("type"), headerCols.get("value"))
if i)):
continue # skip blank row
try:
isHeaderRow = (iRow + 1) in headerRows
isELRrow = hasLinkroleSeparateRow and (iRow + 2) in headerRows
if isHeaderRow:
setHeaderCols(row)
elif isELRrow:
currentELR = currentELRdefinition = None
for colCell in row:
v = str(colCell.value or '')
if v.startswith("http://"):
currentELR = v
elif not currentELRdefinition and v.endswith("　科目一覧"):
currentELRdefinition = v[0:-5]
elif not currentELRdefinition:
currentELRdefinition = v
if currentELR or currentELRdefinition:
if hasPreLB:
preLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasDefLB:
defLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasCalLB:
calLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
calRels = set() # prevent duplications when same rel in different parts of tree
elif headerCols:
if "linkrole" in headerCols and cellHasValue(row, 'linkrole', str):
v = cellValue(row, 'linkrole', strip=True)
_trialELR = _trialELRdefinition = None
if v.startswith("http://"):
_trialELR = v
elif v.endswith("　科目一覧"):
_trialELRdefinition = v[0:-5]
else:
_trialELRdefinition = v
if (_trialELR and _trialELR != currentELR) or (_trialELRdefinition and _trialELRdefinition != currentELRdefinition):
currentELR = _trialELR
currentELRdefinition = _trialELRdefinition
if currentELR or currentELRdefinition:
if hasPreLB:
preLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasDefLB:
defLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasCalLB:
calLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
calRels = set() # prevent duplications when same rel in different parts of tree
prefix = cellValue(row, 'prefix', nameChars=True) or extensionSchemaPrefix
if cellHasValue(row, 'name', str):
name = cellValue(row, 'name', nameChars=True)
else:
name = None
if cellHasValue(row, 'depth', int):
depth = cellValue(row, 'depth')
else:
depth = None
if (not prefix or prefix == extensionSchemaPrefix) and name not in extensionElements and name:
# elements row
eltType = cellValue(row, 'type')
eltTypePrefix = cellValue(row, 'typePrefix')
if not eltType:
eltType = 'xbrli:stringItemType'
elif eltTypePrefix and ':' not in eltType:
eltType = eltTypePrefix + ':' + eltType
elif ':' not in eltType and eltType.endswith("ItemType"):
eltType = 'xbrli:' + eltType
subsGrp = cellValue(row, 'substitutionGroup') or 'xbrli:item'
abstract = cellValue(row, 'abstract') or (cellValue(row, 'abstractMarker') is not None)
nillable = cellValue(row, 'nillable')
balance = cellValue(row, 'balance')
periodType = cellValue(row, 'periodType')
newElt = [ ("name", name), ("id", (prefix or "") + "_" + name) ]
if eltType:
newElt.append( ("type", eltType) )
checkImport(eltType)
if subsGrp:
newElt.append( ("substitutionGroup", subsGrp) )
checkImport(subsGrp)
if abstract or subsGrp in ("xbrldt:hypercubeItem", "xbrldt:dimensionItem"):
newElt.append( ("abstract", abstract or "true") )
if nillable:
newElt.append( ("nillable", nillable) )
if balance:
newElt.append( ("{http://www.xbrl.org/2003/instance}balance", balance) )
if periodType:
newElt.append( ("{http://www.xbrl.org/2003/instance}periodType", periodType) )
extensionElements[name] = newElt
useLabels = True
if depth is not None:
if hasPreLB:
entryList = lbDepthList(preLB, depth)
preferredLabel = cellValue(row, 'preferredLabel')
if preferredLabel and not preferredLabel.startswith("http://"):
preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel
if entryList is not None:
if depth == topDepth:
entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) )
else:
entryList.append( LBentry(prefix=prefix, name=name, arcrole=XbrlConst.parentChild,
role=preferredLabel) )
if hasDefLB:
entryList = lbDepthList(defLB, depth)
if entryList is not None:
if depth == topDepth:
entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) )
else:
if (not preferredLabel or # prevent start/end labels from causing duplicate dim-mem relationships
not any(lbEntry.prefix == prefix and lbEntry.name == name
for lbEntry in entryList)):
entryList.append( LBentry(prefix=prefix, name=name, arcrole="_dimensions_") )
if hasCalLB:
calcParents = cellValue(row, 'calculationParent').split()
calcWeights = (str(cellValue(row, 'calculationWeight')) or '').split() # may be float or string
if calcParents and calcWeights:
# may be multiple parents split by whitespace
for i, calcParent in enumerate(calcParents):
calcWeight = calcWeights[i] if i < len(calcWeights) else calcWeights[-1]
calcParentPrefix, sep, calcParentName = calcParent.partition(":")
entryList = lbDepthList(calLB, topDepth)
if entryList is not None:
calRel = (calcParentPrefix, calcParentName, prefix, name)
if calRel not in calRels:
entryList.append( LBentry(prefix=calcParentPrefix, name=calcParentName, isRoot=True, childStruct=
[LBentry(prefix=prefix, name=name, arcrole=XbrlConst.summationItem, weight=calcWeight )]) )
calRels.add(calRel)
else:
pass
# accumulate extension labels
if useLabels:
prefix = cellValue(row, 'prefix', nameChars=True) or extensionSchemaPrefix
name = cellValue(row, 'name', nameChars=True)
if name is not None:
preferredLabel = cellValue(row, 'preferredLabel')
if preferredLabel and not preferredLabel.startswith("http://"):
preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel
for colItem, iCol in headerCols.items():
if isinstance(colItem, tuple):
colItemType = colItem[0]
role = colItem[1]
lang = colItem[2]
cell = row[iCol]
if cell.value is None:
values = ()
elif colItemType == "label":
values = (cell.value,)
elif colItemType == "labels":
values = cell.value.split('\n')
else:
values = ()
if preferredLabel and "indented" in colItem: # indented column sets preferredLabel if any
role = preferredLabel
for value in values:
extensionLabels[prefix, name, lang, role] = value.strip()
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:exception")
if isUSGAAP and hasDefLB:
# move line items above table
def fixUsggapTableDims(lvl1Struct):
foundLineItems = False
for i1, lvl1Entry in enumerate(lvl1Struct):
for i2, lvl2Entry in enumerate(lvl1Entry.childStruct):
for i3, lvl3Entry in enumerate(lvl2Entry.childStruct):
if lvl3Entry.name.endswith("LineItems") and lvl2Entry.name.endswith("Table"):
foundLineItems = True
break
if foundLineItems:
break
else:
fixUsggapTableDims(lvl1Entry.childStruct)
if foundLineItems:
lvl1Struct.insert(i1 + 1, LBentry(prefix=lvl3Entry.prefix, name=lvl3Entry.name, arcrole=lvl1Entry.arcrole, childStruct=lvl3Entry.childStruct)) # must keep lvl1Rel if it is __root__
lvl3Entry.childStruct.insert(0, lvl2Entry)
if lvl1Entry.name.endswith("Abstract"):
del lvl1Struct[i1]
if i3 < len(lvl2Entry.childStruct):
del lvl2Entry.childStruct[i3]
pass
fixUsggapTableDims(defLB)
dts = cntlr.modelManager.create(newDocumentType=ModelDocument.Type.SCHEMA,
url=extensionSchemaFilename,
isEntry=True,
base='', # block pathname from becomming absolute
initialXml='''
<schema xmlns="http://www.w3.org/2001/XMLSchema"
targetNamespace="{targetNamespace}"
attributeFormDefault="unqualified"
elementFormDefault="qualified"
xmlns:xsd="http://www.w3.org/2001/XMLSchema"
xmlns:{extensionPrefix}="{targetNamespace}"
{importXmlns}
xmlns:nonnum="http://www.xbrl.org/dtr/type/non-numeric"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xbrli="http://www.xbrl.org/2003/instance"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrldt="http://xbrl.org/2005/xbrldt"/>
'''.format(targetNamespace=extensionSchemaNamespaceURI,
extensionPrefix=extensionSchemaPrefix,
importXmlns=''.join('xmlns:{0}="{1}"\n'.format(prefix, namespaceURI)
for prefix, namespaceURI in importXmlns.items())
)
)
dtsSchemaDocument = dts.modelDocument
dtsSchemaDocument.inDTS = True # entry document always in DTS
dtsSchemaDocument.targetNamespace = extensionSchemaNamespaceURI # not set until schemaDiscover too late otherwise
schemaElt = dtsSchemaDocument.xmlRootElement
#foreach linkbase
annotationElt = XmlUtil.addChild(schemaElt, XbrlConst.xsd, "annotation")
appinfoElt = XmlUtil.addChild(annotationElt, XbrlConst.xsd, "appinfo")
# add linkbaseRefs
appinfoElt = XmlUtil.descendant(schemaElt, XbrlConst.xsd, "appinfo")
# don't yet add linkbase refs, want to process imports first to get roleType definitions
# add imports
for importAttributes in sorted(imports.values()):
XmlUtil.addChild(schemaElt,
XbrlConst.xsd, "import",
attributes=importAttributes)
# add elements
for eltName, eltAttrs in sorted(extensionElements.items(), key=lambda item: item[0]):
XmlUtil.addChild(schemaElt,
XbrlConst.xsd, "element",
attributes=eltAttrs)
# add role definitions (for discovery)
for roleURI, roleDefinition in extensionRoles.items():
roleElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "roleType",
attributes=(("roleURI", roleURI),
("id", "roleType_" + roleURI.rpartition("/")[2])))
if roleDefinition:
XmlUtil.addChild(roleElt, XbrlConst.link, "definition", text=roleDefinition)
if hasPreLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:presentationLink")
if hasDefLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:definitionLink")
if hasCalLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:calculationLink")
dtsSchemaDocument.schemaDiscover(schemaElt, False, extensionSchemaNamespaceURI)
def addLinkbaseRef(lbType, lbFilename, lbDoc):
role = "http://www.xbrl.org/2003/role/{0}LinkbaseRef".format(lbType)
lbRefElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "linkbaseRef",
attributes=(("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", lbFilename),
("{http://www.w3.org/1999/xlink}role", role),
("{http://www.w3.org/1999/xlink}arcrole", "http://www.w3.org/1999/xlink/properties/linkbase"),
))
dtsSchemaDocument.referencesDocument[lbDoc] = ModelDocumentReference("href", lbRefElt)
# label linkbase
for lang, filename in labelLinkbases:
lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base="", initialXml="""
<linkbase
xmlns="http://www.xbrl.org/2003/linkbase"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase
http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance"/>
""")
lbDoc.inDTS = True
addLinkbaseRef("label", filename, lbDoc)
lbElt = lbDoc.xmlRootElement
linkElt = XmlUtil.addChild(lbElt,
XbrlConst.link, "labelLink",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/link")))
firstLinkElt = linkElt
locs = set()
roleRefs = set()
for labelKey, text in extensionLabels.items():
prefix, name, labelLang, role = labelKey
if lang == labelLang:
locLabel = prefix + "_" + name
if locLabel not in locs:
locs.add(locLabel)
XmlUtil.addChild(linkElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", extensionHref(prefix, name)),
("{http://www.w3.org/1999/xlink}label", locLabel)))
XmlUtil.addChild(linkElt,
XbrlConst.link, "labelArc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", "http://www.xbrl.org/2003/arcrole/concept-label"),
("{http://www.w3.org/1999/xlink}from", locLabel),
("{http://www.w3.org/1999/xlink}to", "label_" + locLabel),
("order", 1.0)))
XmlUtil.addChild(linkElt,
XbrlConst.link, "label",
attributes=(("{http://www.w3.org/1999/xlink}type", "resource"),
("{http://www.w3.org/1999/xlink}label", "label_" + locLabel),
("{http://www.w3.org/1999/xlink}role", role),
("{http://www.w3.org/XML/1998/namespace}lang", lang)),
text=text)
if role:
if role in dts.roleTypes:
roleType = dts.roleTypes[role][0]
roleRefs.add(("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id))
elif role.startswith("http://www.xbrl.org/2009/role/negated"):
roleRefs.add(("roleRef", role, "http://www.xbrl.org/lrr/role/negated-2009-12-16.xsd#" + role.rpartition("/")[2]))
# add arcrole references
for roleref, roleURI, href in roleRefs:
XmlUtil.addChild(lbElt,
XbrlConst.link, roleref,
attributes=(("arcroleURI" if roleref == "arcroleRef" else "roleURI", roleURI),
("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", href)),
beforeSibling=firstLinkElt)
lbDoc.linkbaseDiscover(lbElt)
def hrefConcept(prefix, name):
qn = schemaElt.prefixedNameQname(prefix + ":" + name)
if qn in dts.qnameConcepts:
return dts.qnameConcepts[qn]
return None
def lbTreeWalk(lbType, parentElt, lbStruct, roleRefs, locs=None, arcsFromTo=None, fromPrefix=None, fromName=None):
order = 1.0
for lbEntry in lbStruct:
if lbEntry.isELR:
role = "unspecified"
if lbEntry.role and lbEntry.role.startswith("http://"): # have a role specified
role = lbEntry.role
elif lbEntry.name: #may be a definition
for linkroleUri, modelRoleTypes in dts.roleTypes.items():
definition = modelRoleTypes[0].definition
if lbEntry.name == definition:
role = linkroleUri
break
if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef
roleType = modelRoleTypes[0]
roleRefs.add(("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id))
linkElt = XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Link",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", role)))
locs = set()
arcsFromTo = set()
lbTreeWalk(lbType, linkElt, lbEntry.childStruct, roleRefs, locs, arcsFromTo)
else:
toPrefix = lbEntry.prefix
toName = lbEntry.name
toHref = extensionHref(toPrefix, toName)
toLabel = toPrefix + "_" + toName
toLabelAlt = None
if not lbEntry.isRoot:
fromLabel = fromPrefix + "_" + fromName
if (fromLabel, toLabel) in arcsFromTo:
# need extra loc to prevent arc from/to duplication in ELR
for i in range(1, 1000):
toLabelAlt = "{}_{}".format(toLabel, i)
if (fromLabel, toLabelAlt) not in arcsFromTo:
toLabel = toLabelAlt
break
if toHref not in locs or toLabelAlt:
XmlUtil.addChild(parentElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", toHref),
("{http://www.w3.org/1999/xlink}label", toLabel)))
locs.add(toHref)
if not lbEntry.isRoot:
arcsFromTo.add( (fromLabel, toLabel) )
if lbType == "calculation" and lbEntry.weight is not None:
otherAttrs = ( ("weight", lbEntry.weight), )
elif lbType == "presentation" and lbEntry.role:
otherAttrs = ( ("preferredLabel", lbEntry.role), )
if lbEntry.role and lbEntry.role in dts.roleTypes:
roleType = dts.roleTypes[lbEntry.role][0]
roleRefs.add(("roleRef", lbEntry.role, roleType.modelDocument.uri + "#" + roleType.id))
else:
otherAttrs = ( )
if lbEntry.arcrole == "_dimensions_": # pick proper consecutive arcrole
fromConcept = hrefConcept(fromPrefix, fromName)
toConcept = hrefConcept(toPrefix, toName)
if toConcept is not None and toConcept.isHypercubeItem:
arcrole = XbrlConst.all
otherAttrs += ( (XbrlConst.qnXbrldtContextElement, "segment"), )
elif toConcept is not None and toConcept.isDimensionItem:
arcrole = XbrlConst.hypercubeDimension
elif fromConcept is not None and fromConcept.isDimensionItem:
arcrole = XbrlConst.dimensionDomain
else:
arcrole = XbrlConst.domainMember
else:
arcrole = lbEntry.arcrole
XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Arc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", arcrole),
("{http://www.w3.org/1999/xlink}from", fromLabel),
("{http://www.w3.org/1999/xlink}to", toLabel),
("order", order)) + otherAttrs )
order += 1.0
if lbType != "calculation" or lbEntry.isRoot:
lbTreeWalk(lbType, parentElt, lbEntry.childStruct, roleRefs, locs, arcsFromTo, toPrefix, toName)
for hasLB, lbType, lbLB in ((hasPreLB, "presentation", preLB),
(hasDefLB, "definition", defLB),
(hasCalLB, "calculation", calLB)):
if hasLB:
for lbRefType, filename in linkbaseRefs:
if lbType == lbRefType:
# output presentation linkbase
lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base='', initialXml="""
<linkbase
xmlns="http://www.xbrl.org/2003/linkbase"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase
http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance"/>
""")
lbDoc.inDTS = True
addLinkbaseRef(lbRefType, filename, lbDoc)
lbElt = lbDoc.xmlRootElement
roleRefs = set()
if lbType == "definition":
roleRefs.update((("arcroleRef", XbrlConst.all, "http://www.xbrl.org/2005/xbrldt-2005.xsd#all"),
("arcroleRef", XbrlConst.dimensionDefault, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-default"),
("arcroleRef", XbrlConst.dimensionDomain, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-domain"),
("arcroleRef", XbrlConst.domainMember, "http://www.xbrl.org/2005/xbrldt-2005.xsd#domain-member"),
("arcroleRef", XbrlConst.hypercubeDimension, "http://www.xbrl.org/2005/xbrldt-2005.xsd#hypercube-dimension")))
lbTreeWalk(lbType, lbElt, lbLB, roleRefs)
firstLinkElt = None
for firstLinkElt in lbElt.iterchildren():
break
# add arcrole references
for roleref, roleURI, href in roleRefs:
XmlUtil.addChild(lbElt,
XbrlConst.link, roleref,
attributes=(("arcroleURI" if roleref == "arcroleRef" else "roleURI", roleURI),
("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", href)),
beforeSibling=firstLinkElt)
lbDoc.linkbaseDiscover(lbElt)
break
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
if priorCWD:
os.chdir(priorCWD) # restore prior current working directory
return dts
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s"),
modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected)
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
ModelDocument.Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = ModelDocument.load(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
else: # not a multi-schemaRef versioning report
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource)
if modelXbrl.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelXbrl.close()
self.determineNotLoadedTestStatus(modelTestcaseVariation)
elif resultIsVersioningReport:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
modelXbrl.close()
elif testcase.type == ModelDocument.Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
parameters = modelTestcaseVariation.parameters.copy()
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
if modelTestcaseVariation.resultIsTable:
RenderingEvaluator.init(modelXbrl)
self.instValidator.validate(modelXbrl, parameters)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource)
if infoset.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl,
modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement),
diffToFile=True) # false to save infoset files
self.determineTestStatus(modelTestcaseVariation, modelXbrl) # include infoset errors in status
self.instValidator.close()
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("formula:resultFactCounts",
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts))
else:
for fact in expectedInstance.facts:
unmatchedFactsStack = []
if formulaOutputInstance.matchFact(fact, unmatchedFactsStack) is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = fact
formulaOutputInstance.error("formula:expectedFactMissing",
_("Formula output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname)
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance)
formulaOutputInstance.close()
del formulaOutputInstance
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def reload(self,nextaction,reloadCache=False):
from arelle import ModelDocument
self.init(keepViews=True)
self.modelDocument = ModelDocument.load(self, self.fileSource.url, isEntry=True, reloadCache=reloadCache)
self.modelManager.showStatus(_("xbrl loading finished, {0}...").format(nextaction),5000)
self.modelManager.reloadViews(self)
def runFromExcel(self, options):
#testGenFileName = options.excelfilename
testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls"
testGenDir = os.path.dirname(testGenFileName)
schemaDir = os.path.dirname(testGenDir) + os.sep + "schema"
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
LogHandler(self) # start logger
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt'
modelTestcases = ModelXbrl.create(self.modelManager,
url=testGenFileName,
isEntry=True)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(
_("[info] xls loaded in {0:.2} secs at {1}").format(
time.time() - startedAt, timeNow))
# start index file
indexFiles = [
testGenDir + os.sep + 'creation-testcases-index.xml',
testGenDir + os.sep + 'consumption-testcases-index.xml'
]
indexDocs = []
testcasesElements = []
for purpose in ("Creation", "Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 {0} Tests" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today))
doc = etree.parse(file)
file.close()
indexDocs.append(doc)
testcasesElements.append(doc.getroot())
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
try:
row = testcaseIndexSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY
or # must have directory
row[1].ctype == xlrd.XL_CELL_EMPTY or # from
row[2].ctype == xlrd.XL_CELL_EMPTY): # to
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
overrideReport = row[3].value
description = row[4].value
if description is None or len(description) == 0:
continue # test not ready to run
assignment = row[5].value
expectedEvents = row[
6].value # comma space separated if multiple
note = row[7].value
useCase = row[8].value
base = os.path.join(os.path.dirname(testGenFileName),
testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and assignment.lower() not in (
"n.a.", "error") and expectedEvents != "N.A.":
modelDTSfrom = modelDTSto = None
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"),
True), (uriTo,
_("loading to DTS"),
False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager,
URIs,
msg,
base=base)
else:
modelDTS = ModelXbrl.create(
self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(
URIs.replace(", ", "_") + ".dts", base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS,
uri.strip(),
base=base)
if doc is not None:
DTSdoc.referencesDocument[
doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom is not None and modelDTSto is not None:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace(
"from.xsd", "report.xml")
reportFile = reportDir + "out" + os.sep + reportName
#reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile, base)
testcasesDir = os.path.dirname(
os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i, testcaseFile in enumerate(
testcaseFiles):
with open(testcaseFile,
"w",
encoding="utf-8") as fh:
XmlUtil.writexml(fh,
testcaseDocs[i],
encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [
testcasesDir + os.sep + testcaseName +
"-creation-testcase.xml",
testcasesDir + os.sep + testcaseName +
"-consumption-testcase.xml"
]
for i, testcaseFile in enumerate(testcaseFiles):
etree.SubElement(
testcasesElements[i],
"testcase",
attrib={
"uri":
testcaseFile[len(testGenDir) +
1:].replace("\\", "/")
})
# start testcase file
testcaseDocs = []
testcaseElements = []
testcaseNumber = testcaseName[0:4]
if testcaseNumber.isnumeric():
testcaseNumberElement = "<number>{0}</number>".format(
testcaseNumber)
testcaseName = testcaseName[5:]
else:
testcaseNumberElement = ""
testDirSegments = testDir.split('/')
if len(testDirSegments
) >= 2 and '-' in testDirSegments[1]:
testedModule = testDirSegments[1][
testDirSegments[1].index('-') + 1:]
else:
testedModule = ''
for purpose in ("Creation", "Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'{0}'
'<name>{1}</name>'
# '<description>{0}</description>'
'<reference>'
'{2}'
'{3}'
'</reference>'
'</testcase>'.format(
testcaseNumberElement, testcaseName,
'<name>{0}</name>'.format(testedModule)
if testedModule else '',
'<id>{0}</id>'.format(useCase)
if useCase else ''))
doc = etree.parse(file)
file.close()
testcaseDocs.append(doc)
testcaseElements.append(doc.getroot())
priorTestcasesDir = testcasesDir
variationSeq = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(
modelTestcases)
modelVersReport.diffDTSes(reportFullPath,
modelDTSfrom,
modelDTSto,
assignment=assignment,
schemaDir=schemaDir)
# check for expected elements
if expectedEvents:
for expectedEvent in expectedEvents.split(","):
if expectedEvent not in ("No change", "N.A."):
prefix, sep, localName = expectedEvent.partition(
':')
if sep and len(
modelVersReport.xmlDocument.
findall('//{{{0}}}{1}'.format(
XbrlConst.verPrefixNS.get(
prefix), localName))) == 0:
modelTestcases.warning(
"warning",
"Generated test case %(reportName)s missing expected event %(event)s",
reportName=reportName,
event=expectedEvent)
modelVersReport.close()
uriFromParts = uriFrom.split('_')
if len(uriFromParts) >= 2:
variationId = uriFromParts[1]
else:
variationId = "_{0:02n}".format(variationSeq)
for i, testcaseElt in enumerate(testcaseElements):
variationElement = etree.SubElement(
testcaseElt,
"{http://xbrl.org/2008/conformance}variation",
attrib={"id": variationId})
nameElement = etree.SubElement(
variationElement,
"{http://xbrl.org/2008/conformance}description"
)
nameElement.text = description
''' (removed per RH 2011/10/04
if note:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
paramElement.text = "Note: " + note
if useCase:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference")
paramElement.set("specification", "versioning-requirements")
paramElement.set("useCase", useCase)
'''
dataElement = etree.SubElement(
variationElement,
"{http://xbrl.org/2008/conformance}data")
if i == 0: # result is report
if expectedEvents:
paramElement = etree.SubElement(
dataElement,
"{http://xbrl.org/2008/conformance}parameter",
attrib={
"name":
"expectedEvent",
"value":
expectedEvents.replace(',', ' ')
},
nsmap={
"conf":
"http://xbrl.org/2008/conformance",
None: ""
})
if assignment:
paramElement = etree.SubElement(
dataElement,
"{http://xbrl.org/2008/conformance}parameter",
attrib={
"name": "assignment",
"value": assignment
},
nsmap={
"conf":
"http://xbrl.org/2008/conformance",
None: ""
})
for schemaURIs, dtsAttr in ((uriFrom, "from"),
(uriTo, "to")):
for schemaURI in schemaURIs.split(","):
schemaElement = etree.SubElement(
dataElement,
"{http://xbrl.org/2008/conformance}schema"
)
schemaElement.set("dts", dtsAttr)
if i == 0:
schemaElement.set(
"readMeFirst", "true")
schemaElement.text = os.path.basename(
schemaURI.strip())
resultElement = etree.SubElement(
variationElement,
"{http://xbrl.org/2008/conformance}result")
reportElement = etree.SubElement(
resultElement if i == 0 else dataElement,
"{http://xbrl.org/2008/conformance}versioningReport"
)
if i == 1:
reportElement.set("readMeFirst", "true")
reportElement.text = "report/" + reportName
variationSeq += 1
except Exception as err:
modelTestcases.error(
"exception",
_("Exception: %(error)s, Excel row: %(excelRow)s"),
error=err,
excelRow=iRow,
exc_info=True)
# add tests-error-code index files to consumption
for testcaseFile in self.testcaseFiles(testGenDir + os.sep +
"tests-error-code"):
etree.SubElement(testcasesElements[1],
"testcase",
attrib={
"uri":
testcaseFile[len(testGenDir) + 1:].replace(
"\\", "/")
})
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i, testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i, indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def runFromExcel(self, options):
#testGenFileName = options.excelfilename
testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls"
testGenDir = os.path.dirname(testGenFileName)
schemaDir = os.path.dirname(testGenDir) + os.sep + "schema"
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
LogHandler(self) # start logger
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt'
modelTestcases = ModelXbrl.create(self.modelManager, url=testGenFileName, isEntry=True)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow))
# start index file
indexFiles = [testGenDir + os.sep + 'creation-testcases-index.xml',
testGenDir + os.sep + 'consumption-testcases-index.xml']
indexDocs = []
testcasesElements = []
for purpose in ("Creation","Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 {0} Tests" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today)
)
doc = etree.parse(file)
file.close()
indexDocs.append(doc)
testcasesElements.append(doc.getroot())
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
try:
row = testcaseIndexSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY or # must have directory
row[1].ctype == xlrd.XL_CELL_EMPTY or # from
row[2].ctype == xlrd.XL_CELL_EMPTY): # to
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
overrideReport = row[3].value
description = row[4].value
if description is None or len(description) == 0:
continue # test not ready to run
assignment = row[5].value
expectedEvents = row[6].value # comma space separated if multiple
note = row[7].value
useCase = row[8].value
base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and assignment.lower() not in ("n.a.", "error") and expectedEvents != "N.A.":
modelDTSfrom = modelDTSto = None
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base)
else:
modelDTS = ModelXbrl.create(self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts",
base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS, uri.strip(), base=base)
if doc is not None:
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom is not None and modelDTSto is not None:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace("from.xsd","report.xml")
reportFile = reportDir + "out" + os.sep + reportName
#reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile,
base)
testcasesDir = os.path.dirname(os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml",
testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"]
for i,testcaseFile in enumerate(testcaseFiles):
etree.SubElement(testcasesElements[i], "testcase",
attrib={"uri":
testcaseFile[len(testGenDir)+1:].replace("\\","/")} )
# start testcase file
testcaseDocs = []
testcaseElements = []
testcaseNumber = testcaseName[0:4]
if testcaseNumber.isnumeric():
testcaseNumberElement = "<number>{0}</number>".format(testcaseNumber)
testcaseName = testcaseName[5:]
else:
testcaseNumberElement = ""
testDirSegments = testDir.split('/')
if len(testDirSegments) >= 2 and '-' in testDirSegments[1]:
testedModule = testDirSegments[1][testDirSegments[1].index('-') + 1:]
else:
testedModule = ''
for purpose in ("Creation","Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'{0}'
'<name>{1}</name>'
# '<description>{0}</description>'
'<reference>'
'{2}'
'{3}'
'</reference>'
'</testcase>'.format(testcaseNumberElement,
testcaseName,
'<name>{0}</name>'.format(testedModule) if testedModule else '',
'<id>{0}</id>'.format(useCase) if useCase else '')
)
doc = etree.parse(file)
file.close()
testcaseDocs.append(doc)
testcaseElements.append(doc.getroot())
priorTestcasesDir = testcasesDir
variationSeq = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(modelTestcases)
modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto,
assignment=assignment,
schemaDir=schemaDir)
# check for expected elements
if expectedEvents:
for expectedEvent in expectedEvents.split(","):
if expectedEvent not in ("No change", "N.A."):
prefix, sep, localName = expectedEvent.partition(':')
if sep and len(modelVersReport.xmlDocument.findall(
'//{{{0}}}{1}'.format(
XbrlConst.verPrefixNS.get(prefix),
localName))) == 0:
modelTestcases.warning("warning",
"Generated test case %(reportName)s missing expected event %(event)s",
reportName=reportName,
event=expectedEvent)
modelVersReport.close()
uriFromParts = uriFrom.split('_')
if len(uriFromParts) >= 2:
variationId = uriFromParts[1]
else:
variationId = "_{0:02n}".format(variationSeq)
for i,testcaseElt in enumerate(testcaseElements):
variationElement = etree.SubElement(testcaseElt, "{http://xbrl.org/2008/conformance}variation",
attrib={"id": variationId})
nameElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
nameElement.text = description
''' (removed per RH 2011/10/04
if note:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
paramElement.text = "Note: " + note
if useCase:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference")
paramElement.set("specification", "versioning-requirements")
paramElement.set("useCase", useCase)
'''
dataElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}data")
if i == 0: # result is report
if expectedEvents:
paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter",
attrib={"name":"expectedEvent",
"value":expectedEvents.replace(',',' ')},
nsmap={"conf":"http://xbrl.org/2008/conformance",
None:""})
if assignment:
paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter",
attrib={"name":"assignment",
"value":assignment},
nsmap={"conf":"http://xbrl.org/2008/conformance",
None:""})
for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")):
for schemaURI in schemaURIs.split(","):
schemaElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}schema")
schemaElement.set("dts",dtsAttr)
if i == 0:
schemaElement.set("readMeFirst","true")
schemaElement.text=os.path.basename(schemaURI.strip())
resultElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}result")
reportElement = etree.SubElement(resultElement if i == 0 else dataElement,
"{http://xbrl.org/2008/conformance}versioningReport")
if i == 1:
reportElement.set("readMeFirst","true")
reportElement.text = "report/" + reportName
variationSeq += 1
except Exception as err:
modelTestcases.error("exception",
_("Exception: %(error)s, Excel row: %(excelRow)s"),
error=err,
excelRow=iRow,
exc_info=True)
# add tests-error-code index files to consumption
for testcaseFile in self.testcaseFiles(testGenDir + os.sep + "tests-error-code"):
etree.SubElement(testcasesElements[1], "testcase",
attrib={"uri":
testcaseFile[len(testGenDir)+1:].replace("\\","/")} )
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i,indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def run(self, options, sourceZipStream=None):
self.entrypointFile = options.entrypointFile
filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream)
if options.validateEFM:
if options.gfmName:
self.addToLog(_("both --efm and --gfm validation are requested, proceeding with --efm only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("efm")
elif options.gfmName:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select(options.gfmName)
elif options.validateHMRC:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("hmrc")
else:
self.modelManager.disclosureSystem.select(None) # just load ordinary mappings
if options.calcDecimals:
if options.calcPrecision:
self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateInferDecimals = True
self.modelManager.validateCalcLB = True
elif options.calcPrecision:
self.modelManager.validateInferDecimals = False
self.modelManager.validateCalcLB = True
if options.utrValidate:
self.modelManager.validateUtr = True
fo = FormulaOptions()
if options.parameters:
fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value))
for param in options.parameters.split(',')
for key,sep,value in (param.partition('='),) ) )
if options.formulaParamExprResult:
fo.traceParameterExpressionResult = True
if options.formulaParamInputValue:
fo.traceParameterInputValue = True
if options.formulaCallExprSource:
fo.traceCallExpressionSource = True
if options.formulaCallExprCode:
fo.traceCallExpressionCode = True
if options.formulaCallExprEval:
fo.traceCallExpressionEvaluation = True
if options.formulaCallExprResult:
fo.traceCallExpressionResult = True
if options.formulaVarSetExprEval:
fo.traceVariableSetExpressionEvaluation = True
if options.formulaVarSetExprResult:
fo.traceVariableSetExpressionResult = True
if options.formulaAsserResultCounts:
fo.traceAssertionResultCounts = True
if options.formulaFormulaRules:
fo.traceFormulaRules = True
if options.formulaVarsOrder:
fo.traceVariablesOrder = True
if options.formulaVarExpressionSource:
fo.traceVariableExpressionSource = True
if options.formulaVarExpressionCode:
fo.traceVariableExpressionCode = True
if options.formulaVarExpressionEvaluation:
fo.traceVariableExpressionEvaluation = True
if options.formulaVarExpressionResult:
fo.traceVariableExpressionResult = True
if options.formulaVarFilterWinnowing:
fo.traceVariableFilterWinnowing = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
self.modelManager.formulaOptions = fo
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
startedAt = time.time()
modelDiffReport = None
success = True
modelXbrl = None
try:
modelXbrl = self.modelManager.load(filesource, _("views loading"))
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False # loading errors, don't attempt to utilize loaded DTS
if modelXbrl and modelXbrl.modelDocument:
self.addToLog(format_string(self.modelManager.locale,
_("loaded in %.2f secs at %s"),
(time.time() - startedAt, timeNow)),
messageCode="info", file=self.entrypointFile)
if options.importFiles:
for importFile in options.importFiles.split("|"):
ModelDocument.load(modelXbrl, importFile.strip())
self.addToLog(format_string(self.modelManager.locale,
_("imported in %.2f secs at %s"),
(time.time() - startedAt, timeNow)),
messageCode="info", file=importFile)
if modelXbrl.errors:
success = False # loading errors, don't attempt to utilize loaded DTS
else:
success = False
if success and options.diffFile and options.versReportFile:
diffFilesource = FileSource.FileSource(options.diffFile,self)
startedAt = time.time()
modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading"))
if modelXbrl2.errors:
if not options.keepOpen:
modelXbrl2.close()
success = False
else:
self.addToLog(format_string(self.modelManager.locale,
_("diff comparison DTS loaded in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
startedAt = time.time()
modelDiffReport = self.modelManager.compareDTSes(options.versReportFile)
self.addToLog(format_string(self.modelManager.locale,
_("compared in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if success:
try:
if options.validate:
startedAt = time.time()
self.modelManager.validate()
self.addToLog(format_string(self.modelManager.locale,
_("validated in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if (options.testReport and
self.modelManager.modelXbrl.modelDocument.type in
(ModelDocument.Type.TESTCASESINDEX,
ModelDocument.Type.TESTCASE,
ModelDocument.Type.REGISTRY)):
ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport)
if options.DTSFile:
ViewFileDTS.viewDTS(modelXbrl, options.DTSFile)
if options.factsFile:
ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols)
if options.factTableFile:
ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang)
if options.conceptsFile:
ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang)
if options.preFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang)
if options.calFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang)
if options.dimFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang)
if options.formulaeFile:
ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang)
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"):
pluginXbrlMethod(self, options, modelXbrl)
except (IOError, EnvironmentError) as err:
self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err))
success = False
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False
if not options.keepOpen:
if modelDiffReport:
modelDiffReport.close()
elif modelXbrl:
modelXbrl.close()
return success
def runFromExcel(self, options):
testGenFileName = options.excelfilename
#testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls"
testGenDir = os.path.dirname(testGenFileName)
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt'
modelTestcases = ModelXbrl.create(self.modelManager)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(
_("[info] xls loaded in {0:.2} secs at {1}").format(
time.time() - startedAt, timeNow))
# start index file
indexFiles = [
testGenDir + os.sep + 'creationTestcasesIndex.xml',
testGenDir + os.sep + 'consumptionTestcasesIndex.xml'
]
indexDocs = [
xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today))
for purpose in ("Creation", "Consumption")
]
testcasesElements = [
XmlUtil.child(indexDoc, None, "testcases")
for indexDoc in indexDocs
]
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
row = testcaseIndexSheet.row(iRow)
if row[0].ctype == xlrd.XL_CELL_EMPTY or row[
1].ctype == xlrd.XL_CELL_EMPTY or row[
2].ctype == xlrd.XL_CELL_EMPTY:
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
intention = row[3].value
if intention is None or len(intention) == 0:
continue # test not ready to run
reason = row[4].value
expectedEvent = row[5].value
base = os.path.join(os.path.dirname(testGenFileName),
testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and reason.lower() not in (
"n.a.", "error") and expectedEvent != "N.A.":
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"),
True), (uriTo, _("loading to DTS"),
False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager,
URIs,
msg,
base=base)
else:
modelDTS = ModelXbrl.create(
self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(
URIs.replace(", ", "_") + ".dts", base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS,
uri.strip(),
base=base)
DTSdoc.referencesDocument[
doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom and modelDTSto:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace(
"from.xsd", "report.xml")
reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile, base)
testcasesDir = os.path.dirname(
os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i, testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w",
encoding="utf-8") as fh:
XmlUtil.writexml(fh,
testcaseDocs[i],
encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [
testcasesDir + os.sep + testcaseName +
"-creation-testcase.xml", testcasesDir + os.sep +
testcaseName + "-consumption-testcase.xml"
]
for i, testcaseFile in enumerate(testcaseFiles):
XmlUtil.addChild(
testcasesElements[i], None, "testcase",
("uri", testcaseFile[len(testGenDir) +
1:].replace("\\", "/")))
# start testcase file
testcaseDocs = [
xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'<name>{0}</name>'
'<description>{0}</description>'
'</testcase>'.format(testcaseName, purpose,
today))
for purpose in ("Creation", "Consumption")
]
testcaseElements = [
XmlUtil.child(testcaseDoc, conformanceNS,
"testcase")
for testcaseDoc in testcaseDocs
]
priorTestcasesDir = testcasesDir
variationID = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(
modelTestcases)
modelVersReport.diffDTSes(reportFullPath, modelDTSfrom,
modelDTSto)
# check for expected elements
if expectedEvent and expectedEvent not in ("No change",
"N.A."):
if len(
modelVersReport.xmlDocument.
getElementsByTagNameNS('*',
expectedEvent)) == 0:
modelTestcases.error(
"Generated test case {0} missing expected event {1}"
.format(reportName,
expectedEvent), "wrn", "missingEvent")
modelVersReport.close([])
for i, testcaseElt in enumerate(testcaseElements):
variationElement = XmlUtil.addChild(
testcaseElt,
conformanceNS,
"variation",
attributes=("id", "_{0:02n}".format(variationID)))
XmlUtil.addChild(variationElement,
conformanceNS,
"name",
text=intention)
dataElement = XmlUtil.addChild(variationElement,
conformanceNS, "data")
for schemaURIs, dtsAttr in ((uriFrom, "from"), (uriTo,
"to")):
for schemaURI in schemaURIs.split(","):
XmlUtil.addChild(
dataElement,
conformanceNS,
"schema",
attributes=((("dts", dtsAttr), ) +
((("readMeFirst",
"true"), ) if i == 0 else
())),
text=os.path.basename(schemaURI.strip()))
resultElement = XmlUtil.addChild(
variationElement, conformanceNS, "result")
XmlUtil.addChild(
resultElement if i == 0 else dataElement,
conformanceNS,
"versioningReport",
attributes=(("readMeFirst", "true") if i == 1 else
()),
text="report/" + reportName)
variationID += 1
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i, testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i, indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def loadFromExcel(cntlr, excelFile):
from arelle import xlrd
from arelle.xlrd.sheet import empty_cell
from arelle import ModelDocument, ModelXbrl, XmlUtil
from arelle.ModelDocument import ModelDocumentReference
from arelle.ModelValue import qname
startedAt = time.time()
importExcelBook = xlrd.open_workbook(excelFile)
controlSheet = importExcelBook.sheet_by_index(1)
imports = {"xbrli": ( ("namespace", XbrlConst.xbrli),
("schemaLocation", "http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd") )} # xml of imports
importXmlns = {}
linkbaseRefs = []
labelLinkbases = []
hasPreLB = hasCalLB = hasDefLB = False
# xxxLB structure [ (elr1, def1, "_ELR_", [roots]), (elr2, def2, "_ELR_", [rootw]) ...]
# roots = (rootHref, None, "_root_", [children])
# children = (childPrefix, childName, arcrole, [grandChildren])
preLB = []
defLB = []
calLB = []
def lbDepthList(lbStruct, depth, parentList=None):
if depth == 0:
return lbStruct[-1].childStruct
return lbDepthList(lbStruct[-1].childStruct, depth-1, list)
extensionElements = {}
extensionRoles = {} # key is roleURI, value is role definition
extensionLabels = {} # key = (prefix, name, lang, role), value = label text
def extensionHref(prefix, name):
if prefix == extensionSchemaPrefix:
filename = extensionSchemaFilename
elif prefix in imports:
filename = imports[prefix][1][1]
else:
return None
return "{0}#{1}_{2}".format(filename, prefix, name)
isUSGAAP = False
for iRow in range(1, controlSheet.nrows):
try:
row = controlSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY): # skip if col 1 is empty
continue
action = row[0].value
filetype = row[1].value
prefix = row[2].value
filename = row[3].value
namespaceURI = row[4].value
lbType = lang = None
if action == "import":
imports[prefix] = ( ("namespace", namespaceURI), ("schemaLocation", filename) )
importXmlns[prefix] = namespaceURI
if re.match(r"http://[^/]+/us-gaap/", namespaceURI):
isUSGAAP = True
elif action == "extension":
if filetype == "schema":
extensionSchemaPrefix = prefix
extensionSchemaFilename = filename
extensionSchemaNamespaceURI = namespaceURI
elif filetype == "linkbase":
typeLang = prefix.split()
if len(typeLang) > 0:
lbType = typeLang[0]
else:
lbType = "unknown"
if len(typeLang) > 1:
lang = typeLang[1]
else:
lang = "en"
if lbType == "label":
labelLinkbases.append((lang, filename))
elif lbType == "presentation":
hasPreLB = True
elif lbType == "definition":
hasDefLB = True
elif lbType == "calculation":
hasCalLB = True
linkbaseRefs.append( (lbType, filename) )
elif filetype == "role" and namespaceURI:
extensionRoles[namespaceURI] = filename
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:exception")
importExcelSheet = importExcelBook.sheet_by_index(0)
# find column headers row
headerCols = {}
headerRows = set()
# find out which rows are header rows
for iRow in range(0, importExcelSheet.nrows):
row = importExcelSheet.row(iRow)
for iCol, colCell in enumerate(row):
v = colCell.value
if v in importColumnHeaders:
headerCols[importColumnHeaders[v]] = iCol
if all(colName in headerCols
for colName in ("name", "type", "depth")): # must have these to be a header col
# it's a header col
headerRows.add(iRow)
headerCols.clear()
def cellValue(row, header):
if header in headerCols:
iCol = headerCols[header]
if iCol < len(row):
return row[iCol].value
return ''
def checkImport(qname):
prefix, sep, localName = qname.partition(":")
if sep:
if prefix not in imports:
if prefix == "xbrldt":
imports["xbrldt"] = ("namespace", XbrlConst.xbrldt), ("schemaLocation", "http://www.xbrl.org/2005/xbrldt-2005.xsd")
elif prefix == "nonnum":
imports["nonnum"] = ("namespace", "http://www.xbrl.org/dtr/type/non-numeric"), ("schemaLocation", "http://www.xbrl.org/dtr/type/nonNumeric-2009-12-16.xsd")
else:
cntlr.addToLog("Warning: prefix schema file is not imported for: {qname}"
.format(qname=qname),
messageCode="importExcel:warning")
# find header rows
currentELR = currentELRdefinition = None
for iRow in range(0, importExcelSheet.nrows):
useLabels = False
try:
row = importExcelSheet.row(iRow)
isHeaderRow = iRow in headerRows
isELRrow = (iRow + 1) in headerRows
if isHeaderRow:
headerCols.clear()
for iCol, colCell in enumerate(row):
v = colCell.value
if v in importColumnHeaders:
headerCols[importColumnHeaders[v]] = iCol
elif isELRrow:
currentELR = currentELRdefinition = None
for colCell in row:
v = colCell.value
if v.startswith("http://"):
currentELR = v
elif not currentELRdefinition and v.endswith("　科目一覧"):
currentELRdefinition = v[0:-5]
elif not currentELRdefinition:
currentELRdefinition = v
if currentELR or currentELRdefinition:
if hasPreLB:
preLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasDefLB:
defLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
if hasCalLB:
calLB.append( LBentry(role=currentELR, name=currentELRdefinition, isELR=True) )
elif headerCols:
prefix = cellValue(row, 'prefix').strip()
name = cellValue(row, 'name').strip()
if "depth" in headerCols:
try:
depth = int(cellValue(row, 'depth'))
except ValueError:
depth = None
else:
depth = None
if prefix == extensionSchemaPrefix and name not in extensionElements:
# elements row
eltType = cellValue(row, 'type')
subsGrp = cellValue(row, 'substitutionGroup')
abstract = cellValue(row, 'abstract')
nillable = cellValue(row, 'nillable')
balance = cellValue(row, 'balance')
periodType = cellValue(row, 'periodType')
newElt = [ ("name", name), ("id", prefix + "_" + name) ]
if eltType:
newElt.append( ("type", eltType) )
checkImport(eltType)
if subsGrp:
newElt.append( ("substitutionGroup", subsGrp) )
checkImport(subsGrp)
if abstract:
newElt.append( ("abstract", abstract) )
if nillable:
newElt.append( ("nillable", nillable) )
if balance:
newElt.append( ("{http://www.xbrl.org/2003/instance}balance", balance) )
if periodType:
newElt.append( ("{http://www.xbrl.org/2003/instance}periodType", periodType) )
extensionElements[name] = newElt
useLabels = True
if depth is not None:
if hasPreLB:
entryList = lbDepthList(preLB, depth)
preferredLabel = cellValue(row, 'preferredLabel')
if preferredLabel and not preferredLabel.startswith("http://"):
preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel
if entryList is not None:
if depth == 0:
entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) )
else:
entryList.append( LBentry(prefix=prefix, name=name, arcrole=XbrlConst.parentChild,
role=preferredLabel) )
if hasDefLB:
entryList = lbDepthList(defLB, depth)
if entryList is not None:
if depth == 0:
entryList.append( LBentry(prefix=prefix, name=name, isRoot=True) )
else:
entryList.append( LBentry(prefix=prefix, name=name, arcrole="_dimensions_") )
if hasCalLB:
calcParent = cellValue(row, 'calculationParent')
calcWeight = cellValue(row, 'calculationWeight')
if calcParent and calcWeight:
calcParentPrefix, sep, calcParentName = calcParent.partition(":")
entryList = lbDepthList(calLB, 0)
if entryList is not None:
entryList.append( LBentry(prefix=calcParentPrefix, name=calcParentName, isRoot=True, childStruct=
[LBentry(prefix=prefix, name=name, arcrole=XbrlConst.summationItem, weight=calcWeight )]) )
# accumulate extension labels
if useLabels:
prefix = cellValue(row, 'prefix').strip()
name = cellValue(row, 'name').strip()
preferredLabel = cellValue(row, 'preferredLabel')
if preferredLabel and not preferredLabel.startswith("http://"):
preferredLabel = "http://www.xbrl.org/2003/role/" + preferredLabel
for colItem, iCol in headerCols.items():
if isinstance(colItem, tuple):
colItemType, role, lang = colItem
cell = row[iCol]
if cell.ctype == xlrd.XL_CELL_EMPTY:
values = ()
elif colItemType == "label":
values = (cell.value,)
elif colItemType == "labels":
values = cell.value.split('\n')
else:
values = ()
if preferredLabel: # first label column sets preferredLabel if any
role = preferredLabel
preferredLabel = None
for value in values:
extensionLabels[prefix, name, lang, role] = value.strip()
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="importExcel:exception")
if isUSGAAP and hasDefLB:
# move line items above table
def fixUsggapTableDims(lvl1Struct):
foundLineItems = False
for i1, lvl1Entry in enumerate(lvl1Struct):
for i2, lvl2Entry in enumerate(lvl1Entry.childStruct):
for i3, lvl3Entry in enumerate(lvl2Entry.childStruct):
if lvl3Entry.name.endswith("LineItems") and lvl2Entry.name.endswith("Table"):
foundLineItems = True
break
if foundLineItems:
break
else:
fixUsggapTableDims(lvl1Entry.childStruct)
if foundLineItems:
lvl1Struct.insert(i1 + 1, LBentry(prefix=lvl3Entry.prefix, name=lvl3Entry.name, arcrole=lvl1Entry.arcrole, childStruct=lvl3Entry.childStruct)) # must keep lvl1Rel if it is __root__
lvl3Entry.childStruct.insert(0, lvl2Entry)
if lvl1Entry.name.endswith("Abstract"):
del lvl1Struct[i1]
del lvl2Entry.childStruct[i3]
pass
fixUsggapTableDims(defLB)
dts = cntlr.modelManager.create(newDocumentType=ModelDocument.Type.SCHEMA,
url=extensionSchemaFilename,
isEntry=True,
base='', # block pathname from becomming absolute
initialXml='''
<schema xmlns="http://www.w3.org/2001/XMLSchema"
targetNamespace="{targetNamespace}"
attributeFormDefault="unqualified"
elementFormDefault="qualified"
xmlns:xsd="http://www.w3.org/2001/XMLSchema"
xmlns:{extensionPrefix}="{targetNamespace}"
{importXmlns}
xmlns:iod="http://disclosure.edinet-fsa.go.jp/taxonomy/common/2013-03-31/iod"
xmlns:nonnum="http://www.xbrl.org/dtr/type/non-numeric"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xbrli="http://www.xbrl.org/2003/instance"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrldt="http://xbrl.org/2005/xbrldt"/>
'''.format(targetNamespace=extensionSchemaNamespaceURI,
extensionPrefix=extensionSchemaPrefix,
importXmlns=''.join('xmlns:{0}="{1}"\n'.format(prefix, namespaceURI)
for prefix, namespaceURI in importXmlns.items())
)
)
dtsSchemaDocument = dts.modelDocument
dtsSchemaDocument.inDTS = True # entry document always in DTS
dtsSchemaDocument.targetNamespace = extensionSchemaNamespaceURI # not set until schemaDiscover too late otherwise
schemaElt = dtsSchemaDocument.xmlRootElement
#foreach linkbase
annotationElt = XmlUtil.addChild(schemaElt, XbrlConst.xsd, "annotation")
appinfoElt = XmlUtil.addChild(annotationElt, XbrlConst.xsd, "appinfo")
for iRow in range(0, importExcelSheet.nrows):
try:
row = importExcelSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY): # skip if col 1 is empty
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
except Exception as err:
cntlr.addToLog("Exception: {error}, Excel row: {excelRow}"
.format(error=err,
excelRow=iRow),
messageCode="loadFromExcel:exception")
# add linkbaseRefs
appinfoElt = XmlUtil.descendant(schemaElt, XbrlConst.xsd, "appinfo")
# don't yet add linkbase refs, want to process imports first to get roleType definitions
# add imports
for importAttributes in sorted(imports.values()):
XmlUtil.addChild(schemaElt,
XbrlConst.xsd, "import",
attributes=importAttributes)
# add elements
for eltName, eltAttrs in sorted(extensionElements.items(), key=lambda item: item[0]):
XmlUtil.addChild(schemaElt,
XbrlConst.xsd, "element",
attributes=eltAttrs)
# add role definitions (for discovery)
for roleURI, roleDefinition in extensionRoles.items():
roleElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "roleType",
attributes=(("roleURI", roleURI),
("id", "roleType_" + roleURI.rpartition("/")[2])))
if roleDefinition:
XmlUtil.addChild(roleElt, XbrlConst.link, "definition", text=roleDefinition)
if hasPreLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:presentationLink")
if hasDefLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:definitionLink")
if hasCalLB:
XmlUtil.addChild(roleElt, XbrlConst.link, "usedOn", text="link:calculationLink")
dtsSchemaDocument.schemaDiscover(schemaElt, False, extensionSchemaNamespaceURI)
def addLinkbaseRef(lbType, lbFilename, lbDoc):
role = "http://www.xbrl.org/2003/role/{0}LinkbaseRef".format(lbType)
lbRefElt = XmlUtil.addChild(appinfoElt, XbrlConst.link, "linkbaseRef",
attributes=(("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", lbFilename),
("{http://www.w3.org/1999/xlink}role", role),
("{http://www.w3.org/1999/xlink}arcrole", "http://www.w3.org/1999/xlink/properties/linkbase"),
))
dtsSchemaDocument.referencesDocument[lbDoc] = ModelDocumentReference("href", lbRefElt)
# label linkbase
for lang, filename in labelLinkbases:
lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base="", initialXml="""
<link:linkbase
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase
http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance"/>
""")
lbDoc.inDTS = True
addLinkbaseRef("label", filename, lbDoc)
lbElt = lbDoc.xmlRootElement
linkElt = XmlUtil.addChild(lbElt,
XbrlConst.link, "labelLink",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/link")))
locs = set()
for labelKey, text in extensionLabels.items():
prefix, name, labelLang, role = labelKey
if lang == labelLang:
locLabel = prefix + "_" + name
if locLabel not in locs:
locs.add(locLabel)
XmlUtil.addChild(linkElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", extensionHref(prefix, name)),
("{http://www.w3.org/1999/xlink}label", locLabel)))
XmlUtil.addChild(linkElt,
XbrlConst.link, "labelArc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", "http://www.xbrl.org/2003/arcrole/concept-label"),
("{http://www.w3.org/1999/xlink}from", locLabel),
("{http://www.w3.org/1999/xlink}to", "label_" + locLabel),
("order", 1.0)))
XmlUtil.addChild(linkElt,
XbrlConst.link, "label",
attributes=(("{http://www.w3.org/1999/xlink}type", "resource"),
("{http://www.w3.org/1999/xlink}label", "label_" + locLabel),
("{http://www.w3.org/1999/xlink}role", role),
("{http://www.w3.org/XML/1998/namespace}lang", lang)),
text=text)
lbDoc.linkbaseDiscover(lbElt)
def hrefConcept(prefix, name):
qn = schemaElt.prefixedNameQname(prefix + ":" + name)
if qn in dts.qnameConcepts:
return dts.qnameConcepts[qn]
return None
def lbTreeWalk(lbType, parentElt, lbStruct, roleRefs, locs=None, fromPrefix=None, fromName=None):
order = 1.0
for lbEntry in lbStruct:
if lbEntry.isELR:
role = "unspecified"
if lbEntry.role and lbEntry.role.startswith("http://"): # have a role specified
role = lbEntry.role
elif lbEntry.name: #may be a definition
for linkroleUri, modelRoleTypes in dts.roleTypes.items():
definition = modelRoleTypes[0].definition
if lbEntry.name == definition:
role = linkroleUri
break
if role != XbrlConst.defaultLinkRole and role in dts.roleTypes: # add roleRef
roleType = modelRoleTypes[0]
roleRef = ("roleRef", role, roleType.modelDocument.uri + "#" + roleType.id)
roleRefs.add(roleRef)
linkElt = XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Link",
attributes=(("{http://www.w3.org/1999/xlink}type", "extended"),
("{http://www.w3.org/1999/xlink}role", role)))
locs = set()
lbTreeWalk(lbType, linkElt, lbEntry.childStruct, roleRefs, locs)
else:
toPrefix = lbEntry.prefix
toName = lbEntry.name
toHref = extensionHref(toPrefix, toName)
toLabel = toPrefix + "_" + toName
if toHref not in locs:
XmlUtil.addChild(parentElt,
XbrlConst.link, "loc",
attributes=(("{http://www.w3.org/1999/xlink}type", "locator"),
("{http://www.w3.org/1999/xlink}href", toHref),
("{http://www.w3.org/1999/xlink}label", toLabel)))
locs.add(toHref)
if not lbEntry.isRoot:
fromLabel = fromPrefix + "_" + fromName
if lbType == "calculation" and lbEntry.weight is not None:
otherAttrs = ( ("weight", lbEntry.weight), )
elif lbType == "presentation" and lbEntry.role is not None:
otherAttrs = ( ("preferredLabel", lbEntry.role), )
else:
otherAttrs = ( )
if lbEntry.arcrole == "_dimensions_": # pick proper consecutive arcrole
fromConcept = hrefConcept(fromPrefix, fromName)
toConcept = hrefConcept(toPrefix, toName)
if toConcept is not None and toConcept.isHypercubeItem:
arcrole = XbrlConst.all
elif toConcept is not None and toConcept.isDimensionItem:
arcrole = XbrlConst.hypercubeDimension
elif fromConcept is not None and fromConcept.isDimensionItem:
arcrole = XbrlConst.dimensionDomain
else:
arcrole = XbrlConst.domainMember
else:
arcrole = lbEntry.arcrole
XmlUtil.addChild(parentElt,
XbrlConst.link, lbType + "Arc",
attributes=(("{http://www.w3.org/1999/xlink}type", "arc"),
("{http://www.w3.org/1999/xlink}arcrole", arcrole),
("{http://www.w3.org/1999/xlink}from", fromLabel),
("{http://www.w3.org/1999/xlink}to", toLabel),
("order", order)) + otherAttrs )
order += 1.0
if lbType != "calculation" or lbEntry.isRoot:
lbTreeWalk(lbType, parentElt, lbEntry.childStruct, roleRefs, locs, toPrefix, toName)
for hasLB, lbType, lbLB in ((hasPreLB, "presentation", preLB),
(hasDefLB, "definition", defLB),
(hasCalLB, "calculation", calLB)):
if hasLB:
for lbRefType, filename in linkbaseRefs:
if lbType == lbRefType:
# output presentation linkbase
lbDoc = ModelDocument.create(dts, ModelDocument.Type.LINKBASE, filename, base='', initialXml="""
<link:linkbase
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.xbrl.org/2003/linkbase
http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance"/>
""")
lbDoc.inDTS = True
addLinkbaseRef(lbRefType, filename, lbDoc)
lbElt = lbDoc.xmlRootElement
roleRefs = set()
if lbType == "definition":
roleRefs.update((("arcroleRef", XbrlConst.all, "http://www.xbrl.org/2005/xbrldt-2005.xsd#all"),
("arcroleRef", XbrlConst.dimensionDefault, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-default"),
("arcroleRef", XbrlConst.dimensionDomain, "http://www.xbrl.org/2005/xbrldt-2005.xsd#dimension-domain"),
("arcroleRef", XbrlConst.domainMember, "http://www.xbrl.org/2005/xbrldt-2005.xsd#domain-member"),
("arcroleRef", XbrlConst.hypercubeDimension, "http://www.xbrl.org/2005/xbrldt-2005.xsd#hypercube-dimension")))
lbTreeWalk(lbType, lbElt, lbLB, roleRefs)
firstLinkElt = None
for firstLinkElt in lbElt.iterchildren():
break
# add arcrole references
for roleref, roleURI, href in roleRefs:
XmlUtil.addChild(lbElt,
XbrlConst.link, roleref,
attributes=(("arcroleURI" if roleref == "arcroleRef" else "roleURI", roleURI),
("{http://www.w3.org/1999/xlink}type", "simple"),
("{http://www.w3.org/1999/xlink}href", href)),
beforeSibling=firstLinkElt)
lbDoc.linkbaseDiscover(lbElt)
break
#cntlr.addToLog("Completed in {0:.2} secs".format(time.time() - startedAt),
# messageCode="loadFromExcel:info")
return dts
def init(modelXbrl):
# setup modelXbrl for rendering evaluation
# dimension defaults required in advance of validation
from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
hasXbrlTables = False
# validate table linkbase dimensions
for baseSetKey in modelXbrl.baseSets.keys():
arcrole, ELR, linkqname, arcqname = baseSetKey
if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(arcrole):
ValidateFormula.checkBaseSet(modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname))
if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableAxis2011):
hasXbrlTables = True
# provide context for view
if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE:
instance = None # use instance of the entry pont
else: # need dummy instance
instance = ModelDocument.create(modelXbrl, ModelDocument.Type.INSTANCE,
"dummy.xml", # fake URI and fake schemaRef
("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd",))
if hasXbrlTables:
# formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables
modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance)
modelXbrl.profileStat(None)
# setup fresh parameters from formula optoins
modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters()
# validate parameters and custom function signatures
ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables"))
# check and extract message expressions into compilable programs
for msgArcrole in (XbrlConst.tableDefinitionNodeMessage, XbrlConst.tableDefinitionNodeSelectionMessage,
XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011):
for msgRel in modelXbrl.relationshipSet(msgArcrole).modelRelationships:
ValidateFormula.checkMessageExpressions(modelXbrl, msgRel.toModelObject)
# compile and validate tables
for modelTable in modelXbrl.modelRenderingTables:
modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining
modelTable.compile()
# check aspectModel
if modelTable.aspectModel not in ("non-dimensional", "dimensional"):
modelXbrl.error("xbrlte:unknownAspectModel",
_("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"),
modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel)
else:
modelTable.priorAspectAxisDisposition = {}
# check ordinate aspects against aspectModel
oppositeAspectModel = (_DICT_SET({'dimensional','non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop()
uncoverableAspects = aspectModels[oppositeAspectModel] - aspectModels[modelTable.aspectModel]
for tblAxisRel in modelXbrl.relationshipSet((XbrlConst.tableBreakdown,XbrlConst.tableAxis2011)).fromModelObject(modelTable):
checkDefinitionNodeAspectModel(modelXbrl, modelTable, tblAxisRel, uncoverableAspects)
del modelTable.priorAspectAxisDisposition
modelXbrl.profileStat(_("compileTables"))
def watchCycle(self):
while not self.stopRequested:
rssWatchOptions = self.rssModelXbrl.modelManager.rssWatchOptions
# check rss expiration
rssHeaders = self.cntlr.webCache.getheaders(self.rssModelXbrl.modelManager.rssWatchOptions.get("feedSourceUri"))
expires = parseRfcDatetime(rssHeaders.get("expires"))
reloadNow = True # texpires and expires > datetime.datetime.now()
# reload rss feed
self.rssModelXbrl.reload('checking RSS items', reloadCache=reloadNow)
if self.stopRequested: break
# setup validator
postLoadActions = []
if rssWatchOptions.get("validateDisclosureSystemRules"):
self.instValidator = ValidateFiling.ValidateFiling(self.rssModelXbrl)
postLoadActions.append(_("validating"))
elif rssWatchOptions.get("validateXbrlRules") or rssWatchOptions.get("validateFormulaAssertions"):
self.instValidator = ValidateXbrl.ValidateXbrl(self.rssModelXbrl)
postLoadActions.append(_("validating"))
if (rssWatchOptions.get("validateFormulaAssertions")):
postLoadActions.append(_("running formulas"))
else:
self.instValidator = None
matchTextExpr = rssWatchOptions.get("matchTextExpr")
if matchTextExpr:
matchPattern = re.compile(matchTextExpr)
postLoadActions.append(_("matching text"))
else:
matchPattern= None
postLoadAction = ', '.join(postLoadActions)
# anything to check new filings for
if (rssWatchOptions.get("validateDisclosureSystemRules") or
rssWatchOptions.get("validateXbrlRules") or
rssWatchOptions.get("validateCalcLinkbase") or
rssWatchOptions.get("validateFormulaAssertions") or
rssWatchOptions.get("alertMatchedFactText") or
any(pluginXbrlMethod(rssWatchOptions)
for pluginXbrlMethod in pluginClassMethods("RssWatch.HasWatchAction"))
):
# form keys in ascending order of pubdate
pubDateRssItems = []
for rssItem in self.rssModelXbrl.modelDocument.rssItems:
pubDateRssItems.append((rssItem.pubDate,rssItem.objectId()))
for pubDate, rssItemObjectId in sorted(pubDateRssItems):
rssItem = self.rssModelXbrl.modelObject(rssItemObjectId)
# update ui thread via modelManager (running in background here)
self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId())
if self.stopRequested:
break
latestPubDate = XmlUtil.datetimeValue(rssWatchOptions.get("latestPubDate"))
if (latestPubDate and
rssItem.pubDate < latestPubDate):
continue
try:
# try zipped URL if possible, else expanded instance document
modelXbrl = ModelXbrl.load(self.rssModelXbrl.modelManager,
openFileSource(rssItem.zippedUrl, self.cntlr),
postLoadAction)
if self.stopRequested:
modelXbrl.close()
break
emailAlert = False
if modelXbrl.modelDocument is None:
modelXbrl.error("arelle.rssWatch",
_("RSS item %(company)s %(form)s document not loaded: %(date)s"),
modelXbrl=modelXbrl, company=rssItem.companyName,
form=rssItem.formType, date=rssItem.filingDate)
rssItem.status = "not loadable"
else:
# validate schema, linkbase, or instance
if self.stopRequested:
modelXbrl.close()
break
if self.instValidator:
self.instValidator.validate(modelXbrl)
if modelXbrl.errors and rssWatchOptions.get("alertValiditionError"):
emailAlert = True
for pluginXbrlMethod in pluginClassMethods("RssWatch.DoWatchAction"):
pluginXbrlMethod(modelXbrl, rssWatchOptions, rssItem)
# check match expression
if matchPattern:
for fact in modelXbrl.factsInInstance:
v = fact.value
if v is not None:
m = matchPattern.search(v)
if m:
fr, to = m.span()
msg = _("Fact Variable {0}\n context {1}\n matched text: {2}").format(
fact.qname, fact.contextID, v[max(0,fr-20):to+20])
modelXbrl.info("arelle.rssInfo",
msg,
modelXbrl=modelXbrl) # msg as code passes it through to the status
if rssWatchOptions.get("alertMatchedFactText"):
emailAlert = True
if (rssWatchOptions.get("formulaFileUri") and rssWatchOptions.get("validateFormulaAssertions") and
self.instValidator):
# attach formulas
ModelDocument.load(modelXbrl, rssWatchOptions["formulaFileUri"])
ValidateFormula.validate(self.instValidator)
rssItem.setResults(modelXbrl)
modelXbrl.close()
del modelXbrl # completely dereference
self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId())
if rssItem.assertionUnsuccessful and rssWatchOptions.get("alertAssertionUnsuccessful"):
emailAlert = True
msg = _("Filing CIK {0}\n "
"company {1}\n "
"published {2}\n "
"form type {3}\n "
"filing date {4}\n "
"period {5}\n "
"year end {6}\n "
"results: {7}").format(
rssItem.cikNumber,
rssItem.companyName,
rssItem.pubDate,
rssItem.formType,
rssItem.filingDate,
rssItem.period,
rssItem.fiscalYearEnd,
rssItem.status)
self.rssModelXbrl.info("arelle:rssWatch", msg, modelXbrl=self.rssModelXbrl)
emailAddress = rssWatchOptions.get("emailAddress")
if emailAlert and emailAddress:
self.rssModelXbrl.modelManager.showStatus(_("sending e-mail alert"))
import smtplib
from email.mime.text import MIMEText
emailMsg = MIMEText(msg)
emailMsg["Subject"] = _("Arelle RSS Watch alert on {0}").format(rssItem.companyName)
emailMsg["From"] = emailAddress
emailMsg["To"] = emailAddress
smtp = smtplib.SMTP()
smtp.sendmail(emailAddress, [emailAddress], emailMsg.as_string())
smtp.quit()
self.rssModelXbrl.modelManager.showStatus(_("RSS item {0}, {1} completed, status {2}").format(rssItem.companyName, rssItem.formType, rssItem.status), 3500)
self.rssModelXbrl.modelManager.cntlr.rssWatchUpdateOption(rssItem.pubDate.strftime('%Y-%m-%dT%H:%M:%S'))
except Exception as err:
self.rssModelXbrl.error("arelle.rssError",
_("RSS item %(company)s, %(form)s, %(date)s, exception: %(error)s"),
modelXbrl=self.rssModelXbrl, company=rssItem.companyName,
form=rssItem.formType, date=rssItem.filingDate, error=err,
exc_info=True)
if self.stopRequested: break
if self.stopRequested:
self.cntlr.showStatus(_("RSS watch, stop requested"), 10000)
else:
import time
time.sleep(600)
self.thread = None # close thread
self.stopRequested = False
def secCorrespondenceLoader(modelXbrl, mappedUri, filepath, **kwargs):
if (mappedUri.startswith("http://www.sec.gov/Archives/edgar/Feed/") and
mappedUri.endswith(".nc.tar.gz")):
# daily feed loader (the rss object)
rssObject = ModelRssObject(modelXbrl, uri=mappedUri, filepath=filepath)
# location for expanded feed files
tempdir = os.path.join(modelXbrl.modelManager.cntlr.userAppDir, "tmp", "edgarFeed")
# remove prior files
if os.path.exists(tempdir):
os.system("rm -fr {}".format(tempdir)) # rmtree does not work with this many files!
os.makedirs(tempdir, exist_ok=True)
# untar to /temp/arelle/edgarFeed for faster operation
startedAt = time.time()
modelXbrl.fileSource.open()
modelXbrl.fileSource.fs.extractall(tempdir)
modelXbrl.info("info", "untar edgarFeed temp files in %.2f sec" % (time.time() - startedAt),
modelObject=modelXbrl)
# find <table> with <a>Download in it
for instanceFile in sorted(os.listdir(tempdir)): # modelXbrl.fileSource.dir:
if instanceFile != ".":
rssObject.rssItems.append(
SECCorrespondenceItem(modelXbrl, instanceFile, mappedUri + '/' + instanceFile))
return rssObject
elif "rssItem" in kwargs and ".nc.tar.gz/" in mappedUri:
rssItem = kwargs["rssItem"]
text = None # no instance information
# parse document
try:
startedAt = time.time()
file, encoding = modelXbrl.fileSource.file(
os.path.join(modelXbrl.modelManager.cntlr.userAppDir, "tmp", "edgarFeed",
os.path.basename(rssItem.url)))
s = file.read()
file.close()
for match in re.finditer(r"[<]([^>]+)[>]([^<\n\r]*)", s, re.MULTILINE):
tag = match.group(1).lower()
v = match.group(2)
if tag == "accession-number":
rssItem.accessionNumber = v
elif tag == "form-type":
rssItem.formType = v
if v != "UPLOAD":
rssItem.doNotProcessRSSitem = True # skip this RSS item in validate loop, don't load DB
elif tag == "filing-date":
try:
rssItem.filingDate = datetime.date(int(v[0:4]), int(v[4:6]), int(v[6:8]))
except (ValueError, IndexError):
pass
elif tag == "conformed-name":
rssItem.companyName = v
elif tag == "cik":
rssItem.cikNumber = v
elif tag == "assigned-sic":
rssItem.assignedSic = v
elif tag == "fiscal-year-end":
try:
rssItem.fiscalYearEnd = v[0:2] + '-' + v[2:4]
except (IndexError, TypeError):
pass
match = re.search("<PDF>(.*)</PDF>", s, re.DOTALL)
if match:
import uu, io
pageText = []
uuIn = io.BytesIO(match.group(1).encode(encoding))
uuOut = io.BytesIO()
uu.decode(uuIn, uuOut)
from pyPdf import PdfFileReader
uuOut.seek(0,0)
try:
pdfIn = PdfFileReader(uuOut)
for pageNum in range(pdfIn.getNumPages()):
pageText.append(pdfIn.getPage(pageNum).extractText())
except:
# do we want a warning here that the PDF can't be read with this library?
pass
uuIn.close()
uuOut.close()
text = ''.join(pageText)
else:
match = re.search("<TEXT>(.*)</TEXT>", s, re.DOTALL)
if match:
text = match.group(1)
except (IOError, EnvironmentError):
pass # give up, no instance
# daily rss item loader, provide unpopulated instance document to be filled in by RssItem.Xbrl.Loaded
if not text:
rssItem.doNotProcessRSSitem = True # skip this RSS item in validate loop, don't load DB
instDoc = ModelDocument.create(modelXbrl,
ModelDocument.Type.UnknownXML,
rssItem.url,
isEntry=True,
base='', # block pathname from becomming absolute
initialXml='<DummyXml/>')
else:
instDoc = ModelDocument.create(modelXbrl,
ModelDocument.Type.INSTANCE,
rssItem.url,
isEntry=True,
base='', # block pathname from becomming absolute
initialXml='''
<xbrli:xbrl xmlns:doc="http://arelle.org/doc/2014-01-31"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance">
<link:schemaRef xlink:type="simple" xlink:href="http://arelle.org/2014/doc-2014-01-31.xsd"/>
<xbrli:context id="pubDate">
<xbrli:entity>
<xbrli:identifier scheme="http://www.sec.gov/CIK">{cik}</xbrli:identifier>
</xbrli:entity>
<xbrli:period>
<xbrli:instant>{pubDate}</xbrli:instant>
</xbrli:period>
</xbrli:context>
<doc:Correspondence contextRef="pubDate">{text}</doc:Correspondence>
</xbrli:xbrl>
'''.format(cik=rssItem.cikNumber,
pubDate=rssItem.pubDate.date(),
text=text.strip().replace("&","&").replace("<","<")))
#modelXbrl.info("info", "loaded in %.2f sec" % (time.time() - startedAt),
# modelDocument=instDoc)
return instDoc
return None
def loadPrimaryDocumentFacts(dts, rssItem, entityInformation):
# identify tables
disclosureSystem = dts.modelManager.disclosureSystem
if disclosureSystem.validationType != "EFM":
return
if rssItem is not None:
formType = rssItem.formType
fileUrl = rssItem.primaryDocumentURL
reloadCache = getattr(rssItem.modelXbrl, "reloadCache", False)
else:
formType = entityInformation.get("form-type")
fileUrl = entityInformation.get("primary-document-url")
reloadCache = False
if fileUrl and formType and (formType.startswith('10-K')
or formType.startswith('10-Q')):
if fileUrl.endswith(".txt") or fileUrl.endswith(".htm"):
if formType.startswith('10-K'):
parts = SEC10Kparts
elif formType.startswith('10-Q'):
parts = SEC10Qparts
# try to load and use it
normalizedUrl = dts.modelManager.cntlr.webCache.normalizeUrl(
fileUrl)
text = ''
try:
filePath = dts.modelManager.cntlr.webCache.getfilename(
normalizedUrl, reload=reloadCache)
if filePath:
if filePath.endswith('.txt'):
with open(filePath, encoding='utf-8') as fh:
text = fh.read()
elif filePath.endswith('.htm'):
doc = html.parse(filePath)
textParts = []
def iterTextParts(parent):
for node in parent.iterchildren():
if isinstance(node, etree._Element):
if node.tag in ('p', 'P', 'br', 'BR',
'div', 'DIV'):
textParts.append('\n')
textParts.append(node.text or '')
iterTextParts(node)
if node.tail: # use tail whether element, comment, or processing instruction
textParts.append(node.tail)
iterTextParts(doc.getroot())
text = ' '.join(textParts)
except (IOError, EnvironmentError, AttributeError
) as err: # attribute err if html has no root element
dts.info(
"xpDB:primaryDocumentLoadingError",
_("Loading XBRL DB: primary document loading error: %(error)s"
),
modelObject=dts,
error=err)
#with open("/Users/hermf/temp/test.txt", "w", encoding='utf-8') as fh:
# fh.write(text)
class Span:
def __init__(self, start):
self.start = start
self.end = -1
# find the parts
partSpan = {}
partPrev = None
missing2ndPart1 = False
for partMatch in partPattern.finditer(text):
part = partMatch.group(1).upper()
if partPrev is not None:
if part != 'I' and part == partPrev:
# two of these parts without second part 1, use signature or first item for 2nd part 1
missing2ndPart1 = True
partSpan[partPrev].end = partMatch.start(0)
partSpan[part] = Span(partMatch.end(1))
partPrev = part
if partPrev is not None:
partSpan[partPrev].end = len(text)
if missing2ndPart1:
# signatures
signaturesStarts = []
for signaturesMatch in signaturesPattern.finditer(text):
signaturesStarts.append(signaturesMatch.start(0))
#check if PART I missing then use first signatures
if 'I' in partSpan and 'II' in partSpan:
if len(signaturesStarts) == 2 and signaturesStarts[
0] > partSpan['I'].start:
partSpan['I'].start = signaturesStarts[0]
partSpan['I'].end = partSpan['II'].start
else:
# use ASSETS as start of part 1
for assetsMatch in assetsPattern.finditer(text):
partSpan['I'].start = assetsMatch.end(0)
partSpan['I'].end = partSpan['II'].start
break
# find the items
itemSpan = {}
for part, span in partSpan.items():
item = None
for itemMatch in itemPattern.finditer(text, span.start,
span.end):
if item is not None:
itemSpan[(part, item)].end = itemMatch.start(0)
item = itemMatch.group(1)
itemSpan[(part, item)] = Span(itemMatch.end(1))
if item is not None:
itemSpan[(part, item)].end = span.end
if any(itemKey in parts for itemKey in itemSpan.keys()):
# find default context
for cntx in dts.contexts.values():
if cntx.isStartEndPeriod:
if not cntx.hasSegment:
# use c as default context
# load extra datapoints taxonomy but not as discovered in DTS
ModelDocument.load(
dts,
"http://arelle.org/2014/doc-2014-01-31.xsd",
)
# add the facts
for itemKey, itemSpan in itemSpan.items():
if itemKey in parts:
dts.createFact(qname(
"{http://arelle.org/doc/2014-01-31}doc:"
+ parts[itemKey]),
attributes=[("contextRef",
cntx.id)],
text=text[itemSpan.start:
itemSpan.end])
break
def loadPrimaryDocumentFacts(dts, rssItem, entityInformation):
# identify tables
disclosureSystem = dts.modelManager.disclosureSystem
if not disclosureSystem.EFM:
return
if rssItem is not None:
formType = rssItem.formType
fileUrl = rssItem.primaryDocumentURL
else:
formType = entityInformation.get("form-type")
fileUrl = entityInformation.get("primary-document-url")
if fileUrl and formType and (formType.startswith("10-K") or formType.startswith("10-Q")):
if fileUrl.endswith(".txt") or fileUrl.endswith(".htm"):
if formType.startswith("10-K"):
parts = SEC10Kparts
elif formType.startswith("10-Q"):
parts = SEC10Qparts
# try to load and use it
normalizedUrl = dts.modelManager.cntlr.webCache.normalizeUrl(fileUrl)
try:
filePath = dts.modelManager.cntlr.webCache.getfilename(normalizedUrl)
if filePath:
if filePath.endswith(".txt"):
with open(filePath, encoding="utf-8") as fh:
text = fh.read()
elif filePath.endswith(".htm"):
doc = html.parse(filePath)
textParts = []
def iterTextParts(parent):
for node in parent.iterchildren():
if isinstance(node, etree._Element):
if node.tag in ("p", "P", "br", "BR", "div", "DIV"):
textParts.append("\n")
textParts.append(node.text or "")
iterTextParts(node)
if node.tail: # use tail whether element, comment, or processing instruction
textParts.append(node.tail)
iterTextParts(doc.getroot())
text = " ".join(textParts)
except (IOError, EnvironmentError) as err:
dts.info(
"xpDB:primaryDocumentLoadingError",
_("Loading XBRL DB: primary document loading error: %(error)s"),
modelObject=dts,
error=err,
)
text = ""
# with open("/Users/hermf/temp/test.txt", "w", encoding='utf-8') as fh:
# fh.write(text)
class Span:
def __init__(self, start):
self.start = start
self.end = -1
# find the parts
partSpan = {}
partPrev = None
missing2ndPart1 = False
for partMatch in partPattern.finditer(text):
part = partMatch.group(1).upper()
if partPrev is not None:
if part != "I" and part == partPrev:
# two of these parts without second part 1, use signature or first item for 2nd part 1
missing2ndPart1 = True
partSpan[partPrev].end = partMatch.start(0)
partSpan[part] = Span(partMatch.end(1))
partPrev = part
if partPrev is not None:
partSpan[partPrev].end = len(text)
if missing2ndPart1:
# signatures
signaturesStarts = []
for signaturesMatch in signaturesPattern.finditer(text):
signaturesStarts.append(signaturesMatch.start(0))
# check if PART I missing then use first signatures
if "I" in partSpan and "II" in partSpan:
if len(signaturesStarts) == 2 and signaturesStarts[0] > partSpan["I"].start:
partSpan["I"].start = signaturesStarts[0]
partSpan["I"].end = partSpan["II"].start
else:
# use ASSETS as start of part 1
for assetsMatch in assetsPattern.finditer(text):
partSpan["I"].start = assetsMatch.end(0)
partSpan["I"].end = partSpan["II"].start
break
# find the items
itemSpan = {}
for part, span in partSpan.items():
item = None
for itemMatch in itemPattern.finditer(text, span.start, span.end):
if item is not None:
itemSpan[(part, item)].end = itemMatch.start(0)
item = itemMatch.group(1)
itemSpan[(part, item)] = Span(itemMatch.end(1))
if item is not None:
itemSpan[(part, item)].end = span.end
if any(itemKey in parts for itemKey in itemSpan.keys()):
# find default context
for cntx in dts.contexts.values():
if cntx.isStartEndPeriod:
if not cntx.hasSegment:
# use c as default context
# load extra datapoints taxonomy but not as discovered in DTS
ModelDocument.load(dts, "http://arelle.org/2014/doc-2014-01-31.xsd")
# add the facts
for itemKey, itemSpan in itemSpan.items():
if itemKey in parts:
dts.createFact(
qname("{http://arelle.org/doc/2014-01-31}doc:" + parts[itemKey]),
attributes=[("contextRef", cntx.id)],
text=text[itemSpan.start : itemSpan.end],
)
break
pass
uuIn.close()
uuOut.close()
text = u''.join(pageText)
else:
match = re.search(u"<TEXT>(.*)</TEXT>", s, re.DOTALL)
if match:
text = match.group(1)
except (IOError, EnvironmentError):
pass # give up, no instance
# daily rss item loader, provide unpopulated instance document to be filled in by RssItem.Xbrl.Loaded
if not text:
rssItem.doNotProcessRSSitem = True # skip this RSS item in validate loop, don't load DB
instDoc = ModelDocument.create(modelXbrl,
ModelDocument.Type.UnknownXML,
rssItem.url,
isEntry=True,
base=u'', # block pathname from becomming absolute
initialXml=u'<DummyXml/>')
else:
instDoc = ModelDocument.create(modelXbrl,
ModelDocument.Type.INSTANCE,
rssItem.url,
isEntry=True,
base=u'', # block pathname from becomming absolute
initialXml=u'''
<xbrli:xbrl xmlns:doc="http://arelle.org/doc/2014-01-31"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance">
<link:schemaRef xlink:type="simple" xlink:href="http://arelle.org/2014/doc-2014-01-31.xsd"/>
<xbrli:context id="pubDate">
def init(modelXbrl):
# setup modelXbrl for rendering evaluation
# dimension defaults required in advance of validation
from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
hasXbrlTables = False
# validate table linkbase dimensions
for baseSetKey in modelXbrl.baseSets.keys():
arcrole, ELR, linkqname, arcqname = baseSetKey
if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(arcrole):
ValidateFormula.checkBaseSet(modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname))
if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011):
hasXbrlTables = True
# provide context for view
if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE:
instance = None # use instance of the entry pont
else: # need dummy instance
instance = ModelDocument.create(modelXbrl, ModelDocument.Type.INSTANCE,
"dummy.xml", # fake URI and fake schemaRef
("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd",))
if hasXbrlTables:
# formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables
modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance)
modelXbrl.profileStat(None)
# setup fresh parameters from formula options
modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters()
# validate parameters and custom function signatures
ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables"))
# deprecated as of 2013-05-17
# check and extract message expressions into compilable programs
for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301, XbrlConst.tableDefinitionNodeSelectionMessage201301,
XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011):
for msgRel in modelXbrl.relationshipSet(msgArcrole).modelRelationships:
ValidateFormula.checkMessageExpressions(modelXbrl, msgRel.toModelObject)
# compile and validate tables
for modelTable in modelXbrl.modelRenderingTables:
modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining
modelTable.compile()
hasNsWithAspectModel = modelTable.namespaceURI in (XbrlConst.euRend, XbrlConst.table2011, XbrlConst.table201301, XbrlConst.table201305)
# check aspectModel (attribute removed 2013-06, now always dimensional)
if modelTable.aspectModel not in ("non-dimensional", "dimensional") and hasNsWithAspectModel:
modelXbrl.error("xbrlte:unknownAspectModel",
_("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"),
modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel)
else:
modelTable.priorAspectAxisDisposition = {}
# check ordinate aspects against aspectModel
oppositeAspectModel = (_DICT_SET({'dimensional','non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop()
if hasNsWithAspectModel:
uncoverableAspects = aspectModels[oppositeAspectModel] - aspectModels[modelTable.aspectModel]
else:
uncoverableAspects = ()
aspectsCovered = set()
for tblAxisRel in modelXbrl.relationshipSet((XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301,XbrlConst.tableAxis2011)).fromModelObject(modelTable):
breakdownAspectsCovered = set()
hasCoveredAspect = checkBreakdownDefinitionNode(modelXbrl, modelTable, tblAxisRel, tblAxisRel.axisDisposition, uncoverableAspects, breakdownAspectsCovered)
''' removed 2013-10
if not hasCoveredAspect:
definitionNode = tblAxisRel.toModelObject
modelXbrl.error("xbrlte:breakdownDefinesNoAspects",
_("Breakdown %(xlinkLabel)s has no participating aspects"),
modelObject=(modelTable,definitionNode), xlinkLabel=definitionNode.xlinkLabel, axis=definitionNode.localName)
'''
aspectsCovered |= breakdownAspectsCovered
checkBreakdownLeafNodeAspects(modelXbrl, modelTable, tblAxisRel, set(), breakdownAspectsCovered)
if Aspect.CONCEPT not in aspectsCovered and not hasNsWithAspectModel:
modelXbrl.error("xbrlte:tableMissingConceptAspect",
_("Table %(xlinkLabel)s does not include the concept aspect as one of its participating aspects"),
modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel)
del modelTable.priorAspectAxisDisposition
# check for table-parameter name clash
parameterNames = {}
for tblParamRel in modelXbrl.relationshipSet((XbrlConst.tableParameter, XbrlConst.tableParameterMMDD)).fromModelObject(modelTable):
parameterName = tblParamRel.variableQname
if parameterName in parameterNames:
modelXbrl.error("xbrlte:tableParameterNameClash ",
_("Table %(xlinkLabel)s has parameter name clash for variable %(name)s"),
modelObject=(modelTable,tblParamRel,parameterNames[parameterName]), xlinkLabel=modelTable.xlinkLabel, name=parameterName)
else:
parameterNames[parameterName] = tblParamRel
modelXbrl.profileStat(_("compileTables"))
def secCorrespondenceLoader(modelXbrl, mappedUri, filepath, **kwargs):
if (mappedUri.startswith("http://www.sec.gov/Archives/edgar/Feed/")
and mappedUri.endswith(".nc.tar.gz")):
# daily feed loader (the rss object)
rssObject = ModelRssObject(modelXbrl, uri=mappedUri, filepath=filepath)
# location for expanded feed files
tempdir = os.path.join(modelXbrl.modelManager.cntlr.userAppDir, "tmp",
"edgarFeed")
# remove prior files
if os.path.exists(tempdir):
os.system("rm -fr {}".format(
tempdir)) # rmtree does not work with this many files!
os.makedirs(tempdir, exist_ok=True)
# untar to /temp/arelle/edgarFeed for faster operation
startedAt = time.time()
modelXbrl.fileSource.open()
modelXbrl.fileSource.fs.extractall(tempdir)
modelXbrl.info("info",
"untar edgarFeed temp files in %.2f sec" %
(time.time() - startedAt),
modelObject=modelXbrl)
# find <table> with <a>Download in it
for instanceFile in sorted(
os.listdir(tempdir)): # modelXbrl.fileSource.dir:
if instanceFile != ".":
rssObject.rssItems.append(
SECCorrespondenceItem(modelXbrl, instanceFile,
mappedUri + '/' + instanceFile))
return rssObject
elif "rssItem" in kwargs and ".nc.tar.gz/" in mappedUri:
rssItem = kwargs["rssItem"]
text = None # no instance information
# parse document
try:
startedAt = time.time()
file, encoding = modelXbrl.fileSource.file(
os.path.join(modelXbrl.modelManager.cntlr.userAppDir, "tmp",
"edgarFeed", os.path.basename(rssItem.url)))
s = file.read()
file.close()
for match in re.finditer(r"[<]([^>]+)[>]([^<\n\r]*)", s,
re.MULTILINE):
tag = match.group(1).lower()
v = match.group(2)
if tag == "accession-number":
rssItem.accessionNumber = v
elif tag == "form-type":
rssItem.formType = v
if v != "UPLOAD":
rssItem.doNotProcessRSSitem = True # skip this RSS item in validate loop, don't load DB
elif tag == "filing-date":
try:
rssItem.filingDate = datetime.date(
int(v[0:4]), int(v[4:6]), int(v[6:8]))
except (ValueError, IndexError):
pass
elif tag == "conformed-name":
rssItem.companyName = v
elif tag == "cik":
rssItem.cikNumber = v
elif tag == "assigned-sic":
rssItem.assignedSic = v
elif tag == "fiscal-year-end":
try:
rssItem.fiscalYearEnd = v[0:2] + '-' + v[2:4]
except (IndexError, TypeError):
pass
match = re.search("<PDF>(.*)</PDF>", s, re.DOTALL)
if match:
import uu, io
pageText = []
uuIn = io.BytesIO(match.group(1).encode(encoding))
uuOut = io.BytesIO()
uu.decode(uuIn, uuOut)
from pyPdf import PdfFileReader
uuOut.seek(0, 0)
try:
pdfIn = PdfFileReader(uuOut)
for pageNum in range(pdfIn.getNumPages()):
pageText.append(pdfIn.getPage(pageNum).extractText())
except:
# do we want a warning here that the PDF can't be read with this library?
pass
uuIn.close()
uuOut.close()
text = ''.join(pageText)
else:
match = re.search("<TEXT>(.*)</TEXT>", s, re.DOTALL)
if match:
text = match.group(1)
except (IOError, EnvironmentError):
pass # give up, no instance
# daily rss item loader, provide unpopulated instance document to be filled in by RssItem.Xbrl.Loaded
if not text:
rssItem.doNotProcessRSSitem = True # skip this RSS item in validate loop, don't load DB
instDoc = ModelDocument.create(
modelXbrl,
ModelDocument.Type.UnknownXML,
rssItem.url,
isEntry=True,
base='', # block pathname from becomming absolute
initialXml='<DummyXml/>')
else:
instDoc = ModelDocument.create(
modelXbrl,
ModelDocument.Type.INSTANCE,
rssItem.url,
isEntry=True,
base='', # block pathname from becomming absolute
initialXml='''
<xbrli:xbrl xmlns:doc="http://arelle.org/doc/2014-01-31"
xmlns:link="http://www.xbrl.org/2003/linkbase"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xbrli="http://www.xbrl.org/2003/instance">
<link:schemaRef xlink:type="simple" xlink:href="http://arelle.org/2014/doc-2014-01-31.xsd"/>
<xbrli:context id="pubDate">
<xbrli:entity>
<xbrli:identifier scheme="http://www.sec.gov/CIK">{cik}</xbrli:identifier>
</xbrli:entity>
<xbrli:period>
<xbrli:instant>{pubDate}</xbrli:instant>
</xbrli:period>
</xbrli:context>
<doc:Correspondence contextRef="pubDate">{text}</doc:Correspondence>
</xbrli:xbrl>
'''.format(cik=rssItem.cikNumber,
pubDate=rssItem.pubDate.date(),
text=text.strip().replace("&",
"&").replace("<",
"<")))
#modelXbrl.info("info", "loaded in %.2f sec" % (time.time() - startedAt),
# modelDocument=instDoc)
return instDoc
return None
def validateTestcase(self, testcase):
self.modelXbrl.error(_("testcase {0}").format(os.path.basename(testcase.uri)))
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation.element)
# try to load instance document
self.modelXbrl.error(_("variation {0} {1}: {2}").format(modelTestcaseVariation.id, modelTestcaseVariation.name, modelTestcaseVariation.expected))
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
ModelDocument.Type.DTSENTRIES,
self.modelXbrl.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = ModelDocument.load(modelXbrl, readMeFirstUri, base=baseForElement)
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
else: # not a multi-schemaRef versioning report
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement)
if modelXbrl.modelDocument is None:
self.modelXbrl.error(_("Testcase {0} {1} document not loaded: {2}").format(
modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(readMeFirstUri)))
modelTestcaseVariation.status = "not loadable"
modelXbrl.close()
elif resultIsVersioningReport:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl) \
.validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
modelXbrl.close()
elif testcase.type == ModelDocument.Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
self.modelXbrl.error(_("Testcase {0} {1} DTSes not loaded, unable to generate versioning report").format(
modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(readMeFirstUri)))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes:
inputDTS.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
parameters = modelTestcaseVariation.parameters.copy()
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName:
parameters[dtsName] = (None, inputDTS)
self.instValidator.validate(modelXbrl, parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
if modelXbrl.formulaOutputInstance and len(modelTestcaseVariation.actual) == 0:
# if an output instance is created, validate it
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance)
if len(modelTestcaseVariation.actual) == 0: # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement)
if expectedInstance.modelDocument is None:
self.modelXbrl.error(_("Testcase {0} {1} expected result instance not loaded: {2}").format(
modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(modelTestcaseVariation.resultXbrlInstance)))
modelTestcaseVariation.status = "result not loadable"
expectedInstance.close()
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error(
_("Formula output {0} facts, expected {1} facts").format(
len(formulaOutputInstance.facts),
len(expectedInstance.facts)),
"err", "formula:resultFactCounts")
else:
for fact in expectedInstance.facts:
if not formulaOutputInstance.matchFact(fact):
formulaOutputInstance.error(
_("Formula output missing expected fact {0}").format(
fact),
"err", "formula:expectedFactMissing")
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance)
formulaOutputInstance.close()
formulaOutputInstance = None
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def run(self, options, sourceZipStream=None):
"""Process command line arguments or web service request, such as to load and validate an XBRL document, or start web server.
When a web server has been requested, this method may be called multiple times, once for each web service (REST) request that requires processing.
Otherwise (when called for a command line request) this method is called only once for the command line arguments request.
:param options: OptionParser options from parse_args of main argv arguments (when called from command line) or corresponding arguments from web service (REST) request.
:type options: optparse.Values
"""
if options.showOptions: # debug options
for optName, optValue in sorted(options.__dict__.items(), key=lambda optItem: optItem[0]):
self.addToLog("Option {0}={1}".format(optName, optValue), messageCode="info")
self.addToLog("sys.argv {0}".format(sys.argv), messageCode="info")
if options.uiLang: # set current UI Lang (but not config setting)
self.setUiLanguage(options.uiLang)
if options.proxy:
if options.proxy != "show":
proxySettings = proxyTuple(options.proxy)
self.webCache.resetProxies(proxySettings)
self.config["proxySettings"] = proxySettings
self.saveConfig()
self.addToLog(_("Proxy configuration has been set."), messageCode="info")
useOsProxy, urlAddr, urlPort, user, password = self.config.get("proxySettings", proxyTuple("none"))
if useOsProxy:
self.addToLog(_("Proxy configured to use {0}.").format(
_('Microsoft Windows Internet Settings') if sys.platform.startswith("win")
else (_('Mac OS X System Configuration') if sys.platform in ("darwin", "macos")
else _('environment variables'))), messageCode="info")
elif urlAddr:
self.addToLog(_("Proxy setting: http://{0}{1}{2}{3}{4}").format(
user if user else "",
":****" if password else "",
"@" if (user or password) else "",
urlAddr,
":{0}".format(urlPort) if urlPort else ""), messageCode="info")
else:
self.addToLog(_("Proxy is disabled."), messageCode="info")
if options.plugins:
from arelle import PluginManager
resetPlugins = False
savePluginChanges = True
showPluginModules = False
for pluginCmd in options.plugins.split('|'):
cmd = pluginCmd.strip()
if cmd == "show":
showPluginModules = True
elif cmd == "temp":
savePluginChanges = False
elif cmd.startswith("+"):
moduleInfo = PluginManager.addPluginModule(cmd[1:])
if moduleInfo:
self.addToLog(_("Addition of plug-in {0} successful.").format(moduleInfo.get("name")),
messageCode="info", file=moduleInfo.get("moduleURL"))
resetPlugins = True
else:
self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:])
elif cmd.startswith("~"):
if PluginManager.reloadPluginModule(cmd[1:]):
self.addToLog(_("Reload of plug-in successful."), messageCode="info", file=cmd[1:])
resetPlugins = True
else:
self.addToLog(_("Unable to reload plug-in."), messageCode="info", file=cmd[1:])
elif cmd.startswith("-"):
if PluginManager.removePluginModule(cmd[1:]):
self.addToLog(_("Deletion of plug-in successful."), messageCode="info", file=cmd[1:])
resetPlugins = True
else:
self.addToLog(_("Unable to delete plug-in."), messageCode="info", file=cmd[1:])
else: # assume it is a module or package
savePluginChanges = False
moduleInfo = PluginManager.addPluginModule(cmd)
if moduleInfo:
self.addToLog(_("Activation of plug-in {0} successful.").format(moduleInfo.get("name")),
messageCode="info", file=moduleInfo.get("moduleURL"))
resetPlugins = True
else:
self.addToLog(_("Unable to load {0} as a plug-in or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd)
if resetPlugins:
PluginManager.reset()
if savePluginChanges:
PluginManager.save(self)
if showPluginModules:
self.addToLog(_("Plug-in modules:"), messageCode="info")
for i, moduleItem in enumerate(sorted(PluginManager.pluginConfig.get("modules", {}).items())):
moduleInfo = moduleItem[1]
self.addToLog(_("Plug-in: {0}; author: {1}; version: {2}; status: {3}; date: {4}; description: {5}; license {6}.").format(
moduleItem[0], moduleInfo.get("author"), moduleInfo.get("version"), moduleInfo.get("status"),
moduleInfo.get("fileDate"), moduleInfo.get("description"), moduleInfo.get("license")),
messageCode="info", file=moduleInfo.get("moduleURL"))
# run utility command line options that don't depend on entrypoint Files
hasUtilityPlugin = False
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Utility.Run"):
hasUtilityPlugin = True
pluginXbrlMethod(self, options)
# if no entrypointFile is applicable, quit now
if options.proxy or options.plugins or hasUtilityPlugin:
if not options.entrypointFile:
return True # success
self.username = options.username
self.password = options.password
self.entrypointFile = options.entrypointFile
if self.entrypointFile:
filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream)
else:
filesource = None
if options.validateEFM:
if options.disclosureSystemName:
self.addToLog(_("both --efm and --disclosureSystem validation are requested, proceeding with --efm only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("efm")
elif options.disclosureSystemName:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select(options.disclosureSystemName)
elif options.validateHMRC:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("hmrc")
else:
self.modelManager.disclosureSystem.select(None) # just load ordinary mappings
self.modelManager.validateDisclosureSystem = False
if options.utrUrl: # override disclosureSystem utrUrl
self.modelManager.disclosureSystem.utrUrl = options.utrUrl
# can be set now because the utr is first loaded at validation time
# disclosure system sets logging filters, override disclosure filters, if specified by command line
if options.logLevelFilter:
self.setLogLevelFilter(options.logLevelFilter)
if options.logCodeFilter:
self.setLogCodeFilter(options.logCodeFilter)
if options.calcDecimals:
if options.calcPrecision:
self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateInferDecimals = True
self.modelManager.validateCalcLB = True
elif options.calcPrecision:
self.modelManager.validateInferDecimals = False
self.modelManager.validateCalcLB = True
if options.utrValidate:
self.modelManager.validateUtr = True
if options.infosetValidate:
self.modelManager.validateInfoset = True
if options.abortOnMajorError:
self.modelManager.abortOnMajorError = True
if options.collectProfileStats:
self.modelManager.collectProfileStats = True
if options.internetConnectivity == "offline":
self.webCache.workOffline = True
elif options.internetConnectivity == "online":
self.webCache.workOffline = False
if options.internetTimeout is not None:
self.webCache.timeout = (options.internetTimeout or None) # use None if zero specified to disable timeout
fo = FormulaOptions()
if options.parameters:
parameterSeparator = (options.parameterSeparator or ',')
fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value))
for param in options.parameters.split(parameterSeparator)
for key,sep,value in (param.partition('='),) ) )
if options.formulaParamExprResult:
fo.traceParameterExpressionResult = True
if options.formulaParamInputValue:
fo.traceParameterInputValue = True
if options.formulaCallExprSource:
fo.traceCallExpressionSource = True
if options.formulaCallExprCode:
fo.traceCallExpressionCode = True
if options.formulaCallExprEval:
fo.traceCallExpressionEvaluation = True
if options.formulaCallExprResult:
fo.traceCallExpressionResult = True
if options.formulaVarSetExprEval:
fo.traceVariableSetExpressionEvaluation = True
if options.formulaVarSetExprResult:
fo.traceVariableSetExpressionResult = True
if options.formulaAsserResultCounts:
fo.traceAssertionResultCounts = True
if options.formulaFormulaRules:
fo.traceFormulaRules = True
if options.formulaVarsOrder:
fo.traceVariablesOrder = True
if options.formulaVarExpressionSource:
fo.traceVariableExpressionSource = True
if options.formulaVarExpressionCode:
fo.traceVariableExpressionCode = True
if options.formulaVarExpressionEvaluation:
fo.traceVariableExpressionEvaluation = True
if options.formulaVarExpressionResult:
fo.traceVariableExpressionResult = True
if options.timeVariableSetEvaluation:
fo.timeVariableSetEvaluation = True
if options.formulaVarFilterWinnowing:
fo.traceVariableFilterWinnowing = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
self.modelManager.formulaOptions = fo
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
firstStartedAt = startedAt = time.time()
modelDiffReport = None
success = True
modelXbrl = None
try:
if filesource:
modelXbrl = self.modelManager.load(filesource, _("views loading"))
except ModelDocument.LoadingException:
pass
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False # loading errors, don't attempt to utilize loaded DTS
if modelXbrl and modelXbrl.modelDocument:
loadTime = time.time() - startedAt
modelXbrl.profileStat(_("load"), loadTime)
self.addToLog(format_string(self.modelManager.locale,
_("loaded in %.2f secs at %s"),
(loadTime, timeNow)),
messageCode="info", file=self.entrypointFile)
if options.importFiles:
for importFile in options.importFiles.split("|"):
fileName = importFile.strip()
if sourceZipStream is not None and not (fileName.startswith('http://') or os.path.isabs(fileName)):
fileName = os.path.dirname(modelXbrl.uri) + os.sep + fileName # make relative to sourceZipStream
ModelDocument.load(modelXbrl, fileName)
loadTime = time.time() - startedAt
self.addToLog(format_string(self.modelManager.locale,
_("import in %.2f secs at %s"),
(loadTime, timeNow)),
messageCode="info", file=importFile)
modelXbrl.profileStat(_("import"), loadTime)
if modelXbrl.errors:
success = False # loading errors, don't attempt to utilize loaded DTS
if modelXbrl.modelDocument.type in ModelDocument.Type.TESTCASETYPES:
for pluginXbrlMethod in pluginClassMethods("Testcases.Start"):
pluginXbrlMethod(self, options, modelXbrl)
else: # not a test case, probably instance or DTS
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Loaded"):
pluginXbrlMethod(self, options, modelXbrl)
else:
success = False
if success and options.diffFile and options.versReportFile:
try:
diffFilesource = FileSource.FileSource(options.diffFile,self)
startedAt = time.time()
modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading"))
if modelXbrl2.errors:
if not options.keepOpen:
modelXbrl2.close()
success = False
else:
loadTime = time.time() - startedAt
modelXbrl.profileStat(_("load"), loadTime)
self.addToLog(format_string(self.modelManager.locale,
_("diff comparison DTS loaded in %.2f secs"),
loadTime),
messageCode="info", file=self.entrypointFile)
startedAt = time.time()
modelDiffReport = self.modelManager.compareDTSes(options.versReportFile)
diffTime = time.time() - startedAt
modelXbrl.profileStat(_("diff"), diffTime)
self.addToLog(format_string(self.modelManager.locale,
_("compared in %.2f secs"),
diffTime),
messageCode="info", file=self.entrypointFile)
except ModelDocument.LoadingException:
success = False
except Exception as err:
success = False
self.addToLog(_("[Exception] Failed to doad diff file: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
if success:
try:
modelXbrl = self.modelManager.modelXbrl
hasFormulae = modelXbrl.hasFormulae
if options.validate:
startedAt = time.time()
if options.formulaAction: # don't automatically run formulas
modelXbrl.hasFormulae = False
self.modelManager.validate()
if options.formulaAction: # restore setting
modelXbrl.hasFormulae = hasFormulae
self.addToLog(format_string(self.modelManager.locale,
_("validated in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if options.formulaAction in ("validate", "run"): # do nothing here if "none"
from arelle import ValidateXbrlDimensions, ValidateFormula
startedAt = time.time()
if not options.validate:
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
# setup fresh parameters from formula optoins
modelXbrl.parameters = fo.typedParameters()
ValidateFormula.validate(modelXbrl, compileOnly=(options.formulaAction != "run"))
self.addToLog(format_string(self.modelManager.locale,
_("formula validation and execution in %.2f secs")
if options.formulaAction == "run"
else _("formula validation only in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if options.testReport:
ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport, options.testReportCols)
if options.rssReport:
ViewFileRssFeed.viewRssFeed(self.modelManager.modelXbrl, options.rssReport, options.rssReportCols)
if options.DTSFile:
ViewFileDTS.viewDTS(modelXbrl, options.DTSFile)
if options.factsFile:
ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols)
if options.factTableFile:
ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang)
if options.conceptsFile:
ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang)
if options.preFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang)
if options.calFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang)
if options.dimFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang)
if options.formulaeFile:
ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang)
if options.viewArcrole and options.viewFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.viewFile, os.path.basename(options.viewArcrole), options.viewArcrole, labelrole=options.labelRole, lang=options.labelLang)
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"):
pluginXbrlMethod(self, options, modelXbrl)
except (IOError, EnvironmentError) as err:
self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err))
success = False
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False
if modelXbrl:
modelXbrl.profileStat(_("total"), time.time() - firstStartedAt)
if options.collectProfileStats and modelXbrl:
modelXbrl.logProfileStats()
if not options.keepOpen:
if modelDiffReport:
self.modelManager.close(modelDiffReport)
elif modelXbrl:
self.modelManager.close(modelXbrl)
self.username = self.password = None #dereference password
return success
def init(modelXbrl):
# setup modelXbrl for rendering evaluation
# dimension defaults required in advance of validation
from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
hasXbrlTables = False
# validate table linkbase dimensions
for baseSetKey in modelXbrl.baseSets.keys():
arcrole, ELR, linkqname, arcqname = baseSetKey
if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(
arcrole):
ValidateFormula.checkBaseSet(
modelXbrl, arcrole, ELR,
modelXbrl.relationshipSet(arcrole, ELR, linkqname, arcqname))
if arcrole in (XbrlConst.tableBreakdown,
XbrlConst.tableBreakdown201301,
XbrlConst.tableAxis2011):
hasXbrlTables = True
# provide context for view
if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE:
instance = None # use instance of the entry pont
else: # need dummy instance
instance = ModelDocument.create(
modelXbrl,
ModelDocument.Type.INSTANCE,
"dummy.xml", # fake URI and fake schemaRef
("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd", ))
if hasXbrlTables:
# formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables
modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance)
modelXbrl.profileStat(None)
# setup fresh parameters from formula optoins
modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters(
)
# validate parameters and custom function signatures
ValidateFormula.validate(modelXbrl,
xpathContext=modelXbrl.rendrCntx,
parametersOnly=True,
statusMsg=_("compiling rendering tables"))
# deprecated as of 2013-05-17
# check and extract message expressions into compilable programs
for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301,
XbrlConst.tableDefinitionNodeSelectionMessage201301,
XbrlConst.tableAxisMessage2011,
XbrlConst.tableAxisSelectionMessage2011):
for msgRel in modelXbrl.relationshipSet(
msgArcrole).modelRelationships:
ValidateFormula.checkMessageExpressions(
modelXbrl, msgRel.toModelObject)
# compile and validate tables
for modelTable in modelXbrl.modelRenderingTables:
modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining
modelTable.compile()
# check aspectModel
if modelTable.aspectModel not in ("non-dimensional",
"dimensional"):
modelXbrl.error(
"xbrlte:unknownAspectModel",
_("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"
),
modelObject=modelTable,
xlinkLabel=modelTable.xlinkLabel,
aspectModel=modelTable.aspectModel)
else:
modelTable.priorAspectAxisDisposition = {}
# check ordinate aspects against aspectModel
oppositeAspectModel = (
_DICT_SET({'dimensional', 'non-dimensional'}) -
_DICT_SET({modelTable.aspectModel})).pop()
uncoverableAspects = aspectModels[
oppositeAspectModel] - aspectModels[modelTable.aspectModel]
for tblAxisRel in modelXbrl.relationshipSet(
(XbrlConst.tableBreakdown, XbrlConst.tableBreakdown201301,
XbrlConst.tableAxis2011)).fromModelObject(modelTable):
checkDefinitionNodeAspectModel(modelXbrl, modelTable,
tblAxisRel,
uncoverableAspects)
del modelTable.priorAspectAxisDisposition
modelXbrl.profileStat(_("compileTables"))
def init(modelXbrl):
# setup modelXbrl for rendering evaluation
# dimension defaults required in advance of validation
from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
hasXbrlTables = False
# validate table linkbase dimensions
for baseSetKey in modelXbrl.baseSets.keys():
arcrole, ELR, linkqname, arcqname = baseSetKey
if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(
arcrole):
ValidateFormula.checkBaseSet(
modelXbrl, arcrole, ELR,
modelXbrl.relationshipSet(arcrole, ELR, linkqname, arcqname))
if arcrole in (XbrlConst.tableBreakdown,
XbrlConst.tableBreakdownMMDD,
XbrlConst.tableBreakdown201305,
XbrlConst.tableBreakdown201301,
XbrlConst.tableAxis2011):
hasXbrlTables = True
# provide context for view
if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE:
instance = None # use instance of the entry pont
else: # need dummy instance
instance = ModelDocument.create(
modelXbrl,
ModelDocument.Type.INSTANCE,
"dummy.xml", # fake URI and fake schemaRef
("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd", ))
if hasXbrlTables:
# formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables
modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance)
modelXbrl.profileStat(None)
# setup fresh parameters from formula options
modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters(
)
# validate parameters and custom function signatures
ValidateFormula.validate(modelXbrl,
xpathContext=modelXbrl.rendrCntx,
parametersOnly=True,
statusMsg=_("compiling rendering tables"))
# deprecated as of 2013-05-17
# check and extract message expressions into compilable programs
for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301,
XbrlConst.tableDefinitionNodeSelectionMessage201301,
XbrlConst.tableAxisMessage2011,
XbrlConst.tableAxisSelectionMessage2011):
for msgRel in modelXbrl.relationshipSet(
msgArcrole).modelRelationships:
ValidateFormula.checkMessageExpressions(
modelXbrl, msgRel.toModelObject)
# compile and validate tables
for modelTable in modelXbrl.modelRenderingTables:
modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining
modelTable.compile()
hasNsWithAspectModel = modelTable.namespaceURI in (
XbrlConst.euRend, XbrlConst.table2011, XbrlConst.table201301,
XbrlConst.table201305)
# check aspectModel (attribute removed 2013-06, now always dimensional)
if modelTable.aspectModel not in (
"non-dimensional", "dimensional") and hasNsWithAspectModel:
modelXbrl.error(
"xbrlte:unknownAspectModel",
_("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"
),
modelObject=modelTable,
xlinkLabel=modelTable.xlinkLabel,
aspectModel=modelTable.aspectModel)
else:
modelTable.priorAspectAxisDisposition = {}
# check ordinate aspects against aspectModel
oppositeAspectModel = (
_DICT_SET({'dimensional', 'non-dimensional'}) -
_DICT_SET({modelTable.aspectModel})).pop()
if hasNsWithAspectModel:
uncoverableAspects = aspectModels[
oppositeAspectModel] - aspectModels[
modelTable.aspectModel]
else:
uncoverableAspects = ()
aspectsCovered = set()
for tblAxisRel in modelXbrl.relationshipSet(
(XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD,
XbrlConst.tableBreakdown201305,
XbrlConst.tableBreakdown201301,
XbrlConst.tableAxis2011)).fromModelObject(modelTable):
breakdownAspectsCovered = set()
hasCoveredAspect = checkBreakdownDefinitionNode(
modelXbrl, modelTable, tblAxisRel,
tblAxisRel.axisDisposition, uncoverableAspects,
breakdownAspectsCovered)
''' removed 2013-10
if not hasCoveredAspect:
definitionNode = tblAxisRel.toModelObject
modelXbrl.error("xbrlte:breakdownDefinesNoAspects",
_("Breakdown %(xlinkLabel)s has no participating aspects"),
modelObject=(modelTable,definitionNode), xlinkLabel=definitionNode.xlinkLabel, axis=definitionNode.localName)
'''
aspectsCovered |= breakdownAspectsCovered
checkBreakdownLeafNodeAspects(modelXbrl, modelTable,
tblAxisRel, set(),
breakdownAspectsCovered)
if Aspect.CONCEPT not in aspectsCovered and not hasNsWithAspectModel:
modelXbrl.error(
"xbrlte:tableMissingConceptAspect",
_("Table %(xlinkLabel)s does not include the concept aspect as one of its participating aspects"
),
modelObject=modelTable,
xlinkLabel=modelTable.xlinkLabel)
del modelTable.priorAspectAxisDisposition
# check for table-parameter name clash
parameterNames = {}
for tblParamRel in modelXbrl.relationshipSet(
(XbrlConst.tableParameter, XbrlConst.tableParameterMMDD
)).fromModelObject(modelTable):
parameterName = tblParamRel.variableQname
if parameterName in parameterNames:
modelXbrl.error(
"xbrlte:tableParameterNameClash ",
_("Table %(xlinkLabel)s has parameter name clash for variable %(name)s"
),
modelObject=(modelTable, tblParamRel,
parameterNames[parameterName]),
xlinkLabel=modelTable.xlinkLabel,
name=parameterName)
else:
parameterNames[parameterName] = tblParamRel
modelXbrl.profileStat(_("compileTables"))
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s"),
modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected)
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
ModelDocument.Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = ModelDocument.load(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
else: # not a multi-schemaRef versioning report
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource)
if modelXbrl.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelXbrl.close()
self.determineNotLoadedTestStatus(modelTestcaseVariation)
elif resultIsVersioningReport:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
modelXbrl.close()
elif testcase.type == ModelDocument.Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
parameters = modelTestcaseVariation.parameters.copy()
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName:
parameters[dtsName] = (None, inputDTS)
self.instValidator.validate(modelXbrl, parameters)
if modelTestcaseVariation.resultIsInfoset:
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource)
if infoset.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
self.determineTestStatus(modelTestcaseVariation, modelXbrl) # include infoset errors in status
self.instValidator.close()
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result not loadable"
expectedInstance.close()
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("formula:resultFactCounts",
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts))
else:
for fact in expectedInstance.facts:
if formulaOutputInstance.matchFact(fact) is None:
formulaOutputInstance.error("formula:expectedFactMissing",
_("Formula output missing expected fact %(fact)s"),
modelXbrl=fact, fact=fact.qname)
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance)
formulaOutputInstance.close()
formulaOutputInstance = None
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
