def runFromXml(self):
testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\Arelle\roland test cases\1000-Concepts\index.xml"
filesource = FileSource.FileSource(testGenFileName)
startedAt = time.time()
LogHandler(self) # start logger
modelTestcases = self.modelManager.load(filesource, _("views loading"))
self.addToLog(
_("[info] loaded in {0:.2} secs").format(time.time() - startedAt))
if modelTestcases.modelDocument.type == ModelDocument.Type.TESTCASESINDEX:
for testcasesElement in modelTestcases.modelDocument.iter(
tag="testcases"):
rootAttr = testcasesElement.get("root")
title = testcasesElement.get("title")
self.addToLog(_("[info] testcases {0}").format(title))
if rootAttr is not None:
base = os.path.join(
os.path.dirname(modelTestcases.modelDocument.filepath),
rootAttr) + os.sep
else:
base = self.filepath
for testcaseElement in testcasesElement.iterchildren(
tag="testcase"):
uriFrom = testcaseElement.get("uriFrom")
uriTo = testcaseElement.get("uriTo")
modelDTSfrom = modelDTSto = None
self.addToLog(
_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom is not None and uriTo is not None:
modelDTSfrom = ModelXbrl.load(
modelTestcases.modelManager,
uriFrom,
_("loading from DTS"),
base=base)
modelDTSto = ModelXbrl.load(
modelTestcases.modelManager,
uriTo,
_("loading to DTS"),
base=base)
if modelDTSfrom is not None and modelDTSto is not None:
# generate differences report
reportName = os.path.basename(uriFrom).replace(
"from.xsd", "report.xml")
reportFile = os.path.dirname(
uriFrom) + "\\report\\" + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile, base)
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
ModelVersReport.ModelVersReport(
modelTestcases).diffDTSes(
reportFullPath, modelDTSfrom, modelDTSto)
def load(self, filesource, nextaction=None):
"""Load an entry point modelDocument object(s), which in turn load documents they discover
(for the case of instance, taxonomies, and versioning reports), but defer loading instances
for test case and RSS feeds.
The modelXbrl that is loaded is 'stacked', by this class, so that any modelXbrl operations such as validate,
and close, operate on the most recently loaded modelXbrl, and compareDTSes operates on the two
most recently loaded modelXbrl's.
:param filesource: may be a FileSource object, with the entry point selected, or string file name (or web URL).
:type filesource: FileSource or str
:param nextAction: status line text string, if any, to show upon completion
:type nextAction: str
"""
try:
if filesource.url.startswith("urn:uuid:"): # request for an open modelXbrl
for modelXbrl in self.loadedModelXbrls:
if not modelXbrl.isClosed and modelXbrl.uuid == filesource.url:
return modelXbrl
raise IOError(_("Open file handle is not open: {0}").format(filesource.url))
except AttributeError:
pass # filesource may be a string, which has no url attribute
self.filesource = filesource
modelXbrl = None # loaded modelXbrl
for customLoader in pluginClassMethods("ModelManager.Load"):
modelXbrl = customLoader(self, filesource)
if modelXbrl is not None:
break # custom loader did the loading
if modelXbrl is None: # use default xbrl loader
modelXbrl = ModelXbrl.load(self, filesource, nextaction)
self.modelXbrl = modelXbrl
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def validateRssFeed(self):
self.modelXbrl.info("info", "RSS Feed", modelDocument=self.modelXbrl)
from arelle.FileSource import openFileSource
for rssItem in self.modelXbrl.modelDocument.rssItems:
self.modelXbrl.info("info", _("RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"),
modelObject=rssItem, accessionNumber=rssItem.accessionNumber, formType=rssItem.formType, companyName=rssItem.companyName, period=rssItem.period)
modelXbrl = None
try:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
openFileSource(rssItem.zippedUrl, self.modelXbrl.modelManager.cntlr),
_("validating"))
self.instValidator.validate(modelXbrl, self.modelXbrl.modelManager.formulaOptions.typedParameters())
self.instValidator.close()
rssItem.setResults(modelXbrl)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, rssItem.objectId())
for pluginXbrlMethod in pluginClassMethods("Validate.RssItem"):
pluginXbrlMethod(self, modelXbrl, rssItem)
modelXbrl.close()
except Exception as err:
self.modelXbrl.error("exception",
_("RSS item validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=(self.modelXbrl, modelXbrl),
instance=rssItem.zippedUrl, error=err,
exc_info=True)
try:
self.instValidator.close()
if modelXbrl is not None:
modelXbrl.close()
except Exception as err:
pass
del modelXbrl # completely dereference
def load(self, filesource, nextaction=None):
"""Load an entry point modelDocument object(s), which in turn load documents they discover
(for the case of instance, taxonomies, and versioning reports), but defer loading instances
for test case and RSS feeds.
The modelXbrl that is loaded is 'stacked', by this class, so that any modelXbrl operations such as validate,
and close, operate on the most recently loaded modelXbrl, and compareDTSes operates on the two
most recently loaded modelXbrl's.
:param filesource: may be a FileSource object, with the entry point selected, or string file name (or web URL).
:type filesource: FileSource or str
:param nextAction: status line text string, if any, to show upon completion
:type nextAction: str
"""
try:
if filesource.url.startswith("urn:uuid:"): # request for an open modelXbrl
for modelXbrl in self.loadedModelXbrls:
if not modelXbrl.isClosed and modelXbrl.uuid == filesource.url:
return modelXbrl
raise IOError(_("Open file handle is not open: {0}").format(filesource.url))
except AttributeError:
pass # filesource may be a string, which has no url attribute
self.filesource = filesource
modelXbrl = None # loaded modelXbrl
for customLoader in pluginClassMethods("ModelManager.Load"):
modelXbrl = customLoader(self, filesource)
if modelXbrl is not None:
break # custom loader did the loading
if modelXbrl is None: # use default xbrl loader
modelXbrl = ModelXbrl.load(self, filesource, nextaction)
self.modelXbrl = modelXbrl
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def deprecatedConceptDatesFile(modelManager, abbrNs, latestTaxonomyDoc):
cntlr = modelManager.cntlr
_fileName = resourcesFilePath(
modelManager,
abbrNs.partition("/")[0] + "-deprecated-concepts.json")
_deprecatedLabelRole = latestTaxonomyDoc["deprecatedLabelRole"]
_deprecatedDateMatchPattern = latestTaxonomyDoc["deprecationDatePattern"]
if os.path.exists(_fileName):
return _fileName
# load labels and store file name
modelManager.addToLog(_("loading {} deprecated concepts into {}").format(
abbrNs, _fileName),
messageCode="info")
deprecatedConceptDates = {}
from arelle import ModelXbrl
for latestTaxonomyLabelFile in flattenSequence(
latestTaxonomyDoc["deprecatedLabels"]):
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = modelManager.validateDisclosureSystem
modelManager.validateDisclosureSystem = False
deprecationsInstance = ModelXbrl.load(
modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
# load from zip (especially after caching) is incredibly faster
openFileSource(latestTaxonomyLabelFile, cntlr),
_("built deprecations table in cache"))
modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
if deprecationsInstance is None:
modelManager.addToLog(_("%(name)s documentation not loaded"),
messageCode="arelle:notLoaded",
messageArgs={
"modelXbrl": val,
"name": _abbrNs
})
else:
# load deprecations
for labelRel in deprecationsInstance.relationshipSet(
XbrlConst.conceptLabel).modelRelationships:
modelLabel = labelRel.toModelObject
conceptName = labelRel.fromModelObject.name
if modelLabel.role == _deprecatedLabelRole:
match = _deprecatedDateMatchPattern.match(modelLabel.text)
if match is not None:
date = match.group(1)
if date:
deprecatedConceptDates[conceptName] = date
jsonStr = _STR_UNICODE(
json.dumps(
OrderedDict(
((k, v)
for k, v in sorted(deprecatedConceptDates.items())
)), # sort in json file
ensure_ascii=False,
indent=0)) # might not be unicode in 2.7
saveFile(cntlr, _fileName, jsonStr) # 2.7 gets unicode this way
deprecationsInstance.close()
del deprecationsInstance # dereference closed modelXbrl
def runFromXml(self):
testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\Arelle\roland test cases\1000-Concepts\index.xml"
filesource = FileSource.FileSource(testGenFileName)
startedAt = time.time()
LogHandler(self) # start logger
modelTestcases = self.modelManager.load(filesource, _("views loading"))
self.addToLog(_("[info] loaded in {0:.2} secs").format(time.time() - startedAt))
if modelTestcases.modelDocument.type == ModelDocument.Type.TESTCASESINDEX:
for testcasesElement in modelTestcases.modelDocument.iter(tag="testcases"):
rootAttr = testcasesElement.get("root")
title = testcasesElement.get("title")
self.addToLog(_("[info] testcases {0}").format(title))
if rootAttr is not None:
base = os.path.join(os.path.dirname(modelTestcases.modelDocument.filepath),rootAttr) + os.sep
else:
base = self.filepath
for testcaseElement in testcasesElement.iterchildren(tag="testcase"):
uriFrom = testcaseElement.get("uriFrom")
uriTo = testcaseElement.get("uriTo")
modelDTSfrom = modelDTSto = None
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom is not None and uriTo is not None:
modelDTSfrom = ModelXbrl.load(modelTestcases.modelManager,
uriFrom,
_("loading from DTS"),
base=base)
modelDTSto = ModelXbrl.load(modelTestcases.modelManager,
uriTo,
_("loading to DTS"),
base=base)
if modelDTSfrom is not None and modelDTSto is not None:
# generate differences report
reportName = os.path.basename(uriFrom).replace("from.xsd","report.xml")
reportFile = os.path.dirname(uriFrom) + "\\report\\" + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile,
base)
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
ModelVersReport.ModelVersReport(modelTestcases).diffDTSes(
reportFullPath,
modelDTSfrom, modelDTSto)
def _make_cache(val, ugt, cntlr, ugt_default_dimensions_json_file):
"""
Creates a new caches for the Taxonomy default dimensions
:param val: ValidateXbrl to be validated
:type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
:param ugt: Taxonomy to check
:type ugt: str
:param ugt_default_dimensions_json_file: location to save json default
dimensions
:type ugt_default_dimensions_json_file: str
:return: no explicit return, but saves caches for dqc_us_0041
:rtype: None
"""
started_at = time.time()
ugt_entry_xsd = ugt["entryXsd"]
val.usgaapDefaultDimensions = {}
prior_validate_disclosure_system = (
val.modelXbrl.modelManager.validateDisclosureSystem)
val.modelXbrl.modelManager.validateDisclosureSystem = False
ugt_entry_xsd_instance = (
ModelXbrl.load(
val.modelXbrl.modelManager,
openFileSource(ugt_entry_xsd, cntlr),
_("opened us-gaap entry xsd") # noqa
))
val.modelXbrl.modelManager.validateDisclosureSystem = (
prior_validate_disclosure_system)
if ugt_entry_xsd_instance is None:
val.modelXbrl.error(
"arelle:notLoaded",
_("US-GAAP entry xsd not loaded: %(file)s"), # noqa
modelXbrl=val,
file=os.path.basename(ugt_entry_xsd))
else:
model_relationships = (ugt_entry_xsd_instance.relationshipSet(
XbrlConst.dimensionDefault).modelRelationships)
for default_dim_rel in model_relationships:
if _default_dim_rel_is_instance(default_dim_rel):
from_name = default_dim_rel.fromModelObject.name
to_name = default_dim_rel.toModelObject.name
val.usgaapDefaultDimensions[from_name] = to_name
json_str = str(
json.dumps(val.usgaapDefaultDimensions,
ensure_ascii=False,
indent=0)) # might not be unicode in 2.7
# 2.7 gets unicode this way
saveFile(cntlr, ugt_default_dimensions_json_file, json_str)
ugt_entry_xsd_instance.close()
del ugt_entry_xsd_instance # dereference closed modelXbrl
val.modelXbrl.profileStat(
_("build default dimensions cache"), # noqa
time.time() - started_at)
def load(self, filesource, nextaction=None):
try:
if filesource.url.startswith("urn:uuid:"): # request for an open modelXbrl
for modelXbrl in self.loadedModelXbrls:
if not modelXbrl.isClosed and modelXbrl.uuid == filesource.url:
return modelXbrl
raise IOError(_("Open file handle is not open: {0}".format(filesource.url)))
except AttributeError:
pass # filesource may be a string, which has no url attribute
self.filesource = filesource
self.modelXbrl = ModelXbrl.load(self, filesource, nextaction)
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def load(self, filesource, nextaction=None):
try:
if filesource.url.startswith(
"urn:uuid:"): # request for an open modelXbrl
for modelXbrl in self.loadedModelXbrls:
if not modelXbrl.isClosed and modelXbrl.uuid == filesource.url:
return modelXbrl
raise IOError(
_("Open file handle is not open: {0}").format(
filesource.url))
except AttributeError:
pass # filesource may be a string, which has no url attribute
self.filesource = filesource
self.modelXbrl = ModelXbrl.load(self, filesource, nextaction)
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def _create_config(val):
"""
Creates the configs needed for dqc_us_0001
:param val: ValidateXbrl needed in order to save the cache
:type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
:return: no explicit return but creates and saves configs in
dqc_us_rule\resources\DQC_US_0001
:rtype: None
"""
val.ugtNamespace = None
cntlr = val.modelXbrl.modelManager.cntlr
year = _EARLIEST_US_GAAP_YEAR
config = _load_config(_DQC_01_AXIS_FILE)
# Create a list of axes in the base config file
for ugt in _UGT_DOCS:
# create taxonomy specific name
config_json_file = os.path.join(os.path.dirname(__file__), 'resources',
'DQC_US_0001',
'dqc_0001_{}.json'.format(str(year)))
# copy the base config file
working_json_file = config
ugtEntryXsd = ugt["entryXsd"]
prior_vds = val.modelXbrl.modelManager.validateDisclosureSystem
val.modelXbrl.modelManager.validateDisclosureSystem = False
dimLoadingInstance = ModelXbrl.load(val.modelXbrl.modelManager,
openFileSource(ugtEntryXsd, cntlr),
("built us-gaap member cache"))
val.modelXbrl.modelManager.validateDisclosureSystem = prior_vds
for axis, info in config.items():
info['defined_members'] = defaultdict(set)
axisConcept = dimLoadingInstance.nameConcepts.get(axis,
(None, ))[0]
if axisConcept is not None:
working_json_file[axis]['defined_members'] = sorted(
_tr_mem(val, ugt, dimLoadingInstance, axisConcept,
XbrlConst.dimensionDomain, None, set()))
json_str = str(
json.dumps(OrderedDict(sorted(working_json_file.items())),
ensure_ascii=False,
indent=4))
saveFile(cntlr, config_json_file, json_str)
dimLoadingInstance.close()
del dimLoadingInstance
year += 1
def deprecatedConceptDatesFile(modelManager, abbrNs, latestTaxonomyDoc):
if latestTaxonomyDoc is None:
return None
if not abbrNs: # none for an unexpected namespace pattern
return None
cntlr = modelManager.cntlr
_fileName = resourcesFilePath(modelManager, abbrNs.partition("/")[0] + "-deprecated-concepts.json")
_deprecatedLabelRole = latestTaxonomyDoc["deprecatedLabelRole"]
_deprecatedDateMatchPattern = latestTaxonomyDoc["deprecationDatePattern"]
if os.path.exists(_fileName):
return _fileName
# load labels and store file name
modelManager.addToLog(_("loading {} deprecated concepts into {}").format(abbrNs, _fileName), messageCode="info")
deprecatedConceptDates = {}
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = modelManager.validateDisclosureSystem
modelManager.validateDisclosureSystem = False
from arelle import ModelXbrl
deprecationsInstance = ModelXbrl.load(modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
# load from zip (especially after caching) is incredibly faster
openFileSource(latestTaxonomyDoc["deprecatedLabels"], cntlr),
_("built deprecations table in cache"))
modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
if deprecationsInstance is None:
modelManager.addToLog(
_("%(name)s documentation not loaded"),
messageCode="arelle:notLoaded", messageArgs={"modelXbrl": val, "name":_abbrNs})
else:
# load deprecations
for labelRel in deprecationsInstance.relationshipSet(XbrlConst.conceptLabel).modelRelationships:
modelLabel = labelRel.toModelObject
conceptName = labelRel.fromModelObject.name
if modelLabel.role == _deprecatedLabelRole:
match = _deprecatedDateMatchPattern.match(modelLabel.text)
if match is not None:
date = match.group(1)
if date:
deprecatedConceptDates[conceptName] = date
jsonStr = _STR_UNICODE(json.dumps(deprecatedConceptDates, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, _fileName, jsonStr) # 2.7 gets unicode this way
deprecationsInstance.close()
del deprecationsInstance # dereference closed modelXbrl
def validateRssFeed(self):
self.modelXbrl.info("info", "RSS Feed", modelDocument=self.modelXbrl)
from arelle.FileSource import openFileSource
reloadCache = getattr(self.modelXbrl, "reloadCache", False)
for rssItem in self.modelXbrl.modelDocument.rssItems:
if getattr(rssItem, "skipRssItem", False):
self.modelXbrl.info("info", _("skipping RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"),
modelObject=rssItem, accessionNumber=rssItem.accessionNumber, formType=rssItem.formType, companyName=rssItem.companyName, period=rssItem.period)
continue
self.modelXbrl.info("info", _("RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"),
modelObject=rssItem, accessionNumber=rssItem.accessionNumber, formType=rssItem.formType, companyName=rssItem.companyName, period=rssItem.period)
modelXbrl = None
try:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
openFileSource(rssItem.zippedUrl, self.modelXbrl.modelManager.cntlr, reloadCache=reloadCache),
_("validating"), rssItem=rssItem)
for pluginXbrlMethod in pluginClassMethods("RssItem.Xbrl.Loaded"):
pluginXbrlMethod(modelXbrl, {}, rssItem)
if getattr(rssItem, "doNotProcessRSSitem", False) or modelXbrl.modelDocument is None:
modelXbrl.close()
continue # skip entry based on processing criteria
self.instValidator.validate(modelXbrl, self.modelXbrl.modelManager.formulaOptions.typedParameters(self.modelXbrl.prefixedNamespaces))
self.instValidator.close()
rssItem.setResults(modelXbrl)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, rssItem.objectId())
for pluginXbrlMethod in pluginClassMethods("Validate.RssItem"):
pluginXbrlMethod(self, modelXbrl, rssItem)
modelXbrl.close()
except Exception as err:
self.modelXbrl.error("exception:" + type(err).__name__,
_("RSS item validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=(self.modelXbrl, modelXbrl),
instance=rssItem.zippedUrl, error=err,
exc_info=True)
try:
self.instValidator.close()
if modelXbrl is not None:
modelXbrl.close()
except Exception as err:
pass
del modelXbrl # completely dereference
def validateRssFeed(self):
self.modelXbrl.info("info", "RSS Feed", modelDocument=self.modelXbrl)
from arelle.FileSource import openFileSource
reloadCache = getattr(self.modelXbrl, "reloadCache", False)
for rssItem in self.modelXbrl.modelDocument.rssItems:
if getattr(rssItem, "skipRssItem", False):
self.modelXbrl.info("info", _("skipping RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"),
modelObject=rssItem, accessionNumber=rssItem.accessionNumber, formType=rssItem.formType, companyName=rssItem.companyName, period=rssItem.period)
continue
self.modelXbrl.info("info", _("RSS Item %(accessionNumber)s %(formType)s %(companyName)s %(period)s"),
modelObject=rssItem, accessionNumber=rssItem.accessionNumber, formType=rssItem.formType, companyName=rssItem.companyName, period=rssItem.period)
modelXbrl = None
try:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
openFileSource(rssItem.zippedUrl, self.modelXbrl.modelManager.cntlr, reloadCache=reloadCache),
_("validating"), rssItem=rssItem)
for pluginXbrlMethod in pluginClassMethods("RssItem.Xbrl.Loaded"):
pluginXbrlMethod(modelXbrl, {}, rssItem)
if getattr(rssItem, "doNotProcessRSSitem", False) or modelXbrl.modelDocument is None:
modelXbrl.close()
continue # skip entry based on processing criteria
self.instValidator.validate(modelXbrl, self.modelXbrl.modelManager.formulaOptions.typedParameters())
self.instValidator.close()
rssItem.setResults(modelXbrl)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, rssItem.objectId())
for pluginXbrlMethod in pluginClassMethods("Validate.RssItem"):
pluginXbrlMethod(self, modelXbrl, rssItem)
modelXbrl.close()
except Exception as err:
self.modelXbrl.error("exception:" + type(err).__name__,
_("RSS item validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=(self.modelXbrl, modelXbrl),
instance=rssItem.zippedUrl, error=err,
exc_info=True)
try:
self.instValidator.close()
if modelXbrl is not None:
modelXbrl.close()
except Exception as err:
pass
del modelXbrl # completely dereference
def buildUgtFullRelsFiles(modelXbrl, dqcRules):
from arelle import ModelXbrl
modelManager = modelXbrl.modelManager
cntlr = modelXbrl.modelManager.cntlr
conceptRule = (
"http://fasb.org/dqcrules/arcrole/concept-rule", # FASB arcrule
"http://fasb.org/dqcrules/arcrole/rule-concept")
rule0015 = "http://fasb.org/us-gaap/role/dqc/0015"
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = modelManager.validateDisclosureSystem
modelManager.validateDisclosureSystem = False
for ugtAbbr, (ugtEntireUrl, dqcrtUrl) in latestEntireUgt.items():
modelManager.addToLog(_("loading {} Entire UGT {}").format(
ugtAbbr, ugtEntireUrl),
messageCode="info")
ugtRels = {}
ugtRels["calcs"] = ugtCalcs = {}
ugtRels["axes"] = ugtAxes = defaultdict(set)
ugtInstance = ModelXbrl.load(
modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
# load from zip (especially after caching) is incredibly faster
openFileSource(ugtEntireUrl, cntlr),
_("built dqcrt table in cache"))
if ugtInstance is None:
modelManager.addToLog(_("%(name)s documentation not loaded"),
messageCode="arelle:notLoaded",
messageArgs={
"modelXbrl": val,
"name": ugtAbbr
})
else:
# load signwarnings from DQC 0015
calcRelSet = ugtInstance.relationshipSet(XbrlConst.summationItem)
for rel in calcRelSet.modelRelationships:
_fromQn = rel.fromModelObject.qname
_toQn = rel.toModelObject.qname
ugtCalcs.setdefault(rel.weight, {}).setdefault(
_fromQn.prefix,
{}).setdefault(_fromQn.localName,
{}).setdefault(_toQn.prefix,
set()).add(_toQn.localName)
for w in ugtCalcs.values():
for fNs in w.values():
for fLn in fNs.values():
for tNs in fLn.keys():
fLn[tNs] = sorted(
fLn[tNs]) # change set to array for json
dimDomRelSet = ugtInstance.relationshipSet(
XbrlConst.dimensionDomain)
axesOfInterest = set()
for rule in dqcRules["DQC.US.0001"]["rules"].values():
axesOfInterest.add(rule["axis"])
for ruleAxesEntry in ("additional-axes", "unallowed-axes"):
for additionalAxis in rule.get(ruleAxesEntry, ()):
axesOfInterest.add(additionalAxis)
for rel in dimDomRelSet.modelRelationships:
axisConcept = rel.fromModelObject
if axisConcept.name in axesOfInterest:
addDomMems(rel, ugtAxes[axisConcept.name], True)
for axis in tuple(ugtAxes.keys()):
ugtAxes[axis] = sorted(
ugtAxes[axis]) # change set to array for json
ugtInstance.close()
del ugtInstance # dereference closed modelXbrl
if dqcrtUrl: # none for 2019
modelManager.addToLog(_("loading {} DQC Rules {}").format(
ugtAbbr, dqcrtUrl),
messageCode="info")
dqcrtInstance = ModelXbrl.load(
modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
# load from zip (especially after caching) is incredibly faster
openFileSource(dqcrtUrl, cntlr),
_("built dqcrt table in cache"))
if dqcrtInstance is None:
modelManager.addToLog(
_("%(name)s documentation not loaded"),
messageCode="arelle:notLoaded",
messageArgs={
"modelXbrl": val,
"name": ugtAbbr
})
else:
ugtRels["DQC.US.0015"] = dqc0015 = defaultdict(dict)
# load DQC 0015
dqcRelSet = dqcrtInstance.relationshipSet(
(
"http://fasb.org/dqcrules/arcrole/concept-rule", # FASB arcrule
"http://fasb.org/dqcrules/arcrole/rule-concept"),
"http://fasb.org/us-gaap/role/dqc/0015")
for dqc0015obj, headEltName in (
("conceptNames", "Dqc_0015_ListOfElements"),
("excludedMemberNames",
"Dqc_0015_ExcludeNonNegMembersAbstract"),
("excludedAxesMembers",
"Dqc_0015_ExcludeNonNegAxisAbstract"),
("excludedAxesMembers",
"Dqc_0015_ExcludeNonNegAxisMembersAbstract"),
("excludedMemberStrings",
"Dqc_0015_ExcludeNonNegMemberStringsAbstract")):
headElts = dqcrtInstance.nameConcepts.get(
headEltName, ())
for headElt in headElts:
if dqc0015obj == "excludedMemberStrings":
for refRel in dqcrtInstance.relationshipSet(
XbrlConst.conceptReference
).fromModelObject(headElt):
for refPart in refRel.toModelObject.iterchildren(
"{*}allowableSubString"):
for subStr in refPart.text.split():
dqc0015[dqc0015obj].setdefault(
"*", []).append(
subStr
) # applies to any namespace
else:
for ruleRel in dqcRelSet.fromModelObject(
headElt):
elt = ruleRel.toModelObject
if dqc0015obj in ("conceptNames",
"excludedMemberNames"):
dqc0015[dqc0015obj].setdefault(
elt.qname.prefix,
[]).append(elt.name)
else:
l = dqc0015[dqc0015obj].setdefault(
elt.qname.prefix,
{}).setdefault(elt.name, {})
if headEltName == "Dqc_0015_ExcludeNonNegAxisAbstract":
l["*"] = None
else:
for memRel in dqcRelSet.fromModelObject(
elt):
l.setdefault(
memRel.toModelObject.qname.
prefix, []).append(
memRel.toModelObject.
name)
dqc0015["conceptRuleIDs"] = conceptRuleIDs = {}
for rel in dqcrtInstance.relationshipSet(
XbrlConst.conceptReference).modelRelationships:
if rel.toModelObject.role == "http://fasb.org/us-gaap/role/dqc/ruleID":
conceptRuleIDs.setdefault(
elt.qname.prefix,
{})[rel.fromModelObject.name] = int(
rel.toModelObject.stringValue.rpartition(
".")[2])
dqcrtInstance.close()
del dqcrtInstance # dereference closed modelXbrl
def sortDqcLists(obj):
if isinstance(obj, list):
obj.sort()
elif isinstance(obj, dict):
for objVal in obj.values():
sortDqcLists(objVal)
sortDqcLists(dqc0015)
jsonStr = _STR_UNICODE(
json.dumps(ugtRels, ensure_ascii=False,
indent=2)) # might not be unicode in 2.7
_ugtRelsFileName = resourcesFilePath(
modelManager,
"us-gaap-rels-{}.json".format(ugtAbbr.rpartition("/")[2]))
saveFile(cntlr, _ugtRelsFileName,
jsonStr) # 2.7 gets unicode this way
modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
def runFromExcel(self, options):
#testGenFileName = options.excelfilename
testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls"
testGenDir = os.path.dirname(testGenFileName)
schemaDir = os.path.dirname(testGenDir) + os.sep + "schema"
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
LogHandler(self) # start logger
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt'
modelTestcases = ModelXbrl.create(self.modelManager, url=testGenFileName, isEntry=True)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow))
# start index file
indexFiles = [testGenDir + os.sep + 'creation-testcases-index.xml',
testGenDir + os.sep + 'consumption-testcases-index.xml']
indexDocs = []
testcasesElements = []
for purpose in ("Creation","Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 {0} Tests" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today)
)
doc = etree.parse(file)
file.close()
indexDocs.append(doc)
testcasesElements.append(doc.getroot())
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
try:
row = testcaseIndexSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY or # must have directory
row[1].ctype == xlrd.XL_CELL_EMPTY or # from
row[2].ctype == xlrd.XL_CELL_EMPTY): # to
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
overrideReport = row[3].value
description = row[4].value
if description is None or len(description) == 0:
continue # test not ready to run
assignment = row[5].value
expectedEvents = row[6].value # comma space separated if multiple
note = row[7].value
useCase = row[8].value
base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and assignment.lower() not in ("n.a.", "error") and expectedEvents != "N.A.":
modelDTSfrom = modelDTSto = None
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base)
else:
modelDTS = ModelXbrl.create(self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts",
base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS, uri.strip(), base=base)
if doc is not None:
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom is not None and modelDTSto is not None:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace("from.xsd","report.xml")
reportFile = reportDir + "out" + os.sep + reportName
#reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile,
base)
testcasesDir = os.path.dirname(os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml",
testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"]
for i,testcaseFile in enumerate(testcaseFiles):
etree.SubElement(testcasesElements[i], "testcase",
attrib={"uri":
testcaseFile[len(testGenDir)+1:].replace("\\","/")} )
# start testcase file
testcaseDocs = []
testcaseElements = []
testcaseNumber = testcaseName[0:4]
if testcaseNumber.isnumeric():
testcaseNumberElement = "<number>{0}</number>".format(testcaseNumber)
testcaseName = testcaseName[5:]
else:
testcaseNumberElement = ""
testDirSegments = testDir.split('/')
if len(testDirSegments) >= 2 and '-' in testDirSegments[1]:
testedModule = testDirSegments[1][testDirSegments[1].index('-') + 1:]
else:
testedModule = ''
for purpose in ("Creation","Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'{0}'
'<name>{1}</name>'
# '<description>{0}</description>'
'<reference>'
'{2}'
'{3}'
'</reference>'
'</testcase>'.format(testcaseNumberElement,
testcaseName,
'<name>{0}</name>'.format(testedModule) if testedModule else '',
'<id>{0}</id>'.format(useCase) if useCase else '')
)
doc = etree.parse(file)
file.close()
testcaseDocs.append(doc)
testcaseElements.append(doc.getroot())
priorTestcasesDir = testcasesDir
variationSeq = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(modelTestcases)
modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto,
assignment=assignment,
schemaDir=schemaDir)
# check for expected elements
if expectedEvents:
for expectedEvent in expectedEvents.split(","):
if expectedEvent not in ("No change", "N.A."):
prefix, sep, localName = expectedEvent.partition(':')
if sep and len(modelVersReport.xmlDocument.findall(
'//{{{0}}}{1}'.format(
XbrlConst.verPrefixNS.get(prefix),
localName))) == 0:
modelTestcases.warning("warning",
"Generated test case %(reportName)s missing expected event %(event)s",
reportName=reportName,
event=expectedEvent)
modelVersReport.close()
uriFromParts = uriFrom.split('_')
if len(uriFromParts) >= 2:
variationId = uriFromParts[1]
else:
variationId = "_{0:02n}".format(variationSeq)
for i,testcaseElt in enumerate(testcaseElements):
variationElement = etree.SubElement(testcaseElt, "{http://xbrl.org/2008/conformance}variation",
attrib={"id": variationId})
nameElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
nameElement.text = description
''' (removed per RH 2011/10/04
if note:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
paramElement.text = "Note: " + note
if useCase:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference")
paramElement.set("specification", "versioning-requirements")
paramElement.set("useCase", useCase)
'''
dataElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}data")
if i == 0: # result is report
if expectedEvents:
paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter",
attrib={"name":"expectedEvent",
"value":expectedEvents.replace(',',' ')},
nsmap={"conf":"http://xbrl.org/2008/conformance",
None:""})
if assignment:
paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter",
attrib={"name":"assignment",
"value":assignment},
nsmap={"conf":"http://xbrl.org/2008/conformance",
None:""})
for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")):
for schemaURI in schemaURIs.split(","):
schemaElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}schema")
schemaElement.set("dts",dtsAttr)
if i == 0:
schemaElement.set("readMeFirst","true")
schemaElement.text=os.path.basename(schemaURI.strip())
resultElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}result")
reportElement = etree.SubElement(resultElement if i == 0 else dataElement,
"{http://xbrl.org/2008/conformance}versioningReport")
if i == 1:
reportElement.set("readMeFirst","true")
reportElement.text = "report/" + reportName
variationSeq += 1
except Exception as err:
modelTestcases.error("exception",
_("Exception: %(error)s, Excel row: %(excelRow)s"),
error=err,
excelRow=iRow,
exc_info=True)
# add tests-error-code index files to consumption
for testcaseFile in self.testcaseFiles(testGenDir + os.sep + "tests-error-code"):
etree.SubElement(testcasesElements[1], "testcase",
attrib={"uri":
testcaseFile[len(testGenDir)+1:].replace("\\","/")} )
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i,indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def runFromExcel(self, options):
testGenFileName = options.excelfilename
#testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls"
testGenDir = os.path.dirname(testGenFileName)
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt'
modelTestcases = ModelXbrl.create(self.modelManager)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow))
# start index file
indexFiles = [testGenDir + os.sep + 'creationTestcasesIndex.xml',
testGenDir + os.sep + 'consumptionTestcasesIndex.xml']
indexDocs = [xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today)
) for purpose in ("Creation","Consumption")]
testcasesElements = [XmlUtil.child(indexDoc, None, "testcases") for indexDoc in indexDocs]
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
row = testcaseIndexSheet.row(iRow)
if row[0].ctype == xlrd.XL_CELL_EMPTY or row[1].ctype == xlrd.XL_CELL_EMPTY or row[2].ctype == xlrd.XL_CELL_EMPTY:
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
intention = row[3].value
if intention is None or len(intention) == 0:
continue # test not ready to run
reason = row[4].value
expectedEvent = row[5].value
base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and reason.lower() not in ("n.a.", "error") and expectedEvent != "N.A.":
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base)
else:
modelDTS = ModelXbrl.create(self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts",
base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS, uri.strip(), base=base)
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom and modelDTSto:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace("from.xsd","report.xml")
reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile,
base)
testcasesDir = os.path.dirname(os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml",
testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"]
for i,testcaseFile in enumerate(testcaseFiles):
XmlUtil.addChild(testcasesElements[i], None, "testcase",
("uri",
testcaseFile[len(testGenDir)+1:].replace("\\","/")) )
# start testcase file
testcaseDocs = [xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'<name>{0}</name>'
'<description>{0}</description>'
'</testcase>'.format(testcaseName,purpose,today)
) for purpose in ("Creation","Consumption")]
testcaseElements = [XmlUtil.child(testcaseDoc, conformanceNS, "testcase") for testcaseDoc in testcaseDocs]
priorTestcasesDir = testcasesDir
variationID = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(modelTestcases)
modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto)
# check for expected elements
if expectedEvent and expectedEvent not in (
"No change", "N.A."):
if len(modelVersReport.xmlDocument.getElementsByTagNameNS('*',expectedEvent)) == 0:
modelTestcases.error(
"Generated test case {0} missing expected event {1}".format(
reportName,
expectedEvent),
"wrn", "missingEvent")
modelVersReport.close([])
for i,testcaseElt in enumerate(testcaseElements):
variationElement = XmlUtil.addChild(testcaseElt, conformanceNS, "variation",
attributes=("id", "_{0:02n}".format(variationID)))
XmlUtil.addChild(variationElement, conformanceNS, "name", text=intention)
dataElement = XmlUtil.addChild(variationElement, conformanceNS, "data")
for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")):
for schemaURI in schemaURIs.split(","):
XmlUtil.addChild(dataElement, conformanceNS, "schema",
attributes=((("dts",dtsAttr),) +
((("readMeFirst","true"),) if i == 0 else ())),
text=os.path.basename(schemaURI.strip()))
resultElement = XmlUtil.addChild(variationElement, conformanceNS, "result")
XmlUtil.addChild(resultElement if i == 0 else dataElement,
conformanceNS, "versioningReport",
attributes=(("readMeFirst","true") if i == 1 else ()),
text="report/" + reportName)
variationID += 1
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i,indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def setup(val):
val.linroleDefinitionIsDisclosure = re.compile(r"-\s+Disclosure\s+-\s",
re.IGNORECASE)
val.linkroleDefinitionStatementSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*", # no restriction to type of statement
re.IGNORECASE)
val.ugtNamespace = None
cntlr = val.modelXbrl.modelManager.cntlr
# load deprecated concepts for filed year of us-gaap
for ugt in ugtDocs:
ugtNamespace = ugt["namespace"]
if ugtNamespace in val.modelXbrl.namespaceDocs and len(val.modelXbrl.namespaceDocs[ugtNamespace]) > 0:
val.ugtNamespace = ugtNamespace
usgaapDoc = val.modelXbrl.namespaceDocs[ugtNamespace][0]
deprecationsJsonFile = usgaapDoc.filepathdir + os.sep + "deprecated-concepts.json"
file = None
try:
file = openFileStream(cntlr, deprecationsJsonFile, 'rt', encoding='utf-8')
val.usgaapDeprecations = json.load(file)
file.close()
except Exception:
if file:
file.close()
val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} deprecated concepts into cache").format(ugt["year"]))
startedAt = time.time()
ugtDocLB = ugt["docLB"]
val.usgaapDeprecations = {}
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem
val.modelXbrl.modelManager.validateDisclosureSystem = False
deprecationsInstance = ModelXbrl.load(val.modelXbrl.modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
# load from zip (especially after caching) is incredibly faster
openFileSource(ugtDocLB, cntlr),
_("built deprecations table in cache"))
val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
if deprecationsInstance is None:
val.modelXbrl.error("arelle:notLoaded",
_("US-GAAP documentation not loaded: %(file)s"),
modelXbrl=val, file=os.path.basename(ugtDocLB))
else:
# load deprecations
for labelRel in deprecationsInstance.relationshipSet(XbrlConst.conceptLabel).modelRelationships:
modelDocumentation = labelRel.toModelObject
conceptName = labelRel.fromModelObject.name
if modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedLabel':
val.usgaapDeprecations[conceptName] = (val.usgaapDeprecations.get(conceptName, ('',''))[0], modelDocumentation.text)
elif modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedDateLabel':
val.usgaapDeprecations[conceptName] = (modelDocumentation.text, val.usgaapDeprecations.get(conceptName, ('',''))[1])
jsonStr = _STR_UNICODE(json.dumps(val.usgaapDeprecations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, deprecationsJsonFile, jsonStr) # 2.7 gets unicode this way
deprecationsInstance.close()
del deprecationsInstance # dereference closed modelXbrl
val.modelXbrl.profileStat(_("build us-gaap deprecated concepts cache"), time.time() - startedAt)
ugtCalcsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-calculations.json"
ugtDefaultDimensionsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-default-dimensions.json"
file = None
try:
file = openFileStream(cntlr, ugtCalcsJsonFile, 'rt', encoding='utf-8')
val.usgaapCalculations = json.load(file)
file.close()
file = openFileStream(cntlr, ugtDefaultDimensionsJsonFile, 'rt', encoding='utf-8')
val.usgaapDefaultDimensions = json.load(file)
file.close()
except Exception:
if file:
file.close()
val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} calculations and default dimensions into cache").format(ugt["year"]))
startedAt = time.time()
ugtEntryXsd = ugt["entryXsd"]
val.usgaapCalculations = {}
val.usgaapDefaultDimensions = {}
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem
val.modelXbrl.modelManager.validateDisclosureSystem = False
calculationsInstance = ModelXbrl.load(val.modelXbrl.modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/entire/us-gaap-entryPoint-std-2012-01-31.xsd",
# load from zip (especially after caching) is incredibly faster
openFileSource(ugtEntryXsd, cntlr),
_("built us-gaap calculations cache"))
val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
if calculationsInstance is None:
val.modelXbrl.error("arelle:notLoaded",
_("US-GAAP calculations not loaded: %(file)s"),
modelXbrl=val, file=os.path.basename(ugtEntryXsd))
else:
# load calculations
for ELR in calculationsInstance.relationshipSet(XbrlConst.summationItem).linkRoleUris:
elrRelSet = calculationsInstance.relationshipSet(XbrlConst.summationItem, ELR)
definition = ""
for roleType in calculationsInstance.roleTypes.get(ELR,()):
definition = roleType.definition
break
isStatementSheet = bool(val.linkroleDefinitionStatementSheet.match(definition))
elrUgtCalcs = {"#roots": [c.name for c in elrRelSet.rootConcepts],
"#definition": definition,
"#isStatementSheet": isStatementSheet}
for relFrom, rels in elrRelSet.fromModelObjects().items():
elrUgtCalcs[relFrom.name] = [rel.toModelObject.name for rel in rels]
val.usgaapCalculations[ELR] = elrUgtCalcs
jsonStr = _STR_UNICODE(json.dumps(val.usgaapCalculations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, ugtCalcsJsonFile, jsonStr) # 2.7 gets unicode this way
# load default dimensions
for defaultDimRel in calculationsInstance.relationshipSet(XbrlConst.dimensionDefault).modelRelationships:
if defaultDimRel.fromModelObject is not None and defaultDimRel.toModelObject is not None:
val.usgaapDefaultDimensions[defaultDimRel.fromModelObject.name] = defaultDimRel.toModelObject.name
jsonStr = _STR_UNICODE(json.dumps(val.usgaapDefaultDimensions, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, ugtDefaultDimensionsJsonFile, jsonStr) # 2.7 gets unicode this way
calculationsInstance.close()
del calculationsInstance # dereference closed modelXbrl
val.modelXbrl.profileStat(_("build us-gaap calculations and default dimensions cache"), time.time() - startedAt)
break
val.deprecatedFactConcepts = defaultdict(list)
val.deprecatedDimensions = defaultdict(list)
val.deprecatedMembers = defaultdict(list)
def load(self, filesource, nextaction):
self.filesource = filesource
self.modelXbrl = ModelXbrl.load(self, filesource, nextaction)
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def _create_cache(val):
"""
Creates the caches needed for dqc_us_0018
:param val: ValidateXbrl needed in order to save the cache
:type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
:return: no explicit return but creates and saves a cache in
dqc_us_rule\resources\DQC_US_0018
:rtype: None
"""
val.ugtNamespace = None
cntlr = val.modelXbrl.modelManager.cntlr
year = _EARLIEST_US_GAAP_YEAR
for ugt in ugtDocs:
deprecations_json_file = os.path.join(
os.path.dirname(__file__),
'resources',
'DQC_US_0018',
'{}_deprecated-concepts.json'.format(str(year))
)
if not os.path.isfile(deprecations_json_file):
ugt_doc_lb = ugt["docLB"]
val.usgaapDeprecations = {}
disclosure_system = (
val.modelXbrl.modelManager.validateDisclosureSystem
)
prior_validate_disclosure_system = disclosure_system
val.modelXbrl.modelManager.validateDisclosureSystem = False
deprecations_instance = ModelXbrl.load(
val.modelXbrl.modelManager,
openFileSource(ugt_doc_lb, cntlr),
_("built deprecations table in cache") # noqa
)
val.modelXbrl.modelManager.validateDisclosureSystem = (
prior_validate_disclosure_system
)
if deprecations_instance is not None:
dep_label = 'http://www.xbrl.org/2009/role/deprecatedLabel'
dep_date_label = (
'http://www.xbrl.org/2009/role/deprecatedDateLabel'
)
concept_label = XbrlConst.conceptLabel
relationship_set = (
deprecations_instance.relationshipSet(concept_label)
)
model_relationships = relationship_set.modelRelationships
for labelRel in model_relationships:
model_documentation = labelRel.toModelObject
concept = labelRel.fromModelObject.name
if model_documentation.role == dep_label:
val.usgaapDeprecations[concept] = (
model_documentation.text,
val.usgaapDeprecations.get(concept, ('', ''))[0]
)
elif model_documentation.role == dep_date_label:
val.usgaapDeprecations[concept] = (
model_documentation.text,
val.usgaapDeprecations.get(concept, ('', ''))[1]
)
json_str = str(
json.dumps(
val.usgaapDeprecations,
ensure_ascii=False, indent=0
)
) # might not be unicode in 2.7
saveFile(cntlr, deprecations_json_file, json_str)
deprecations_instance.close()
del deprecations_instance # dereference closed modelXbrl
year += 1
def runFromExcel(self, options):
#testGenFileName = options.excelfilename
testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls"
testGenDir = os.path.dirname(testGenFileName)
schemaDir = os.path.dirname(testGenDir) + os.sep + "schema"
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
LogHandler(self) # start logger
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt'
modelTestcases = ModelXbrl.create(self.modelManager,
url=testGenFileName,
isEntry=True)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(
_("[info] xls loaded in {0:.2} secs at {1}").format(
time.time() - startedAt, timeNow))
# start index file
indexFiles = [
testGenDir + os.sep + 'creation-testcases-index.xml',
testGenDir + os.sep + 'consumption-testcases-index.xml'
]
indexDocs = []
testcasesElements = []
for purpose in ("Creation", "Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 {0} Tests" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today))
doc = etree.parse(file)
file.close()
indexDocs.append(doc)
testcasesElements.append(doc.getroot())
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
try:
row = testcaseIndexSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY
or # must have directory
row[1].ctype == xlrd.XL_CELL_EMPTY or # from
row[2].ctype == xlrd.XL_CELL_EMPTY): # to
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
overrideReport = row[3].value
description = row[4].value
if description is None or len(description) == 0:
continue # test not ready to run
assignment = row[5].value
expectedEvents = row[
6].value # comma space separated if multiple
note = row[7].value
useCase = row[8].value
base = os.path.join(os.path.dirname(testGenFileName),
testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and assignment.lower() not in (
"n.a.", "error") and expectedEvents != "N.A.":
modelDTSfrom = modelDTSto = None
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"),
True), (uriTo,
_("loading to DTS"),
False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager,
URIs,
msg,
base=base)
else:
modelDTS = ModelXbrl.create(
self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(
URIs.replace(", ", "_") + ".dts", base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS,
uri.strip(),
base=base)
if doc is not None:
DTSdoc.referencesDocument[
doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom is not None and modelDTSto is not None:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace(
"from.xsd", "report.xml")
reportFile = reportDir + "out" + os.sep + reportName
#reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile, base)
testcasesDir = os.path.dirname(
os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i, testcaseFile in enumerate(
testcaseFiles):
with open(testcaseFile,
"w",
encoding="utf-8") as fh:
XmlUtil.writexml(fh,
testcaseDocs[i],
encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [
testcasesDir + os.sep + testcaseName +
"-creation-testcase.xml",
testcasesDir + os.sep + testcaseName +
"-consumption-testcase.xml"
]
for i, testcaseFile in enumerate(testcaseFiles):
etree.SubElement(
testcasesElements[i],
"testcase",
attrib={
"uri":
testcaseFile[len(testGenDir) +
1:].replace("\\", "/")
})
# start testcase file
testcaseDocs = []
testcaseElements = []
testcaseNumber = testcaseName[0:4]
if testcaseNumber.isnumeric():
testcaseNumberElement = "<number>{0}</number>".format(
testcaseNumber)
testcaseName = testcaseName[5:]
else:
testcaseNumberElement = ""
testDirSegments = testDir.split('/')
if len(testDirSegments
) >= 2 and '-' in testDirSegments[1]:
testedModule = testDirSegments[1][
testDirSegments[1].index('-') + 1:]
else:
testedModule = ''
for purpose in ("Creation", "Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'{0}'
'<name>{1}</name>'
# '<description>{0}</description>'
'<reference>'
'{2}'
'{3}'
'</reference>'
'</testcase>'.format(
testcaseNumberElement, testcaseName,
'<name>{0}</name>'.format(testedModule)
if testedModule else '',
'<id>{0}</id>'.format(useCase)
if useCase else ''))
doc = etree.parse(file)
file.close()
testcaseDocs.append(doc)
testcaseElements.append(doc.getroot())
priorTestcasesDir = testcasesDir
variationSeq = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(
modelTestcases)
modelVersReport.diffDTSes(reportFullPath,
modelDTSfrom,
modelDTSto,
assignment=assignment,
schemaDir=schemaDir)
# check for expected elements
if expectedEvents:
for expectedEvent in expectedEvents.split(","):
if expectedEvent not in ("No change", "N.A."):
prefix, sep, localName = expectedEvent.partition(
':')
if sep and len(
modelVersReport.xmlDocument.
findall('//{{{0}}}{1}'.format(
XbrlConst.verPrefixNS.get(
prefix), localName))) == 0:
modelTestcases.warning(
"warning",
"Generated test case %(reportName)s missing expected event %(event)s",
reportName=reportName,
event=expectedEvent)
modelVersReport.close()
uriFromParts = uriFrom.split('_')
if len(uriFromParts) >= 2:
variationId = uriFromParts[1]
else:
variationId = "_{0:02n}".format(variationSeq)
for i, testcaseElt in enumerate(testcaseElements):
variationElement = etree.SubElement(
testcaseElt,
"{http://xbrl.org/2008/conformance}variation",
attrib={"id": variationId})
nameElement = etree.SubElement(
variationElement,
"{http://xbrl.org/2008/conformance}description"
)
nameElement.text = description
''' (removed per RH 2011/10/04
if note:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
paramElement.text = "Note: " + note
if useCase:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference")
paramElement.set("specification", "versioning-requirements")
paramElement.set("useCase", useCase)
'''
dataElement = etree.SubElement(
variationElement,
"{http://xbrl.org/2008/conformance}data")
if i == 0: # result is report
if expectedEvents:
paramElement = etree.SubElement(
dataElement,
"{http://xbrl.org/2008/conformance}parameter",
attrib={
"name":
"expectedEvent",
"value":
expectedEvents.replace(',', ' ')
},
nsmap={
"conf":
"http://xbrl.org/2008/conformance",
None: ""
})
if assignment:
paramElement = etree.SubElement(
dataElement,
"{http://xbrl.org/2008/conformance}parameter",
attrib={
"name": "assignment",
"value": assignment
},
nsmap={
"conf":
"http://xbrl.org/2008/conformance",
None: ""
})
for schemaURIs, dtsAttr in ((uriFrom, "from"),
(uriTo, "to")):
for schemaURI in schemaURIs.split(","):
schemaElement = etree.SubElement(
dataElement,
"{http://xbrl.org/2008/conformance}schema"
)
schemaElement.set("dts", dtsAttr)
if i == 0:
schemaElement.set(
"readMeFirst", "true")
schemaElement.text = os.path.basename(
schemaURI.strip())
resultElement = etree.SubElement(
variationElement,
"{http://xbrl.org/2008/conformance}result")
reportElement = etree.SubElement(
resultElement if i == 0 else dataElement,
"{http://xbrl.org/2008/conformance}versioningReport"
)
if i == 1:
reportElement.set("readMeFirst", "true")
reportElement.text = "report/" + reportName
variationSeq += 1
except Exception as err:
modelTestcases.error(
"exception",
_("Exception: %(error)s, Excel row: %(excelRow)s"),
error=err,
excelRow=iRow,
exc_info=True)
# add tests-error-code index files to consumption
for testcaseFile in self.testcaseFiles(testGenDir + os.sep +
"tests-error-code"):
etree.SubElement(testcasesElements[1],
"testcase",
attrib={
"uri":
testcaseFile[len(testGenDir) + 1:].replace(
"\\", "/")
})
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i, testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i, indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def load(self, filesource, nextaction):
self.filesource = filesource
self.modelXbrl = ModelXbrl.load(self, filesource, nextaction)
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def runFromExcel(self, options):
testGenFileName = options.excelfilename
#testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls"
testGenDir = os.path.dirname(testGenFileName)
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt'
modelTestcases = ModelXbrl.create(self.modelManager)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(
_("[info] xls loaded in {0:.2} secs at {1}").format(
time.time() - startedAt, timeNow))
# start index file
indexFiles = [
testGenDir + os.sep + 'creationTestcasesIndex.xml',
testGenDir + os.sep + 'consumptionTestcasesIndex.xml'
]
indexDocs = [
xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today))
for purpose in ("Creation", "Consumption")
]
testcasesElements = [
XmlUtil.child(indexDoc, None, "testcases")
for indexDoc in indexDocs
]
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
row = testcaseIndexSheet.row(iRow)
if row[0].ctype == xlrd.XL_CELL_EMPTY or row[
1].ctype == xlrd.XL_CELL_EMPTY or row[
2].ctype == xlrd.XL_CELL_EMPTY:
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
intention = row[3].value
if intention is None or len(intention) == 0:
continue # test not ready to run
reason = row[4].value
expectedEvent = row[5].value
base = os.path.join(os.path.dirname(testGenFileName),
testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and reason.lower() not in (
"n.a.", "error") and expectedEvent != "N.A.":
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"),
True), (uriTo, _("loading to DTS"),
False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager,
URIs,
msg,
base=base)
else:
modelDTS = ModelXbrl.create(
self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(
URIs.replace(", ", "_") + ".dts", base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS,
uri.strip(),
base=base)
DTSdoc.referencesDocument[
doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom and modelDTSto:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace(
"from.xsd", "report.xml")
reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile, base)
testcasesDir = os.path.dirname(
os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i, testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w",
encoding="utf-8") as fh:
XmlUtil.writexml(fh,
testcaseDocs[i],
encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [
testcasesDir + os.sep + testcaseName +
"-creation-testcase.xml", testcasesDir + os.sep +
testcaseName + "-consumption-testcase.xml"
]
for i, testcaseFile in enumerate(testcaseFiles):
XmlUtil.addChild(
testcasesElements[i], None, "testcase",
("uri", testcaseFile[len(testGenDir) +
1:].replace("\\", "/")))
# start testcase file
testcaseDocs = [
xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'<name>{0}</name>'
'<description>{0}</description>'
'</testcase>'.format(testcaseName, purpose,
today))
for purpose in ("Creation", "Consumption")
]
testcaseElements = [
XmlUtil.child(testcaseDoc, conformanceNS,
"testcase")
for testcaseDoc in testcaseDocs
]
priorTestcasesDir = testcasesDir
variationID = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(
modelTestcases)
modelVersReport.diffDTSes(reportFullPath, modelDTSfrom,
modelDTSto)
# check for expected elements
if expectedEvent and expectedEvent not in ("No change",
"N.A."):
if len(
modelVersReport.xmlDocument.
getElementsByTagNameNS('*',
expectedEvent)) == 0:
modelTestcases.error(
"Generated test case {0} missing expected event {1}"
.format(reportName,
expectedEvent), "wrn", "missingEvent")
modelVersReport.close([])
for i, testcaseElt in enumerate(testcaseElements):
variationElement = XmlUtil.addChild(
testcaseElt,
conformanceNS,
"variation",
attributes=("id", "_{0:02n}".format(variationID)))
XmlUtil.addChild(variationElement,
conformanceNS,
"name",
text=intention)
dataElement = XmlUtil.addChild(variationElement,
conformanceNS, "data")
for schemaURIs, dtsAttr in ((uriFrom, "from"), (uriTo,
"to")):
for schemaURI in schemaURIs.split(","):
XmlUtil.addChild(
dataElement,
conformanceNS,
"schema",
attributes=((("dts", dtsAttr), ) +
((("readMeFirst",
"true"), ) if i == 0 else
())),
text=os.path.basename(schemaURI.strip()))
resultElement = XmlUtil.addChild(
variationElement, conformanceNS, "result")
XmlUtil.addChild(
resultElement if i == 0 else dataElement,
conformanceNS,
"versioningReport",
attributes=(("readMeFirst", "true") if i == 1 else
()),
text="report/" + reportName)
variationID += 1
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i, testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i, indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def validateTestcase(self, testcase):
self.modelXbrl.error(_("testcase {0}").format(os.path.basename(testcase.uri)))
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation.element)
# try to load instance document
self.modelXbrl.error(_("variation {0} {1}: {2}").format(modelTestcaseVariation.id, modelTestcaseVariation.name, modelTestcaseVariation.expected))
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
ModelDocument.Type.DTSENTRIES,
self.modelXbrl.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = ModelDocument.load(modelXbrl, readMeFirstUri, base=baseForElement)
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
else: # not a multi-schemaRef versioning report
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement)
if modelXbrl.modelDocument is None:
self.modelXbrl.error(_("Testcase {0} {1} document not loaded: {2}").format(
modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(readMeFirstUri)))
modelTestcaseVariation.status = "not loadable"
modelXbrl.close()
elif resultIsVersioningReport:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl) \
.validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
modelXbrl.close()
elif testcase.type == ModelDocument.Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
self.modelXbrl.error(_("Testcase {0} {1} DTSes not loaded, unable to generate versioning report").format(
modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(readMeFirstUri)))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes:
inputDTS.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
parameters = modelTestcaseVariation.parameters.copy()
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName:
parameters[dtsName] = (None, inputDTS)
self.instValidator.validate(modelXbrl, parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
if modelXbrl.formulaOutputInstance and len(modelTestcaseVariation.actual) == 0:
# if an output instance is created, validate it
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance)
if len(modelTestcaseVariation.actual) == 0: # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement)
if expectedInstance.modelDocument is None:
self.modelXbrl.error(_("Testcase {0} {1} expected result instance not loaded: {2}").format(
modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(modelTestcaseVariation.resultXbrlInstance)))
modelTestcaseVariation.status = "result not loadable"
expectedInstance.close()
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error(
_("Formula output {0} facts, expected {1} facts").format(
len(formulaOutputInstance.facts),
len(expectedInstance.facts)),
"err", "formula:resultFactCounts")
else:
for fact in expectedInstance.facts:
if not formulaOutputInstance.matchFact(fact):
formulaOutputInstance.error(
_("Formula output missing expected fact {0}").format(
fact),
"err", "formula:expectedFactMissing")
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance)
formulaOutputInstance.close()
formulaOutputInstance = None
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def watchCycle(self):
while not self.stopRequested:
rssWatchOptions = self.rssModelXbrl.modelManager.rssWatchOptions
# check rss expiration
rssHeaders = self.cntlr.webCache.getheaders(self.rssModelXbrl.modelManager.rssWatchOptions.get("feedSourceUri"))
expires = parseRfcDatetime(rssHeaders.get("expires"))
reloadNow = True # texpires and expires > datetime.datetime.now()
# reload rss feed
self.rssModelXbrl.reload('checking RSS items', reloadCache=reloadNow)
if self.stopRequested: break
# setup validator
postLoadActions = []
if rssWatchOptions.get("validateDisclosureSystemRules"):
self.instValidator = ValidateFiling.ValidateFiling(self.rssModelXbrl)
postLoadActions.append(_("validating"))
elif rssWatchOptions.get("validateXbrlRules") or rssWatchOptions.get("validateFormulaAssertions"):
self.instValidator = ValidateXbrl.ValidateXbrl(self.rssModelXbrl)
postLoadActions.append(_("validating"))
if (rssWatchOptions.get("validateFormulaAssertions")):
postLoadActions.append(_("running formulas"))
else:
self.instValidator = None
matchTextExpr = rssWatchOptions.get("matchTextExpr")
if matchTextExpr:
matchPattern = re.compile(matchTextExpr)
postLoadActions.append(_("matching text"))
else:
matchPattern= None
postLoadAction = ', '.join(postLoadActions)
# anything to check new filings for
if (rssWatchOptions.get("validateDisclosureSystemRules") or
rssWatchOptions.get("validateXbrlRules") or
rssWatchOptions.get("validateCalcLinkbase") or
rssWatchOptions.get("validateFormulaAssertions") or
rssWatchOptions.get("alertMatchedFactText") or
any(pluginXbrlMethod(rssWatchOptions)
for pluginXbrlMethod in pluginClassMethods("RssWatch.HasWatchAction"))
):
# form keys in ascending order of pubdate
pubDateRssItems = []
for rssItem in self.rssModelXbrl.modelDocument.rssItems:
pubDateRssItems.append((rssItem.pubDate,rssItem.objectId()))
for pubDate, rssItemObjectId in sorted(pubDateRssItems):
rssItem = self.rssModelXbrl.modelObject(rssItemObjectId)
# update ui thread via modelManager (running in background here)
self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId())
if self.stopRequested:
break
latestPubDate = XmlUtil.datetimeValue(rssWatchOptions.get("latestPubDate"))
if (latestPubDate and
rssItem.pubDate < latestPubDate):
continue
try:
# try zipped URL if possible, else expanded instance document
modelXbrl = ModelXbrl.load(self.rssModelXbrl.modelManager,
openFileSource(rssItem.zippedUrl, self.cntlr),
postLoadAction)
if self.stopRequested:
modelXbrl.close()
break
emailAlert = False
if modelXbrl.modelDocument is None:
modelXbrl.error("arelle.rssWatch",
_("RSS item %(company)s %(form)s document not loaded: %(date)s"),
modelXbrl=modelXbrl, company=rssItem.companyName,
form=rssItem.formType, date=rssItem.filingDate)
rssItem.status = "not loadable"
else:
# validate schema, linkbase, or instance
if self.stopRequested:
modelXbrl.close()
break
if self.instValidator:
self.instValidator.validate(modelXbrl)
if modelXbrl.errors and rssWatchOptions.get("alertValiditionError"):
emailAlert = True
for pluginXbrlMethod in pluginClassMethods("RssWatch.DoWatchAction"):
pluginXbrlMethod(modelXbrl, rssWatchOptions, rssItem)
# check match expression
if matchPattern:
for fact in modelXbrl.factsInInstance:
v = fact.value
if v is not None:
m = matchPattern.search(v)
if m:
fr, to = m.span()
msg = _("Fact Variable {0}\n context {1}\n matched text: {2}").format(
fact.qname, fact.contextID, v[max(0,fr-20):to+20])
modelXbrl.info("arelle.rssInfo",
msg,
modelXbrl=modelXbrl) # msg as code passes it through to the status
if rssWatchOptions.get("alertMatchedFactText"):
emailAlert = True
if (rssWatchOptions.get("formulaFileUri") and rssWatchOptions.get("validateFormulaAssertions") and
self.instValidator):
# attach formulas
ModelDocument.load(modelXbrl, rssWatchOptions["formulaFileUri"])
ValidateFormula.validate(self.instValidator)
rssItem.setResults(modelXbrl)
modelXbrl.close()
del modelXbrl # completely dereference
self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId())
if rssItem.assertionUnsuccessful and rssWatchOptions.get("alertAssertionUnsuccessful"):
emailAlert = True
msg = _("Filing CIK {0}\n "
"company {1}\n "
"published {2}\n "
"form type {3}\n "
"filing date {4}\n "
"period {5}\n "
"year end {6}\n "
"results: {7}").format(
rssItem.cikNumber,
rssItem.companyName,
rssItem.pubDate,
rssItem.formType,
rssItem.filingDate,
rssItem.period,
rssItem.fiscalYearEnd,
rssItem.status)
self.rssModelXbrl.info("arelle:rssWatch", msg, modelXbrl=self.rssModelXbrl)
emailAddress = rssWatchOptions.get("emailAddress")
if emailAlert and emailAddress:
self.rssModelXbrl.modelManager.showStatus(_("sending e-mail alert"))
import smtplib
from email.mime.text import MIMEText
emailMsg = MIMEText(msg)
emailMsg["Subject"] = _("Arelle RSS Watch alert on {0}").format(rssItem.companyName)
emailMsg["From"] = emailAddress
emailMsg["To"] = emailAddress
smtp = smtplib.SMTP()
smtp.sendmail(emailAddress, [emailAddress], emailMsg.as_string())
smtp.quit()
self.rssModelXbrl.modelManager.showStatus(_("RSS item {0}, {1} completed, status {2}").format(rssItem.companyName, rssItem.formType, rssItem.status), 3500)
self.rssModelXbrl.modelManager.cntlr.rssWatchUpdateOption(rssItem.pubDate.strftime('%Y-%m-%dT%H:%M:%S'))
except Exception as err:
self.rssModelXbrl.error("arelle.rssError",
_("RSS item %(company)s, %(form)s, %(date)s, exception: %(error)s"),
modelXbrl=self.rssModelXbrl, company=rssItem.companyName,
form=rssItem.formType, date=rssItem.filingDate, error=err,
exc_info=True)
if self.stopRequested: break
if self.stopRequested:
self.cntlr.showStatus(_("RSS watch, stop requested"), 10000)
else:
import time
time.sleep(600)
self.thread = None # close thread
self.stopRequested = False
def _create_cache(val):
"""
Creates the caches needed for dqc_us_0018
:param val: ValidateXbrl needed in order to save the cache
:type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
:return: no explicit return but creates and saves a cache in
dqc_us_rule\resources\DQC_US_0018
:rtype: None
"""
val.ugtNamespace = None
cntlr = val.modelXbrl.modelManager.cntlr
year = _EARLIEST_US_GAAP_YEAR
for ugt in ugtDocs:
deprecations_json_file = os.path.join(
os.path.dirname(__file__),
'resources',
'DQC_US_0018',
'{}_deprecated-concepts.json'.format(str(year))
)
if not os.path.isfile(deprecations_json_file):
ugt_doc_lb = ugt["docLB"]
val.usgaapDeprecations = {}
disclosure_system = (
val.modelXbrl.modelManager.validateDisclosureSystem
)
prior_validate_disclosure_system = disclosure_system
val.modelXbrl.modelManager.validateDisclosureSystem = False
deprecations_instance = ModelXbrl.load(
val.modelXbrl.modelManager,
openFileSource(ugt_doc_lb, cntlr),
_("built deprecations table in cache") # noqa
)
val.modelXbrl.modelManager.validateDisclosureSystem = (
prior_validate_disclosure_system
)
if deprecations_instance is not None:
dep_label = 'http://www.xbrl.org/2009/role/deprecatedLabel'
dep_date_label = (
'http://www.xbrl.org/2009/role/deprecatedDateLabel'
)
concept_label = XbrlConst.conceptLabel
relationship_set = (
deprecations_instance.relationshipSet(concept_label)
)
model_relationships = relationship_set.modelRelationships
for labelRel in model_relationships:
model_documentation = labelRel.toModelObject
concept = labelRel.fromModelObject.name
if model_documentation.role == dep_label:
val.usgaapDeprecations[concept] = (
model_documentation.text,
val.usgaapDeprecations.get(concept, ('', ''))[0]
)
elif model_documentation.role == dep_date_label:
val.usgaapDeprecations[concept] = (
model_documentation.text,
val.usgaapDeprecations.get(concept, ('', ''))[1]
)
json_str = str(
json.dumps(
val.usgaapDeprecations,
ensure_ascii=False, indent=0
)
) # might not be unicode in 2.7
saveFile(cntlr, deprecations_json_file, json_str)
deprecations_instance.close()
del deprecations_instance # dereference closed modelXbrl
year += 1
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s"),
modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected)
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
ModelDocument.Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = ModelDocument.load(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
else: # not a multi-schemaRef versioning report
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource)
if modelXbrl.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelXbrl.close()
self.determineNotLoadedTestStatus(modelTestcaseVariation)
elif resultIsVersioningReport:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
modelXbrl.close()
elif testcase.type == ModelDocument.Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
parameters = modelTestcaseVariation.parameters.copy()
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName:
parameters[dtsName] = (None, inputDTS)
self.instValidator.validate(modelXbrl, parameters)
if modelTestcaseVariation.resultIsInfoset:
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource)
if infoset.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
self.determineTestStatus(modelTestcaseVariation, modelXbrl) # include infoset errors in status
self.instValidator.close()
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result not loadable"
expectedInstance.close()
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("formula:resultFactCounts",
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts))
else:
for fact in expectedInstance.facts:
if formulaOutputInstance.matchFact(fact) is None:
formulaOutputInstance.error("formula:expectedFactMissing",
_("Formula output missing expected fact %(fact)s"),
modelXbrl=fact, fact=fact.qname)
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance)
formulaOutputInstance.close()
formulaOutputInstance = None
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY):
for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri):
self.validateTestcase(doc) # testcases doc's are sorted by their uri (file names), e.g., for formula
elif hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings:
errorCaptureLevel = logging._checkLevel("WARNING")
else:
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
elif resultIsTaxonomyPackage:
from arelle import PackageManager, PrototypeInstanceObject
dtsName = readMeFirstUri
modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
else: # need own file source, may need instance discovery
filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
if filesource and not filesource.selection and filesource.isArchive:
for _archiveFile in filesource.dir or (): # find instance document in archive
filesource.select(_archiveFile)
if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
break # use this selection
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel)
modelXbrl.isTestcaseVariation = True
if modelXbrl.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif resultIsVersioningReport or resultIsTaxonomyPackage:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif resultIsTaxonomyPackage:
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement)
for d in modelTestcaseVariation.dataUris.values() for uri in d
if not UrlUtil.isAbsolute(uri))
foundDataFiles = set()
variationBase = os.path.dirname(baseForElement)
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
for _inputDTS in inputDTS:
for docUrl, doc in _inputDTS.urlDocs.items():
if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET:
if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file
if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles:
docUrl = docUrl.replace("-formula.xml", ".xf")
foundDataFiles.add(docUrl)
if expectedDataFiles - foundDataFiles:
modelXbrl.info("arelle:testcaseDataNotUsed",
_("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"),
modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles)))
if foundDataFiles - expectedDataFiles:
modelXbrl.info("arelle:testcaseDataUnexpected",
_("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"),
modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles)))
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
try:
RenderingEvaluator.init(modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrlHasFormulae = modelXbrl.hasFormulae
if modelXbrlHasFormulae:
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
try:
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and
modelTestcaseVariation.resultXbrlInstanceUri is not None)
if compareIxResultInstance:
formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
errMsgPrefix = "ix"
else: # delete input instances before formula output comparision
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
errMsgPrefix = "formula"
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
_matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
_("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts),
messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
else:
formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
def factFootnotes(fact, footnotesRelSet):
footnotes = {}
footnoteRels = footnotesRelSet.fromModelObject(fact)
if footnoteRels:
# most process rels in same order between two instances, use labels to sort
for i, footnoteRel in enumerate(sorted(footnoteRels,
key=lambda r: (r.fromLabel,r.toLabel))):
modelObject = footnoteRel.toModelObject
if isinstance(modelObject, ModelResource):
xml = modelObject.viewText().strip()
footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))
elif isinstance(modelObject, ModelFact):
footnotes["Footnoted fact {}".format(i+1)] = \
"{} context: {} value: {}".format(
modelObject.qname,
modelObject.contextID,
collapseWhitespace(modelObject.value))
return footnotes
for expectedInstanceFact in expectedInstance.facts:
unmatchedFactsStack = []
formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False)
#formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True)
if formulaOutputFact is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = expectedInstanceFact
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else: # compare footnotes
expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or
set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())):
formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
_("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()),
messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
if compareIxResultInstance:
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
_statusCounts = OrderedDict((("pass",0),("fail",0)))
for tv in getattr(testcase, "testcaseVariations", ()):
_statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1
self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k))
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(
self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(
modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info(
"info",
_("Variation %(id)s %(name)s: %(expected)s - %(description)s"
),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri, tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(
self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.
normalizeUrl(readMeFirstUri[:-4] + ".dts",
baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl,
readMeFirstUri,
base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[
doc] = ModelDocumentReference(
"import",
DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
else: # not a multi-schemaRef versioning report
modelXbrl = ModelXbrl.load(
self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if modelXbrl.modelDocument is None:
self.modelXbrl.error(
"arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"
),
modelXbrl=testcase,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
file=os.path.basename(readMeFirstUri))
modelXbrl.close()
self.determineNotLoadedTestStatus(
modelTestcaseVariation)
elif resultIsVersioningReport:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(
self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation,
modelXbrl)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(
modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(
modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall,
modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation,
modelXbrl)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri,
baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(
self.modelXbrl.modelManager, versReportFile,
_("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(
self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation,
modelVersReport)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(
self.modelXbrl).diffDTSes(versReportFile,
inputDTSes["from"],
inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
self.modelXbrl.error(
"arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"
),
modelXbrl=testcase,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
for pluginXbrlMethod in pluginClassMethods(
"TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
parameters = modelTestcaseVariation.parameters.copy()
for dtsName, inputDTS in inputDTSes.items(
): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (
None, inputDTS
) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(
inputDTS
) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (
None, inputDTS
) # allow error detection in validateFormula
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
RenderingEvaluator.init(modelXbrl)
try:
self.instValidator.validate(modelXbrl, parameters)
except Exception as err:
self.modelXbrl.error(
"exception",
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"
),
modelXbrl=modelXbrl,
instance=modelXbrl.modelDocument.basename,
error=err,
exc_info=True)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods(
"Validate.Infoset"):
pluginXbrlMethod(
modelXbrl,
modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(
self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
self.modelXbrl.error(
"arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"
),
modelXbrl=testcase,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
file=os.path.basename(
modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator,
modelXbrl, infoset)
infoset.close()
if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.resultTableUri,
baseForElement)
if not any(
alternativeValidation(modelXbrl,
resultTableUri)
for alternativeValidation in
pluginClassMethods("Validate.TableInfoset")):
ViewFileRenderedGrid.viewRenderedGrid(
modelXbrl, resultTableUri,
diffToFile=True) # false to save infoset files
self.determineTestStatus(
modelTestcaseVariation,
modelXbrl) # include infoset errors in status
self.instValidator.close()
if modelXbrl.formulaOutputInstance and self.noErrorCodes(
modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(
modelXbrl.formulaOutputInstance,
modelTestcaseVariation.parameters)
self.determineTestStatus(
modelTestcaseVariation,
modelXbrl.formulaOutputInstance)
if self.noErrorCodes(
modelTestcaseVariation.actual
): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(
self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error(
"arelle:notLoaded",
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"
),
modelXbrl=testcase,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
file=os.path.basename(
modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(
formulaOutputInstance.facts):
formulaOutputInstance.error(
"formula:resultFactCounts",
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"
),
modelXbrl=modelXbrl,
countFacts=len(
formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts))
else:
for fact in expectedInstance.facts:
unmatchedFactsStack = []
if formulaOutputInstance.matchFact(
fact,
unmatchedFactsStack,
deemP0inf=True) is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[
-1]
else:
missingFact = fact
formulaOutputInstance.error(
"formula:expectedFactMissing",
_("Formula output missing expected fact %(fact)s"
),
modelXbrl=missingFact,
fact=missingFact.qname)
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation,
formulaOutputInstance)
formulaOutputInstance.close()
del formulaOutputInstance
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(
self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY):
for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri):
self.validateTestcase(doc) # testcases doc's are sorted by their uri (file names), e.g., for formula
elif hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcaseVariationsByTarget(testcase.testcaseVariations):
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s%(name)s%(target)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=(" {}".format(modelTestcaseVariation.name) if modelTestcaseVariation.name else ""),
target=(" target {}".format(modelTestcaseVariation.ixdsTarget) if modelTestcaseVariation.ixdsTarget else ""),
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings:
errorCaptureLevel = logging._checkLevel("WARNING")
else:
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
elif resultIsTaxonomyPackage:
from arelle import PackageManager, PrototypeInstanceObject
dtsName = readMeFirstUri
modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel,
ixdsTarget=modelTestcaseVariation.ixdsTarget)
else: # need own file source, may need instance discovery
filesource = FileSource.openFileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr, base=baseForElement)
if filesource and not filesource.selection and filesource.isArchive:
try:
if filesource.isTaxonomyPackage:
_rptPkgIxdsOptions = {}
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxdsOptions"):
pluginXbrlMethod(self, _rptPkgIxdsOptions)
filesource.loadTaxonomyPackageMappings()
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxds"):
filesource.select(pluginXbrlMethod(filesource, **_rptPkgIxdsOptions))
else:
from arelle.CntlrCmdLine import filesourceEntrypointFiles
entrypoints = filesourceEntrypointFiles(filesource)
if entrypoints:
# resolve an IXDS in entrypoints
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ArchiveIxds"):
pluginXbrlMethod(self, filesource,entrypoints)
filesource.select(entrypoints[0].get("file", None) )
except Exception as err:
self.modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, entry URL: %(instance)s"),
modelXbrl=self.modelXbrl, instance=readMeFirstUri, error=err)
continue # don't try to load this entry URL
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel,
ixdsTarget=modelTestcaseVariation.ixdsTarget)
modelXbrl.isTestcaseVariation = True
if modelXbrl.modelDocument is None:
modelXbrl.info("arelle:notLoaded",
_("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif resultIsVersioningReport or resultIsTaxonomyPackage:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError))
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif resultIsTaxonomyPackage:
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement)
for d in modelTestcaseVariation.dataUris.values() for uri in d
if not UrlUtil.isAbsolute(uri))
foundDataFiles = set()
variationBase = os.path.dirname(baseForElement)
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
for _inputDTS in inputDTS:
for docUrl, doc in _inputDTS.urlDocs.items():
if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET:
if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file
if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles:
docUrl = docUrl.replace("-formula.xml", ".xf")
foundDataFiles.add(docUrl)
if expectedDataFiles - foundDataFiles:
modelXbrl.info("arelle:testcaseDataNotUsed",
_("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"),
modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles)))
if foundDataFiles - expectedDataFiles:
modelXbrl.info("arelle:testcaseDataUnexpected",
_("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"),
modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles)))
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
try:
RenderingEvaluator.init(modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrlHasFormulae = modelXbrl.hasFormulae
if modelXbrlHasFormulae and self.modelXbrl.modelManager.formulaOptions.formulaAction != "none":
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError))
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
try:
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and
modelTestcaseVariation.resultXbrlInstanceUri is not None)
if compareIxResultInstance:
formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
errMsgPrefix = "ix"
else: # delete input instances before formula output comparision
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
errMsgPrefix = "formula"
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
_matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
_("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.ExpectedInstance.Loaded"):
pluginXbrlMethod(expectedInstance, formulaOutputInstance)
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts),
messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
else:
formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
def factFootnotes(fact, footnotesRelSet):
footnotes = {}
footnoteRels = footnotesRelSet.fromModelObject(fact)
if footnoteRels:
# most process rels in same order between two instances, use labels to sort
for i, footnoteRel in enumerate(sorted(footnoteRels,
key=lambda r: (r.fromLabel,r.toLabel))):
modelObject = footnoteRel.toModelObject
if isinstance(modelObject, ModelResource):
xml = collapseWhitespace(modelObject.viewText().strip())
footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))
elif isinstance(modelObject, ModelFact):
footnotes["Footnoted fact {}".format(i+1)] = \
"{} context: {} value: {}".format(
modelObject.qname,
modelObject.contextID,
collapseWhitespace(modelObject.value))
return footnotes
for expectedInstanceFact in expectedInstance.facts:
unmatchedFactsStack = []
formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False)
#formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True)
if formulaOutputFact is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = expectedInstanceFact
# is it possible to show value mismatches?
expectedFacts = formulaOutputInstance.factsByQname.get(missingFact.qname)
if len(expectedFacts) == 1:
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s, extracted value \"%(value1)s\", expected value \"%(value2)s\""),
modelXbrl=missingFact, fact=missingFact.qname, value1=missingFact.xValue, value2=next(iter(expectedFacts)).xValue,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else:
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else: # compare footnotes
expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or
set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())):
formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
_("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()),
messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
if compareIxResultInstance:
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
_statusCounts = OrderedDict((("pass",0),("fail",0)))
for tv in getattr(testcase, "testcaseVariations", ()):
_statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1
self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k))
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def setup(val):
if not val.validateLoggingSemantic: # all checks herein are SEMANTIC
return
val.linroleDefinitionIsDisclosure = re.compile(r"-\s+Disclosure\s+-\s",
re.IGNORECASE)
val.linkroleDefinitionStatementSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*", # no restriction to type of statement
re.IGNORECASE)
val.ugtNamespace = None
cntlr = val.modelXbrl.modelManager.cntlr
# load deprecated concepts for filed year of us-gaap
for ugt in ugtDocs:
ugtNamespace = ugt["namespace"]
if ugtNamespace in val.modelXbrl.namespaceDocs and len(val.modelXbrl.namespaceDocs[ugtNamespace]) > 0:
val.ugtNamespace = ugtNamespace
usgaapDoc = val.modelXbrl.namespaceDocs[ugtNamespace][0]
deprecationsJsonFile = usgaapDoc.filepathdir + os.sep + "deprecated-concepts.json"
file = None
try:
file = openFileStream(cntlr, deprecationsJsonFile, 'rt', encoding='utf-8')
val.usgaapDeprecations = json.load(file)
file.close()
except Exception:
if file:
file.close()
val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} deprecated concepts into cache").format(ugt["year"]))
startedAt = time.time()
ugtDocLB = ugt["docLB"]
val.usgaapDeprecations = {}
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem
val.modelXbrl.modelManager.validateDisclosureSystem = False
deprecationsInstance = ModelXbrl.load(val.modelXbrl.modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
# load from zip (especially after caching) is incredibly faster
openFileSource(ugtDocLB, cntlr),
_("built deprecations table in cache"))
val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
if deprecationsInstance is None:
val.modelXbrl.error("arelle:notLoaded",
_("US-GAAP documentation not loaded: %(file)s"),
modelXbrl=val, file=os.path.basename(ugtDocLB))
else:
# load deprecations
for labelRel in deprecationsInstance.relationshipSet(XbrlConst.conceptLabel).modelRelationships:
modelDocumentation = labelRel.toModelObject
conceptName = labelRel.fromModelObject.name
if modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedLabel':
val.usgaapDeprecations[conceptName] = (val.usgaapDeprecations.get(conceptName, ('',''))[0], modelDocumentation.text)
elif modelDocumentation.role == 'http://www.xbrl.org/2009/role/deprecatedDateLabel':
val.usgaapDeprecations[conceptName] = (modelDocumentation.text, val.usgaapDeprecations.get(conceptName, ('',''))[1])
jsonStr = _STR_UNICODE(json.dumps(val.usgaapDeprecations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, deprecationsJsonFile, jsonStr) # 2.7 gets unicode this way
deprecationsInstance.close()
del deprecationsInstance # dereference closed modelXbrl
val.modelXbrl.profileStat(_("build us-gaap deprecated concepts cache"), time.time() - startedAt)
ugtCalcsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-calculations.json"
ugtDefaultDimensionsJsonFile = usgaapDoc.filepathdir + os.sep + "ugt-default-dimensions.json"
file = None
try:
file = openFileStream(cntlr, ugtCalcsJsonFile, 'rt', encoding='utf-8')
val.usgaapCalculations = json.load(file)
file.close()
file = openFileStream(cntlr, ugtDefaultDimensionsJsonFile, 'rt', encoding='utf-8')
val.usgaapDefaultDimensions = json.load(file)
file.close()
except Exception:
if file:
file.close()
val.modelXbrl.modelManager.addToLog(_("loading us-gaap {0} calculations and default dimensions into cache").format(ugt["year"]))
startedAt = time.time()
ugtEntryXsd = ugt["entryXsd"]
val.usgaapCalculations = {}
val.usgaapDefaultDimensions = {}
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = val.modelXbrl.modelManager.validateDisclosureSystem
val.modelXbrl.modelManager.validateDisclosureSystem = False
calculationsInstance = ModelXbrl.load(val.modelXbrl.modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/entire/us-gaap-entryPoint-std-2012-01-31.xsd",
# load from zip (especially after caching) is incredibly faster
openFileSource(ugtEntryXsd, cntlr),
_("built us-gaap calculations cache"))
val.modelXbrl.modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
if calculationsInstance is None:
val.modelXbrl.error("arelle:notLoaded",
_("US-GAAP calculations not loaded: %(file)s"),
modelXbrl=val, file=os.path.basename(ugtEntryXsd))
else:
# load calculations
for ELR in calculationsInstance.relationshipSet(XbrlConst.summationItem).linkRoleUris:
elrRelSet = calculationsInstance.relationshipSet(XbrlConst.summationItem, ELR)
definition = ""
for roleType in calculationsInstance.roleTypes.get(ELR,()):
definition = roleType.definition
break
isStatementSheet = bool(val.linkroleDefinitionStatementSheet.match(definition))
elrUgtCalcs = {"#roots": [c.name for c in elrRelSet.rootConcepts],
"#definition": definition,
"#isStatementSheet": isStatementSheet}
for relFrom, rels in elrRelSet.fromModelObjects().items():
elrUgtCalcs[relFrom.name] = [rel.toModelObject.name for rel in rels]
val.usgaapCalculations[ELR] = elrUgtCalcs
jsonStr = _STR_UNICODE(json.dumps(val.usgaapCalculations, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, ugtCalcsJsonFile, jsonStr) # 2.7 gets unicode this way
# load default dimensions
for defaultDimRel in calculationsInstance.relationshipSet(XbrlConst.dimensionDefault).modelRelationships:
if defaultDimRel.fromModelObject is not None and defaultDimRel.toModelObject is not None:
val.usgaapDefaultDimensions[defaultDimRel.fromModelObject.name] = defaultDimRel.toModelObject.name
jsonStr = _STR_UNICODE(json.dumps(val.usgaapDefaultDimensions, ensure_ascii=False, indent=0)) # might not be unicode in 2.7
saveFile(cntlr, ugtDefaultDimensionsJsonFile, jsonStr) # 2.7 gets unicode this way
calculationsInstance.close()
del calculationsInstance # dereference closed modelXbrl
val.modelXbrl.profileStat(_("build us-gaap calculations and default dimensions cache"), time.time() - startedAt)
break
val.deprecatedFactConcepts = defaultdict(list)
val.deprecatedDimensions = defaultdict(list)
val.deprecatedMembers = defaultdict(list)
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
else: # need own file source, may need instance discovery
filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
if filesource and not filesource.selection and filesource.isArchive:
for _archiveFile in filesource.dir: # find instance document in archive
filesource.select(_archiveFile)
if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
break # use this selection
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel)
if modelXbrl.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelXbrl.close()
self.determineNotLoadedTestStatus(modelTestcaseVariation)
elif resultIsVersioningReport:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
self.modelXbrl.error("exception",
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
RenderingEvaluator.init(modelXbrl)
if modelXbrl.hasFormulae:
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
self.modelXbrl.error("exception",
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("formula:resultFactCounts",
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts))
else:
for fact in expectedInstance.facts:
unmatchedFactsStack = []
if formulaOutputInstance.matchFact(fact, unmatchedFactsStack, deemP0inf=True) is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = fact
formulaOutputInstance.error("formula:expectedFactMissing",
_("Formula output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname)
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def loadDqc0015signwarningRules(modelXbrl):
conceptRule = "http://fasb.org/dqcrules/arcrole/concept-rule" # FASB arcrule
rule0015 = "http://fasb.org/us-gaap/role/dqc/0015"
modelManager = modelXbrl.modelManager
cntlr = modelXbrl.modelManager.cntlr
# check for cached completed signwarnings
_signwarningsFileName = resourcesFilePath(modelManager, "signwarnings.json")
if os.path.exists(_signwarningsFileName):
_file = openFileStream(modelManager.cntlr, _signwarningsFileName, 'rt', encoding='utf-8')
signwarnings = json.load(_file) # {localName: date, ...}
_file.close()
return signwarnings
# load template rules
_fileName = resourcesFilePath(modelManager, "signwarnings-template.json")
if _fileName:
_file = openFileStream(modelXbrl.modelManager.cntlr, _fileName, 'rt', encoding='utf-8')
signwarnings = json.load(_file, object_pairs_hook=OrderedDict) # {localName: date, ...}
_file.close()
# load rules and add to signwarnings template
for dqcAbbr, dqcrtUrl in latestDqcrtDocs.items():
modelManager.addToLog(_("loading {} DQC Rules {}").format(dqcAbbr, dqcrtUrl), messageCode="info")
# load without SEC/EFM validation (doc file would not be acceptable)
priorValidateDisclosureSystem = modelManager.validateDisclosureSystem
modelManager.validateDisclosureSystem = False
from arelle import ModelXbrl
dqcrtInstance = ModelXbrl.load(modelManager,
# "http://xbrl.fasb.org/us-gaap/2012/elts/us-gaap-doc-2012-01-31.xml",
# load from zip (especially after caching) is incredibly faster
openFileSource(dqcrtUrl, cntlr),
_("built dqcrt table in cache"))
modelManager.validateDisclosureSystem = priorValidateDisclosureSystem
if dqcrtInstance is None:
modelManager.addToLog(
_("%(name)s documentation not loaded"),
messageCode="arelle:notLoaded", messageArgs={"modelXbrl": val, "name":dqcAbbr})
else:
# load signwarnings from DQC 0015
dqcRelSet = dqcrtInstance.relationshipSet(conceptRule, rule0015)
for signWrnObj, headEltName in (("conceptNames", "Dqc_0015_ListOfElements"),
("excludedMemberNames", "Dqc_0015_ExcludeNonNegMembersAbstract"),
("excludedAxesMembers", "Dqc_0015_ExcludeNonNegAxisAbstract"),
("excludedAxesMembers", "Dqc_0015_ExcludeNonNegAxisMembersAbstract"),
("excludedMemberStrings", "Dqc_0015_ExcludeNonNegMemberStringsAbstract")):
headElts = dqcrtInstance.nameConcepts.get(headEltName,())
for headElt in headElts:
if signWrnObj == "excludedMemberStrings":
for refRel in dqcrtInstance.relationshipSet(XbrlConst.conceptReference).fromModelObject(headElt):
for refPart in refRel.toModelObject.iterchildren("{*}allowableSubString"):
for subStr in refPart.text.split():
signwarnings[signWrnObj].setdefault(nsAbbr, []).append(subStr)
else:
for ruleRel in dqcRelSet.fromModelObject(headElt):
elt = ruleRel.toModelObject
nsAbbr = abbreviatedNamespace(elt.qname.namespaceURI)
if signWrnObj in ("conceptNames", "excludedMemberNames"):
signwarnings[signWrnObj].setdefault(nsAbbr, []).append(elt.name)
else:
l = signwarnings[signWrnObj].setdefault(nsAbbr, {}).setdefault(elt.name, [])
if headEltName == "Dqc_0015_ExcludeNonNegAxisAbstract":
l.append("*")
else:
for memRel in dqcRelSet.fromModelObject(elt):
l.append(memRel.toModelObject.name)
jsonStr = _STR_UNICODE(json.dumps(signwarnings, ensure_ascii=False, indent=2)) # might not be unicode in 2.7
saveFile(cntlr, _signwarningsFileName, jsonStr) # 2.7 gets unicode this way
dqcrtInstance.close()
del dqcrtInstance # dereference closed modelXbrl
return signwarnings
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
elif resultIsTaxonomyPackage:
from arelle import PackageManager, PrototypeInstanceObject
dtsName = readMeFirstUri
modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
else: # need own file source, may need instance discovery
filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
if filesource and not filesource.selection and filesource.isArchive:
for _archiveFile in filesource.dir: # find instance document in archive
filesource.select(_archiveFile)
if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
break # use this selection
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel)
modelXbrl.isTestcaseVariation = True
if modelXbrl.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif resultIsVersioningReport or resultIsTaxonomyPackage:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif resultIsTaxonomyPackage:
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
RenderingEvaluator.init(modelXbrl)
if modelXbrl.hasFormulae:
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
compareIxResultInstance = getattr(modelXbrl, "extractedInlineInstance", False) and modelTestcaseVariation.resultXbrlInstanceUri
if compareIxResultInstance:
formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
errMsgPrefix = "ix"
else: # delete input instances before formula output comparision
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
errMsgPrefix = "formula"
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts),
messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
else:
formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
def factFootnotes(fact, footnotesRelSet):
footnotes = []
footnoteRels = footnotesRelSet.fromModelObject(fact)
if footnoteRels:
# most process rels in same order between two instances, use labels to sort
for i, footnoteRel in enumerate(sorted(footnoteRels,
key=lambda r: (r.fromLabel,r.toLabel))):
modelObject = footnoteRel.toModelObject
if isinstance(modelObject, ModelResource):
xml = modelObject.viewText().strip()
footnotes.append("Footnote {}: {}".format(
i+1, # compare footnote with HTML serialized
xml,
#re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))
))
elif isinstance(modelObject, ModelFact):
footnotes.append("Footnoted fact {}: {} context: {} value: {}".format(
i+1,
modelObject.qname,
modelObject.contextID,
collapseWhitespace(modelObject.value)))
return footnotes
for expectedInstanceFact in expectedInstance.facts:
unmatchedFactsStack = []
formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True)
if formulaOutputFact is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = expectedInstanceFact
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else: # compare footnotes
expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
if expectedInstanceFactFootnotes != formulaOutputFactFootnotes:
formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
_("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=expectedInstanceFactFootnotes, footnotes2=formulaOutputFactFootnotes,
messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
if compareIxResultInstance:
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def _make_cache(val, ugt, cntlr, ugt_default_dimensions_json_file):
"""
Creates a new caches for the Taxonomy default dimensions
:param val: ValidateXbrl to be validated
:type val: :class: '~arelle.ValidateXbrl.ValidateXbrl'
:param ugt: Taxonomy to check
:type ugt: str
:param ugt_default_dimensions_json_file: location to save json default
dimensions
:type ugt_default_dimensions_json_file: str
:return: no explicit return, but saves caches for dqc_us_0041
:rtype: None
"""
started_at = time.time()
ugt_entry_xsd = ugt["entryXsd"]
val.usgaapDefaultDimensions = {}
prior_validate_disclosure_system = (
val.modelXbrl.modelManager.validateDisclosureSystem
)
val.modelXbrl.modelManager.validateDisclosureSystem = False
ugt_entry_xsd_instance = (
ModelXbrl.load(
val.modelXbrl.modelManager,
openFileSource(ugt_entry_xsd, cntlr),
_("opened us-gaap entry xsd") # noqa
)
)
val.modelXbrl.modelManager.validateDisclosureSystem = (
prior_validate_disclosure_system
)
if ugt_entry_xsd_instance is None:
val.modelXbrl.error(
"arelle:notLoaded",
_("US-GAAP entry xsd not loaded: %(file)s"), # noqa
modelXbrl=val,
file=os.path.basename(ugt_entry_xsd)
)
else:
model_relationships = (
ugt_entry_xsd_instance.relationshipSet(
XbrlConst.dimensionDefault
).modelRelationships
)
for default_dim_rel in model_relationships:
if _default_dim_rel_is_instance(default_dim_rel):
from_name = default_dim_rel.fromModelObject.name
to_name = default_dim_rel.toModelObject.name
val.usgaapDefaultDimensions[from_name] = to_name
json_str = str(
json.dumps(
val.usgaapDefaultDimensions,
ensure_ascii=False,
indent=0
)
) # might not be unicode in 2.7
# 2.7 gets unicode this way
saveFile(cntlr, ugt_default_dimensions_json_file, json_str)
ugt_entry_xsd_instance.close()
del ugt_entry_xsd_instance # dereference closed modelXbrl
val.modelXbrl.profileStat(
_("build default dimensions cache"), # noqa
time.time() - started_at
)
