def get_test_data(args):
"""
Produces a list of Pytest Params that can be fed into a parameterized pytest function
:param args: The args to be parsed by arelle in order to correctly produce the desired result set
:type args: list of strings
:return: A list of PyTest Params that can be used to run a parameterized pytest function
:rtype: list of ::class:: `~pytest.param`
"""
cntlr = parseAndRun(args)
results = []
model_document = cntlr.modelManager.modelXbrl.modelDocument
if model_document is not None:
if model_document.type == ModelDocument.Type.TESTCASESINDEX:
for tc in sorted(model_document.referencesDocument.keys(),
key=lambda doc: doc.uri):
uri_dir_parts = os.path.dirname(tc.uri).split('/')
test_case_dir = '/'.join(uri_dir_parts[-2:])
if hasattr(tc, "testcaseVariations"):
for mv in tc.testcaseVariations:
param = pytest.param(
{
'status': mv.status,
'expected': mv.expected,
'actual': mv.actual
},
id='{}/{}'.format(test_case_dir,
str(mv.id or mv.name)))
results.append(param)
cntlr.modelManager.close()
PackageManager.close()
PluginManager.close()
return results
def runTest(section, args):
print ("run tests") # ?? print does not come out to console or log, want to show progress
cntlr = parseAndRun(args) # log to print (only failed assertions are captured)
outcomes = []
if '--validate' in args:
modelDocument = cntlr.modelManager.modelXbrl.modelDocument
if modelDocument is not None:
if modelDocument.type in (ModelDocument.Type.TESTCASESINDEX,
ModelDocument.Type.REGISTRY):
index = os.path.basename(modelDocument.uri)
for tc in sorted(modelDocument.referencesDocument.keys(), key=lambda doc: doc.uri):
test_case = os.path.basename(tc.uri)
if hasattr(tc, "testcaseVariations"):
for mv in tc.testcaseVariations:
outcomes.append({'section': section,
'testcase': test_case,
'variation': str(mv.id or mv.name), # copy string to dereference mv
'name': str(mv.description or mv.name),
'status': str(mv.status),
'expected': str(mv.expected),
'actual': str(mv.actual)})
elif modelDocument.type in (ModelDocument.Type.TESTCASE,
ModelDocument.Type.REGISTRYTESTCASE):
tc = modelDocument
test_case = os.path.basename(tc.uri)
if hasattr(tc, "testcaseVariations"):
for mv in tc.testcaseVariations:
outcomes.append({'section': section,
'testcase': test_case,
'variation': str(mv.id or mv.name),
'name': str(mv.description or mv.name),
'status': str(mv.status),
'expected': str(mv.expected),
'actual': str(mv.actual)})
elif modelDocument.type == ModelDocument.Type.RSSFEED:
tc = modelDocument
if hasattr(tc, "rssItems"):
for rssItem in tc.rssItems:
outcomes.append({'section': section,
'testcase': os.path.basename(rssItem.url),
'variation': str(rssItem.accessionNumber),
'name': str(rssItem.formType + " " +
rssItem.cikNumber + " " +
rssItem.companyName + " " +
str(rssItem.period) + " " +
str(rssItem.filingDate)),
'status': str(rssItem.status),
'expected': rssItem.url,
'actual': " ".join(str(result) for result in (rssItem.results or [])) +
((" " + str(rssItem.assertions)) if rssItem.assertions else "")})
del modelDocument # dereference
cntlr.modelManager.close()
del cntlr # dereference
return outcomes
def runTest(section, args):
print ("run tests") # ?? print does not come out to console or log, want to show progress
cntlr = parseAndRun(args) # log to print (only failed assertions are captured)
outcomes = []
if '--validate' in args:
modelDocument = cntlr.modelManager.modelXbrl.modelDocument
if modelDocument is not None:
if modelDocument.type in (ModelDocument.Type.TESTCASESINDEX,
ModelDocument.Type.REGISTRY):
index = os.path.basename(modelDocument.uri)
for tc in sorted(modelDocument.referencesDocument.keys(), key=lambda doc: doc.uri):
test_case = os.path.basename(tc.uri)
if hasattr(tc, "testcaseVariations"):
for mv in tc.testcaseVariations:
outcomes.append({'section': section,
'testcase': test_case,
'variation': str(mv.id or mv.name), # copy string to dereference mv
'name': str(mv.description or mv.name),
'status': str(mv.status),
'expected': str(mv.expected),
'actual': str(mv.actual)})
elif modelDocument.type in (ModelDocument.Type.TESTCASE,
ModelDocument.Type.REGISTRYTESTCASE):
tc = modelDocument
test_case = os.path.basename(tc.uri)
if hasattr(tc, "testcaseVariations"):
for mv in tc.testcaseVariations:
outcomes.append({'section': section,
'testcase': test_case,
'variation': str(mv.id or mv.name),
'name': str(mv.description or mv.name),
'status': str(mv.status),
'expected': str(mv.expected),
'actual': str(mv.actual)})
elif modelDocument.type == ModelDocument.Type.RSSFEED:
tc = modelDocument
if hasattr(tc, "rssItems"):
for rssItem in tc.rssItems:
outcomes.append({'section': section,
'testcase': os.path.basename(rssItem.url),
'variation': str(rssItem.accessionNumber),
'name': str(rssItem.formType + " " +
rssItem.cikNumber + " " +
rssItem.companyName + " " +
str(rssItem.period) + " " +
str(rssItem.filingDate)),
'status': str(rssItem.status),
'expected': rssItem.url,
'actual': " ".join(str(result) for result in (rssItem.results or [])) +
((" " + str(rssItem.assertions)) if rssItem.assertions else "")})
del modelDocument # dereference
cntlr.modelManager.close()
del cntlr # dereference
return outcomes
def run_efm_conformance_suite(args):
"""
Kicks off the validation of the EFM conformance suite.
:param args: The arguments to pass to Arelle in order to accurately validate the EFM conformance suite
:param args: list of str
:return: Returns the result of parseAndRun which in this case is the created controller object
:rtype: ::class:: `~arelle.CntlrCmdLine.CntlrCmdLine`
"""
return parseAndRun(args)
def test_utr_structure_malformed_utrs_conformance_suite(malformed_utr_file):
args = ARGS + [
# any valid file that refers to units, hopefully small, since it's not the point of the test
'--file',
os.path.join(STRUCTURE_CONFORMANCE_SUITE, 'tests', '01-simple',
'simpleValid.xml'),
'--utrUrl',
os.path.join(STRUCTURE_CONFORMANCE_SUITE_ZIP, malformed_utr_file),
]
controller = parseAndRun(args)
errors = controller.modelManager.modelXbrl.errors
# when it can't find the file, it returns 'arelleUtrLoader:error'
assert errors and errors != ['arelleUtrLoader:error']
def runTest(section, args):
print(
"run tests"
) # ?? print does not come out to console or log, want to show progress
cntlr = parseAndRun(
args) # log to print (only failed assertions are captured)
outcomes = []
if '--validate' in args:
modelDocument = cntlr.modelManager.modelXbrl.modelDocument
if modelDocument is not None:
if modelDocument.type in (ModelDocument.Type.TESTCASESINDEX,
ModelDocument.Type.REGISTRY):
index = os.path.basename(modelDocument.uri)
for tc in sorted(modelDocument.referencesDocument.keys(),
key=lambda doc: doc.uri):
test_case = os.path.basename(tc.uri)
if hasattr(tc, "testcaseVariations"):
for mv in tc.testcaseVariations:
outcomes.append({
'section': section,
'testcase': test_case,
'variation':
str(mv.id), # copy string to dereference mv
'name': str(mv.name),
'status': str(mv.status),
'expected': str(mv.expected),
'actual': str(mv.actual)
})
elif modelDocument.type in (ModelDocument.Type.TESTCASE,
ModelDocument.Type.REGISTRYTESTCASE):
tc = modelDocument
test_case = os.path.basename(tc.uri)
if hasattr(tc, "testcaseVariations"):
for mv in tc.testcaseVariations:
outcomes.append({
'section': section,
'testcase': test_case,
'variation': str(mv.id),
'name': str(mv.name),
'status': str(mv.status),
'expected': str(mv.expected),
'actual': str(mv.actual)
})
del modelDocument # dereference
cntlr.modelManager.close()
del cntlr # dereference
return outcomes
def runSuite(self, suite, reportFilename, logf, additionalOptions=None):
testDir = self.getTestDir()
testSuite = testDir + "/" + suite
print("Running " + testSuite)
csvReport = testDir + "/tmp/" + reportFilename
logFile = testDir + "/tmp/" + logf
try:
os.remove(csvReport)
except:
pass
try:
os.remove(logFile)
except:
pass
arelleRunArgs = ['--csvTestReport', csvReport, '--logFile', 'logToStdErr', '--logfile', logFile, '--file', testSuite]
if additionalOptions is not None:
arelleRunArgs.extend(additionalOptions)
parseAndRun(arelleRunArgs)
# Here we compare the whole report to a reference
# (at least we should have the same statuses (pass,fail,valid...)
assert isSameFileContent(csvReport, testDir + "/references/" + reportFilename)
def test(self):
tr = tracker.SummaryTracker()
testDir = os.path.dirname(os.path.abspath(sys.modules[__name__].__file__))
testFileSmall = testDir + "/solvency/2.0/random/spv_20_instance.xbrl"
logFile = testDir + "/tmp/test.log"
dumpFilePrefix = testDir + "/tmp/dump_"
prevNumObjects = 0
for idx in range(3): # increase this range for testing
print("\nIteration " + str(idx))
arelleRunArgs = ['--keepOpen', '--logFile', 'logToStdErr', '--logfile', logFile, '--file', testFileSmall]
cntlr = parseAndRun(arelleRunArgs)
cntlr.modelManager.close()
cntlr.close()
del cntlr
gc.collect()
all_objects = muppy.get_objects()
numObjects = len(all_objects)
diffObjects = numObjects - prevNumObjects
prevNumObjects = numObjects
print(str(numObjects) + " (" + str(diffObjects) + " more)")
browserRoot = None
if False: # <<<--- set this to get object dump file
with open(dumpFilePrefix + str(idx) + ".txt", "w") as text_file:
idx = 0
for o in all_objects:
if browserRoot is None and isinstance(o, arelle.ModelValue.QName):
browserRoot = o
idx += 1
otype = ""
try:
otype = str(type(o))
except:
pass
try:
print("type=" + otype + " " + str(o), file=text_file)
except:
pass
if False:
ibrowser = refbrowser.InteractiveBrowser(browserRoot)
ibrowser.main()
all_objects= None
gc.collect()
tr.print_diff()
if idx > 1:
assert diffObjects < 50, "Check for new objects leak"
def runTest(section, args):
print ("run tests") # ?? print does not come out to console or log, want to show progress
cntlr = parseAndRun(args) # log to print (only failed assertions are captured)
outcomes = []
if '--validate' in args:
modelDocument = cntlr.modelManager.modelXbrl.modelDocument
if modelDocument is not None:
if modelDocument.type in (ModelDocument.Type.TESTCASESINDEX,
ModelDocument.Type.REGISTRY):
index = os.path.basename(modelDocument.uri)
for tc in sorted(modelDocument.referencesDocument.keys(), key=lambda doc: doc.uri):
test_case = os.path.basename(tc.uri)
if hasattr(tc, "testcaseVariations"):
for mv in tc.testcaseVariations:
outcomes.append({'section': section,
'testcase': test_case,
'variation': str(mv.id), # copy string to dereference mv
'name': str(mv.name),
'status': str(mv.status),
'expected': str(mv.expected),
'actual': str(mv.actual)})
elif modelDocument.type in (ModelDocument.Type.TESTCASE,
ModelDocument.Type.REGISTRYTESTCASE):
tc = modelDocument
test_case = os.path.basename(tc.uri)
if hasattr(tc, "testcaseVariations"):
for mv in tc.testcaseVariations:
outcomes.append({'section': section,
'testcase': test_case,
'variation': str(mv.id),
'name': str(mv.name),
'status': str(mv.status),
'expected': str(mv.expected),
'actual': str(mv.actual)})
del modelDocument # dereference
cntlr.modelManager.close()
del cntlr # dereference
return outcomes
'--csvTestReport', 'UTRunit-report.csv',
'--logFile', 'UTRunit-log.txt',
]
STRUCTURE_ARGS = BASE_ARGS + [
'--file', os.path.join(STRUCTURE_CONFORMANCE_SUITE, 'index.xml'),
'--utrUrl', os.path.join(STRUCTURE_CONFORMANCE_SUITE, 'utr-for-structure-conformance-tests.xml'),
'--csvTestReport', 'UTRstr-report.csv',
'--logFile', 'UTRstr-log.txt',
]
if __name__ == "__main__":
print('Running registry tests...')
parseAndRun(REGISTRY_ARGS)
print('Running structure tests...')
parseAndRun(STRUCTURE_ARGS)
print('Running malformed UTRs tests...')
malformed_utr_files = []
with zipfile.ZipFile(STRUCTURE_CONFORMANCE_SUITE_ZIP, 'r') as zipf:
for f in zipfile.Path(zipf, 'conf/utr-structure/malformed-utrs/').iterdir():
if f.is_file() and f.name.endswith('.xml'):
malformed_utr_files.append((f.at, f.name))
for path_in_zip, name in malformed_utr_files:
basename = name.removesuffix('.xml')
args = BASE_ARGS + [
'--file', os.path.join(STRUCTURE_CONFORMANCE_SUITE, 'tests', '01-simple', 'simpleValid.xml'),
'--utrUrl', os.path.join(STRUCTURE_CONFORMANCE_SUITE_ZIP, path_in_zip),
