def get_test_data(args): """ Produces a list of Pytest Params that can be fed into a parameterized pytest function :param args: The args to be parsed by arelle in order to correctly produce the desired result set :type args: list of strings :return: A list of PyTest Params that can be used to run a parameterized pytest function :rtype: list of ::class:: `~pytest.param` """ cntlr = parseAndRun(args) results = [] model_document = cntlr.modelManager.modelXbrl.modelDocument if model_document is not None: if model_document.type == ModelDocument.Type.TESTCASESINDEX: for tc in sorted(model_document.referencesDocument.keys(), key=lambda doc: doc.uri): uri_dir_parts = os.path.dirname(tc.uri).split('/') test_case_dir = '/'.join(uri_dir_parts[-2:]) if hasattr(tc, "testcaseVariations"): for mv in tc.testcaseVariations: param = pytest.param( { 'status': mv.status, 'expected': mv.expected, 'actual': mv.actual }, id='{}/{}'.format(test_case_dir, str(mv.id or mv.name))) results.append(param) cntlr.modelManager.close() PackageManager.close() PluginManager.close() return results
def runTest(section, args): print ("run tests") # ?? print does not come out to console or log, want to show progress cntlr = parseAndRun(args) # log to print (only failed assertions are captured) outcomes = [] if '--validate' in args: modelDocument = cntlr.modelManager.modelXbrl.modelDocument if modelDocument is not None: if modelDocument.type in (ModelDocument.Type.TESTCASESINDEX, ModelDocument.Type.REGISTRY): index = os.path.basename(modelDocument.uri) for tc in sorted(modelDocument.referencesDocument.keys(), key=lambda doc: doc.uri): test_case = os.path.basename(tc.uri) if hasattr(tc, "testcaseVariations"): for mv in tc.testcaseVariations: outcomes.append({'section': section, 'testcase': test_case, 'variation': str(mv.id or mv.name), # copy string to dereference mv 'name': str(mv.description or mv.name), 'status': str(mv.status), 'expected': str(mv.expected), 'actual': str(mv.actual)}) elif modelDocument.type in (ModelDocument.Type.TESTCASE, ModelDocument.Type.REGISTRYTESTCASE): tc = modelDocument test_case = os.path.basename(tc.uri) if hasattr(tc, "testcaseVariations"): for mv in tc.testcaseVariations: outcomes.append({'section': section, 'testcase': test_case, 'variation': str(mv.id or mv.name), 'name': str(mv.description or mv.name), 'status': str(mv.status), 'expected': str(mv.expected), 'actual': str(mv.actual)}) elif modelDocument.type == ModelDocument.Type.RSSFEED: tc = modelDocument if hasattr(tc, "rssItems"): for rssItem in tc.rssItems: outcomes.append({'section': section, 'testcase': os.path.basename(rssItem.url), 'variation': str(rssItem.accessionNumber), 'name': str(rssItem.formType + " " + rssItem.cikNumber + " " + rssItem.companyName + " " + str(rssItem.period) + " " + str(rssItem.filingDate)), 'status': str(rssItem.status), 'expected': rssItem.url, 'actual': " ".join(str(result) for result in (rssItem.results or [])) + ((" " + str(rssItem.assertions)) if rssItem.assertions else "")}) del modelDocument # dereference cntlr.modelManager.close() del cntlr # dereference return outcomes
def run_efm_conformance_suite(args): """ Kicks off the validation of the EFM conformance suite. :param args: The arguments to pass to Arelle in order to accurately validate the EFM conformance suite :param args: list of str :return: Returns the result of parseAndRun which in this case is the created controller object :rtype: ::class:: `~arelle.CntlrCmdLine.CntlrCmdLine` """ return parseAndRun(args)
def test_utr_structure_malformed_utrs_conformance_suite(malformed_utr_file): args = ARGS + [ # any valid file that refers to units, hopefully small, since it's not the point of the test '--file', os.path.join(STRUCTURE_CONFORMANCE_SUITE, 'tests', '01-simple', 'simpleValid.xml'), '--utrUrl', os.path.join(STRUCTURE_CONFORMANCE_SUITE_ZIP, malformed_utr_file), ] controller = parseAndRun(args) errors = controller.modelManager.modelXbrl.errors # when it can't find the file, it returns 'arelleUtrLoader:error' assert errors and errors != ['arelleUtrLoader:error']
def runTest(section, args): print( "run tests" ) # ?? print does not come out to console or log, want to show progress cntlr = parseAndRun( args) # log to print (only failed assertions are captured) outcomes = [] if '--validate' in args: modelDocument = cntlr.modelManager.modelXbrl.modelDocument if modelDocument is not None: if modelDocument.type in (ModelDocument.Type.TESTCASESINDEX, ModelDocument.Type.REGISTRY): index = os.path.basename(modelDocument.uri) for tc in sorted(modelDocument.referencesDocument.keys(), key=lambda doc: doc.uri): test_case = os.path.basename(tc.uri) if hasattr(tc, "testcaseVariations"): for mv in tc.testcaseVariations: outcomes.append({ 'section': section, 'testcase': test_case, 'variation': str(mv.id), # copy string to dereference mv 'name': str(mv.name), 'status': str(mv.status), 'expected': str(mv.expected), 'actual': str(mv.actual) }) elif modelDocument.type in (ModelDocument.Type.TESTCASE, ModelDocument.Type.REGISTRYTESTCASE): tc = modelDocument test_case = os.path.basename(tc.uri) if hasattr(tc, "testcaseVariations"): for mv in tc.testcaseVariations: outcomes.append({ 'section': section, 'testcase': test_case, 'variation': str(mv.id), 'name': str(mv.name), 'status': str(mv.status), 'expected': str(mv.expected), 'actual': str(mv.actual) }) del modelDocument # dereference cntlr.modelManager.close() del cntlr # dereference return outcomes
def runSuite(self, suite, reportFilename, logf, additionalOptions=None): testDir = self.getTestDir() testSuite = testDir + "/" + suite print("Running " + testSuite) csvReport = testDir + "/tmp/" + reportFilename logFile = testDir + "/tmp/" + logf try: os.remove(csvReport) except: pass try: os.remove(logFile) except: pass arelleRunArgs = ['--csvTestReport', csvReport, '--logFile', 'logToStdErr', '--logfile', logFile, '--file', testSuite] if additionalOptions is not None: arelleRunArgs.extend(additionalOptions) parseAndRun(arelleRunArgs) # Here we compare the whole report to a reference # (at least we should have the same statuses (pass,fail,valid...) assert isSameFileContent(csvReport, testDir + "/references/" + reportFilename)
def test(self): tr = tracker.SummaryTracker() testDir = os.path.dirname(os.path.abspath(sys.modules[__name__].__file__)) testFileSmall = testDir + "/solvency/2.0/random/spv_20_instance.xbrl" logFile = testDir + "/tmp/test.log" dumpFilePrefix = testDir + "/tmp/dump_" prevNumObjects = 0 for idx in range(3): # increase this range for testing print("\nIteration " + str(idx)) arelleRunArgs = ['--keepOpen', '--logFile', 'logToStdErr', '--logfile', logFile, '--file', testFileSmall] cntlr = parseAndRun(arelleRunArgs) cntlr.modelManager.close() cntlr.close() del cntlr gc.collect() all_objects = muppy.get_objects() numObjects = len(all_objects) diffObjects = numObjects - prevNumObjects prevNumObjects = numObjects print(str(numObjects) + " (" + str(diffObjects) + " more)") browserRoot = None if False: # <<<--- set this to get object dump file with open(dumpFilePrefix + str(idx) + ".txt", "w") as text_file: idx = 0 for o in all_objects: if browserRoot is None and isinstance(o, arelle.ModelValue.QName): browserRoot = o idx += 1 otype = "" try: otype = str(type(o)) except: pass try: print("type=" + otype + " " + str(o), file=text_file) except: pass if False: ibrowser = refbrowser.InteractiveBrowser(browserRoot) ibrowser.main() all_objects= None gc.collect() tr.print_diff() if idx > 1: assert diffObjects < 50, "Check for new objects leak"
def runTest(section, args): print ("run tests") # ?? print does not come out to console or log, want to show progress cntlr = parseAndRun(args) # log to print (only failed assertions are captured) outcomes = [] if '--validate' in args: modelDocument = cntlr.modelManager.modelXbrl.modelDocument if modelDocument is not None: if modelDocument.type in (ModelDocument.Type.TESTCASESINDEX, ModelDocument.Type.REGISTRY): index = os.path.basename(modelDocument.uri) for tc in sorted(modelDocument.referencesDocument.keys(), key=lambda doc: doc.uri): test_case = os.path.basename(tc.uri) if hasattr(tc, "testcaseVariations"): for mv in tc.testcaseVariations: outcomes.append({'section': section, 'testcase': test_case, 'variation': str(mv.id), # copy string to dereference mv 'name': str(mv.name), 'status': str(mv.status), 'expected': str(mv.expected), 'actual': str(mv.actual)}) elif modelDocument.type in (ModelDocument.Type.TESTCASE, ModelDocument.Type.REGISTRYTESTCASE): tc = modelDocument test_case = os.path.basename(tc.uri) if hasattr(tc, "testcaseVariations"): for mv in tc.testcaseVariations: outcomes.append({'section': section, 'testcase': test_case, 'variation': str(mv.id), 'name': str(mv.name), 'status': str(mv.status), 'expected': str(mv.expected), 'actual': str(mv.actual)}) del modelDocument # dereference cntlr.modelManager.close() del cntlr # dereference return outcomes
'--csvTestReport', 'UTRunit-report.csv', '--logFile', 'UTRunit-log.txt', ] STRUCTURE_ARGS = BASE_ARGS + [ '--file', os.path.join(STRUCTURE_CONFORMANCE_SUITE, 'index.xml'), '--utrUrl', os.path.join(STRUCTURE_CONFORMANCE_SUITE, 'utr-for-structure-conformance-tests.xml'), '--csvTestReport', 'UTRstr-report.csv', '--logFile', 'UTRstr-log.txt', ] if __name__ == "__main__": print('Running registry tests...') parseAndRun(REGISTRY_ARGS) print('Running structure tests...') parseAndRun(STRUCTURE_ARGS) print('Running malformed UTRs tests...') malformed_utr_files = [] with zipfile.ZipFile(STRUCTURE_CONFORMANCE_SUITE_ZIP, 'r') as zipf: for f in zipfile.Path(zipf, 'conf/utr-structure/malformed-utrs/').iterdir(): if f.is_file() and f.name.endswith('.xml'): malformed_utr_files.append((f.at, f.name)) for path_in_zip, name in malformed_utr_files: basename = name.removesuffix('.xml') args = BASE_ARGS + [ '--file', os.path.join(STRUCTURE_CONFORMANCE_SUITE, 'tests', '01-simple', 'simpleValid.xml'), '--utrUrl', os.path.join(STRUCTURE_CONFORMANCE_SUITE_ZIP, path_in_zip),