def read_xml(self): if (len(self._env_files_that_need_rewrite) > 0): files = "" for env_file in self._env_files_that_need_rewrite: files += " " + env_file.filename expect( False, "Object(s) %s seem to have newer data than the corresponding case file" % files) self._env_entryid_files = [] self._env_entryid_files.append(EnvCase(self._caseroot, components=None)) components = self._env_entryid_files[0].get_values("COMP_CLASSES") self._env_entryid_files.append( EnvRun(self._caseroot, components=components)) self._env_entryid_files.append( EnvBuild(self._caseroot, components=components)) self._env_entryid_files.append( EnvMachPes(self._caseroot, components=components)) if os.path.isfile(os.path.join(self._caseroot, "env_test.xml")): self._env_entryid_files.append( EnvTest(self._caseroot, components=components)) self._env_generic_files = [] self._env_generic_files.append(EnvBatch(self._caseroot)) self._env_generic_files.append(EnvMachSpecific(self._caseroot)) self._env_generic_files.append(EnvArchive(self._caseroot)) self._files = self._env_entryid_files + self._env_generic_files
def check_lockedfile(self, filebase): caseroot = self.get_value("CASEROOT") cfile = os.path.join(caseroot, filebase) lfile = os.path.join(caseroot, "LockedFiles", filebase) components = self.get_values("COMP_CLASSES") if os.path.isfile(cfile): objname = filebase.split('.')[0] if objname == "env_build": f1obj = self.get_env('build') f2obj = EnvBuild(caseroot, lfile) elif objname == "env_mach_pes": f1obj = self.get_env('mach_pes') f2obj = EnvMachPes(caseroot, lfile, components=components) elif objname == "env_case": f1obj = self.get_env('case') f2obj = EnvCase(caseroot, lfile) elif objname == "env_batch": f1obj = self.get_env('batch') f2obj = EnvBatch(caseroot, lfile) else: logging.warning("Locked XML file '{}' is not current being handled".format(filebase)) return diffs = f1obj.compare_xml(f2obj) if diffs: logging.warning("File {} has been modified".format(lfile)) toggle_build_status = False for key in diffs.keys(): if key != "BUILD_COMPLETE": print(" found difference in {} : case {} locked {}" .format(key, repr(diffs[key][0]), repr(diffs[key][1]))) toggle_build_status = True if objname == "env_mach_pes": expect(False, "Invoke case.setup --reset ") elif objname == "env_case": expect(False, "Cannot change file env_case.xml, please" " recover the original copy from LockedFiles") elif objname == "env_build": if toggle_build_status: logging.warning("Setting build complete to False") self.set_value("BUILD_COMPLETE", False) if "PIO_VERSION" in diffs: self.set_value("BUILD_STATUS", 2) logging.critical("Changing PIO_VERSION requires running " "case.build --clean-all and rebuilding") else: self.set_value("BUILD_STATUS", 1) f2obj.set_value("BUILD_COMPLETE", False) elif objname == "env_batch": expect(False, "Batch configuration has changed, please run case.setup --reset") else: expect(False, "'{}' diff was not handled".format(objname))
def check_lockedfiles(case): """ Check that all lockedfiles match what's in case If caseroot is not specified, it is set to the current working directory """ caseroot = case.get_value("CASEROOT") lockedfiles = glob.glob(os.path.join(caseroot, "LockedFiles", "*.xml")) for lfile in lockedfiles: fpart = os.path.basename(lfile) # ignore files used for tests such as env_mach_pes.ERP1.xml by looking for extra dots in the name if fpart.count('.') > 1: continue cfile = os.path.join(caseroot, fpart) components = case.get_values("COMP_CLASSES") if os.path.isfile(cfile): objname = fpart.split('.')[0] if objname == "env_build": f1obj = case.get_env('build') f2obj = EnvBuild(caseroot, lfile) elif objname == "env_mach_pes": f1obj = case.get_env('mach_pes') f2obj = EnvMachPes(caseroot, lfile, components=components) elif objname == "env_case": f1obj = case.get_env('case') f2obj = EnvCase(caseroot, lfile) elif objname == "env_batch": f1obj = case.get_env('batch') f2obj = EnvBatch(caseroot, lfile) else: logging.warning( "Locked XML file '{}' is not current being handled".format( fpart)) continue diffs = f1obj.compare_xml(f2obj) if diffs: logging.warning("File {} has been modified".format(lfile)) for key in diffs.keys(): print( " found difference in {} : case {} locked {}".format( key, repr(diffs[key][0]), repr(diffs[key][1]))) if objname == "env_mach_pes": expect(False, "Invoke case.setup --reset ") elif objname == "env_case": expect( False, "Cannot change file env_case.xml, please" " recover the original copy from LockedFiles") elif objname == "env_build": logging.warning("Setting build complete to False") case.set_value("BUILD_COMPLETE", False) if "PIO_VERSION" in diffs: case.set_value("BUILD_STATUS", 2) logging.critical( "Changing PIO_VERSION requires running " "case.build --clean-all and rebuilding") else: case.set_value("BUILD_STATUS", 1) elif objname == "env_batch": expect( False, "Batch configuration has changed, please run case.setup --reset" ) else: expect(False, "'{}' diff was not handled".format(objname))
def check_lockedfiles(caseroot=os.getcwd()): """ Check that all lockedfiles match what's in case """ lockedfiles = glob.glob(os.path.join(caseroot, "LockedFiles", "*.xml")) for lfile in lockedfiles: fpart = os.path.basename(lfile) cfile = os.path.join(caseroot, fpart) if os.path.isfile(cfile): objname = fpart.split('.')[0] logging.info("Checking file %s"%objname) if objname == "env_build": f1obj = EnvBuild(caseroot, cfile) f2obj = EnvBuild(caseroot, lfile) elif objname == "env_mach_pes": f1obj = EnvMachPes(caseroot, cfile) f2obj = EnvMachPes(caseroot, lfile) elif objname == "env_case": f1obj = EnvCase(caseroot, cfile) f2obj = EnvCase(caseroot, lfile) diffs = f1obj.compare_xml(f2obj) if diffs: logging.warn("File %s has been modified"%lfile) for key in diffs.keys(): print(" found difference in %s : case %s locked %s" % (key, repr(diffs[key][0]), repr(diffs[key][1]))) if objname == "env_mach_pes": expect(False, "Invoke case.setup --clean followed by case.setup") elif objname == "env_case": expect(False, "Cannot change file env_case.xml, please" " recover the original copy from LockedFiles") elif objname == "env_build": logging.warn("Setting build complete to False") f1obj.set_value("BUILD_COMPLETE", False) if "PIO_VERSION" in diffs.keys(): f1obj.set_value("BUILD_STATUS", 2) f1obj.write() logging.critical("Changing PIO_VERSION requires running " "case.build --clean-all and rebuilding") else: f1obj.set_value("BUILD_STATUS", 1) f1obj.write() else: # JGF : Error? pass
def check_lockedfiles(caseroot=None): """ Check that all lockedfiles match what's in case If caseroot is not specified, it is set to the current working directory """ caseroot = os.getcwd() if caseroot is None else caseroot lockedfiles = glob.glob(os.path.join(caseroot, "LockedFiles", "*.xml")) for lfile in lockedfiles: fpart = os.path.basename(lfile) cfile = os.path.join(caseroot, fpart) if os.path.isfile(cfile): objname = fpart.split('.')[0] logging.info("Checking file %s" % objname) if objname == "env_build": f1obj = EnvBuild(caseroot, cfile) f2obj = EnvBuild(caseroot, lfile) elif objname == "env_mach_pes": f1obj = EnvMachPes(caseroot, cfile) f2obj = EnvMachPes(caseroot, lfile) elif objname == "env_case": f1obj = EnvCase(caseroot, cfile) f2obj = EnvCase(caseroot, lfile) diffs = f1obj.compare_xml(f2obj) if diffs: logging.warn("File %s has been modified" % lfile) for key in diffs.keys(): print(" found difference in %s : case %s locked %s" % (key, repr(diffs[key][0]), repr(diffs[key][1]))) if objname == "env_mach_pes": expect(False, "Invoke case.setup --reset ") elif objname == "env_case": expect( False, "Cannot change file env_case.xml, please" " recover the original copy from LockedFiles") elif objname == "env_build": logging.warn("Setting build complete to False") f1obj.set_value("BUILD_COMPLETE", False) if "PIO_VERSION" in diffs.keys(): f1obj.set_value("BUILD_STATUS", 2) f1obj.write() logging.critical( "Changing PIO_VERSION requires running " "case.build --clean-all and rebuilding") else: f1obj.set_value("BUILD_STATUS", 1) f1obj.write() else: # JGF : Error? pass
def get_testreporter_xml(testroot, testid, tagname, testtype): ############################################################################### os.chdir(testroot) # # Retrieve compiler name and mpi library # xml_file = glob.glob("*" + testid + "/env_build.xml") expect( len(xml_file) > 0, "Tests not found. It's possible your testid, {} is wrong.".format( testid)) envxml = (EnvBuild(".", infile=xml_file[0])) compiler = envxml.get_value("COMPILER") mpilib = envxml.get_value("MPILIB") # # Retrieve machine name # xml_file = glob.glob("*" + testid + "/env_case.xml") envxml = (EnvCase(".", infile=xml_file[0])) machine = envxml.get_value("MACH") # # Retrieve baseline tag to compare to # xml_file = glob.glob("*" + testid + "/env_test.xml") envxml = (EnvTest(".", infile=xml_file[0])) baseline = envxml.get_value("BASELINE_NAME_CMP") # # Create XML header # testxml = TestReporter() testxml.setup_header(tagname, machine, compiler, mpilib, testroot, testtype, baseline) # # Create lists on tests based on the testid in the testroot directory. # test_names = glob.glob("*" + testid) # # Loop over all tests and parse the test results # test_status = {} for test_name in test_names: if not os.path.isfile(test_name + "/TestStatus"): continue test_status['COMMENT'] = "" test_status['BASELINE'] = '----' test_status['MEMCOMP'] = '----' test_status['MEMLEAK'] = '----' test_status['NLCOMP'] = '----' test_status['STATUS'] = '----' test_status['TPUTCOMP'] = '----' # # Check to see if TestStatus is present, if not then continue # I might want to set the status to fail # try: lines = [ line.rstrip('\n') for line in open(test_name + "/TestStatus") ] except (IOError, OSError): test_status['STATUS'] = "FAIL" test_status['COMMENT'] = "TestStatus missing. " continue # # Loop over each line of TestStatus, and check for different types of failures. # for line in lines: if "NLCOMP" in line: test_status['NLCOMP'] = line[0:4] if "MEMLEAK" in line: test_status['MEMLEAK'] = line[0:4] if "MEMCOMP" in line: test_status['MEMCOMP'] = line[0:4] if "BASELINE" in line: test_status['BASELINE'] = line[0:4] if "TPUTCOMP" in line: test_status['TPUTCOMP'] = line[0:4] if "FAIL PFS" in line: test_status['STATUS'] = "FAIL" if "INIT" in line: test_status['INIT'] = line[0:4] if line[0:4] in ("FAIL", "PEND"): test_status['STATUS'] = "SFAIL" test_status['COMMENT'] += "INIT fail! " break if "CREATE_NEWCASE" in line: test_status['CREATE_NEWCASE'] = line[0:4] if line[0:4] in ("FAIL", "PEND"): test_status['STATUS'] = "SFAIL" test_status['COMMENT'] += "CREATE_NEWCASE fail! " break if "XML" in line: test_status['XML'] = line[0:4] if line[0:4] in ("FAIL", "PEND"): test_status['STATUS'] = "SFAIL" test_status['COMMENT'] += "XML fail! " break if "SETUP" in line: test_status['SETUP'] = line[0:4] if line[0:4] in ("FAIL", "PEND"): test_status['STATUS'] = "SFAIL" test_status['COMMENT'] += "SETUP fail! " break if "SHAREDLIB_BUILD" in line: test_status['SHAREDLIB_BUILD'] = line[0:4] if line[0:4] in ("FAIL", "PEND"): test_status['STATUS'] = "CFAIL" test_status['COMMENT'] += "SHAREDLIB_BUILD fail! " break if "MODEL_BUILD" in line: test_status['MODEL_BUILD'] = line[0:4] if line[0:4] in ("FAIL", "PEND"): test_status['STATUS'] = "CFAIL" test_status['COMMENT'] += "MODEL_BUILD fail! " break if "SUBMIT" in line: test_status['STATUS'] = line[0:4] if line[0:4] in ("FAIL", "PEND"): test_status['COMMENT'] += "SUBMIT fail! " break if "RUN" in line: test_status['STATUS'] = line[0:4] if line[0:4] in ("FAIL", "PEND"): test_status['COMMENT'] += "RUN fail! " break if "COMPARE_base_rest" in line: test_status['STATUS'] = line[0:4] if line[0:4] in ("FAIL", "PEND"): test_status['COMMENT'] += "Restart fail! " break if "COMPARE_base_hybrid" in line: test_status['STATUS'] = line[0:4] if line[0:4] in ("FAIL", "PEND"): test_status['COMMENT'] += "Hybrid fail! " break if "COMPARE_base_multiinst" in line: test_status['STATUS'] = line[0:4] if line[0:4] in ("FAIL", "PEND"): test_status['COMMENT'] += "Multi instance fail! " break if "COMPARE_base_test" in line: test_status['STATUS'] = line[0:4] if line[0:4] in ("FAIL", "PEND"): test_status['COMMENT'] += "Base test fail! " break if "COMPARE_base_single_thread" in line: test_status['STATUS'] = line[0:4] if line[0:4] in ("FAIL", "PEND"): test_status['COMMENT'] += "Thread test fail! " break # # Do not include time comments. Just a preference to have cleaner comments in the test database # try: if 'time=' not in line and 'GENERATE' not in line: if 'BASELINE' not in line: test_status['COMMENT'] += line.split(' ', 3)[3] + ' ' else: test_status['COMMENT'] += line.split(' ', 4)[4] + ' ' except Exception: # Probably want to be more specific here pass # # Fill in the xml with the test results # testxml.add_result(test_name, test_status) return testxml
cleanflag = comp.lower() run_cmd_no_fail("./case.build --clean {}".format(cleanflag)) unlock_file("env_mach_pes.xml", case.get_value("CASEROOT")) def check_lockedfile(case, filebase): caseroot = case.get_value("CASEROOT") cfile = os.path.join(caseroot, filebase) lfile = os.path.join(caseroot, "LockedFiles", filebase) components = case.get_values("COMP_CLASSES") if os.path.isfile(cfile): objname = filebase.split('.')[0] if objname == "env_build": f1obj = case.get_env('build') f2obj = EnvBuild(caseroot, lfile) elif objname == "env_mach_pes": f1obj = case.get_env('mach_pes') f2obj = EnvMachPes(caseroot, lfile, components=components) elif objname == "env_case": f1obj = case.get_env('case') f2obj = EnvCase(caseroot, lfile) elif objname == "env_batch": f1obj = case.get_env('batch') f2obj = EnvBatch(caseroot, lfile) else: logging.warning("Locked XML file '{}' is not current being handled".format(filebase)) return diffs = f1obj.compare_xml(f2obj) if diffs: