def test_setVerbosityFromString(self): """Test that the log verbosity can be set with a string.""" log = runLog._RunLog(1) expectedStrVerbosity = "error" verbosityRank = log.getLogVerbosityRank(expectedStrVerbosity) runLog.setVerbosity(expectedStrVerbosity) self.assertEqual(verbosityRank, runLog.getVerbosity()) self.assertEqual(verbosityRank, logging.ERROR)
def test_setVerbosityFromInteger(self): """Test that the log verbosity can be set with an integer.""" log = runLog._RunLog(1) expectedStrVerbosity = "debug" verbosityRank = log.getLogVerbosityRank(expectedStrVerbosity) runLog.setVerbosity(verbosityRank) self.assertEqual(verbosityRank, runLog.getVerbosity()) self.assertEqual(verbosityRank, logging.DEBUG)
def cleanTempDirs(olderThanDays=None): """ Clean up temporary files after a run. The Windows HPC system sends a SIGBREAK signal when the user cancels a job, which is NOT handled by ``atexit``. Notably SIGBREAK doesn't exist off Windows. For the SIGBREAK signal to work with a Microsoft HPC, the ``TaskCancelGracePeriod`` option must be configured to be non-zero. This sets the period between SIGBREAK and SIGTERM/SIGINT. To do cleanups in this case, we must use the ``signal`` module. Actually, even then it does not work because MS ``mpiexec`` does not pass signals through. Parameters ---------- olderThanDays: int, optional If provided, deletes other ARMI directories if they are older than the requested time. """ disconnectAllHdfDBs() if os.path.exists(FAST_PATH): if runLog.getVerbosity() <= runLog.getLogVerbosityRank("extra"): print( "Cleaning up temporary files in: {}".format(FAST_PATH), file=sys.stdout ) try: shutil.rmtree(FAST_PATH) except Exception as error: # pylint: disable=broad-except for outputStream in (sys.stderr, sys.stdout): print( "Failed to delete temporary files in: {}\n" " error: {}".format(FAST_PATH, error), file=outputStream, ) # Also delete anything still here after `olderThanDays` days (in case this failed on # earlier runs) if olderThanDays is not None: gracePeriod = datetime.timedelta(days=olderThanDays) now = datetime.datetime.now() thisRunFolder = os.path.basename(FAST_PATH) for dirname in os.listdir(APP_DATA): dirPath = os.path.join(APP_DATA, dirname) if not os.path.isdir(dirPath): continue try: fromThisRun = dirname == thisRunFolder # second chance to delete _rank, dateString = dirname.split("-") dateOfFolder = datetime.datetime.strptime(dateString, "%Y%m%d%H%M%S%f") runIsOldAndLikleyComplete = (now - dateOfFolder) > gracePeriod if runIsOldAndLikleyComplete or fromThisRun: # Delete old files shutil.rmtree(dirPath) except: # pylint: disable=bare-except pass
def convertDatabase( inputDBName: str, outputDBName: Optional[str] = None, outputVersion: Optional[str] = None, ): """ Convert database files between different versions. Parameters ---------- inputDB name of the complete hierarchy database outputDB name of the output database that should be consistent with XTView outputVersion version of the database to convert to. Defaults to latest version """ dbIn = databaseFactory(inputDBName, permission=Permissions.READ_ONLY_FME) if dbIn.version == outputVersion: runLog.important( "The input database ({}) appears to already be in the desired " "format ({})".format(inputDBName, dbIn.version) ) return outputDBName = outputDBName or "-converted".join(os.path.splitext(inputDBName)) dbOut = databaseFactory( outputDBName, permission=Permissions.CREATE_FILE_TIE, version=outputVersion ) # each DB load resets the verbosity to that of the run. Here we allow # conversion users to overpower it. conversionVerbosity = runLog.getVerbosity() runLog.extra(f"Converting {dbIn} to DB version {outputVersion}") with dbIn, dbOut: # Making the bold assumption that we are working with HDF5 h5In = _getH5File(dbIn) h5Out = _getH5File(dbOut) dbOut.writeInputsToDB(None, *dbIn.readInputsFromDB()) for cycle, timeNode in dbIn.genTimeSteps(): runLog.extra(f"Converting cycle={cycle}, timeNode={timeNode}") r = dbIn.load(cycle, timeNode) runLog.setVerbosity(conversionVerbosity) dbOut.writeToDB(r) for auxPath in dbIn.genAuxiliaryData((cycle, timeNode)): name = next(reversed(auxPath.split("/"))) auxOutPath = dbOut.getAuxiliaryDataPath((cycle, timeNode), name) runLog.important( "Copying auxiliary data for time ({}, {}): {} -> {}".format( cycle, timeNode, auxPath, auxOutPath ) ) h5In.copy(auxPath, h5Out, name=auxOutPath)
def cleanTempDirs(olderThanDays=None): """ Clean up temporary files after a run. The Windows HPC system sends a SIGBREAK signal when the user cancels a job, which is NOT handled by ``atexit``. Notably SIGBREAK doesn't exist off Windows. For the SIGBREAK signal to work with a Microsoft HPC, the ``TaskCancelGracePeriod`` option must be configured to be non-zero. This sets the period between SIGBREAK and SIGTERM/SIGINT. To do cleanups in this case, we must use the ``signal`` module. Actually, even then it does not work because MS ``mpiexec`` does not pass signals through. Parameters ---------- olderThanDays: int, optional If provided, deletes other ARMI directories if they are older than the requested time. """ # pylint: disable=import-outside-toplevel # avoid cyclic import from armi import runLog from armi.utils.pathTools import cleanPath disconnectAllHdfDBs() printMsg = runLog.getVerbosity() <= DEBUG if _FAST_PATH_IS_TEMPORARY and os.path.exists(_FAST_PATH): if printMsg: print( "Cleaning up temporary files in: {}".format(_FAST_PATH), file=sys.stdout, ) try: cleanPath(_FAST_PATH) except Exception as error: # pylint: disable=broad-except for outputStream in (sys.stderr, sys.stdout): if printMsg: print( "Failed to delete temporary files in: {}\n" " error: {}".format(_FAST_PATH, error), file=outputStream, ) if olderThanDays is not None: cleanAllArmiTempDirs(olderThanDays)
def convertDatabase( inputDBName: str, outputDBName: Optional[str] = None, outputVersion: Optional[str] = None, nodes: Optional[List[Tuple[int, int]]] = None, ): """ Convert database files between different versions. Parameters ---------- inputDB name of the complete hierarchy database outputDB name of the output database that should be consistent with XTView outputVersion version of the database to convert to. Defaults to latest version nodes optional list of specific (cycle,node)s to convert """ dbIn = databaseFactory(inputDBName, permission=Permissions.READ_ONLY_FME) if dbIn.version == outputVersion: runLog.important( "The input database ({}) appears to already be in the desired " "format ({})".format(inputDBName, dbIn.version) ) return outputDBName = outputDBName or "-converted".join(os.path.splitext(inputDBName)) dbOut = databaseFactory( outputDBName, permission=Permissions.CREATE_FILE_TIE, version=outputVersion ) # each DB load resets the verbosity to that of the run. Here we allow # conversion users to overpower it. conversionVerbosity = runLog.getVerbosity() runLog.extra(f"Converting {dbIn} to DB version {outputVersion}") with dbIn, dbOut: dbNodes = list(dbIn.genTimeSteps()) if nodes is not None and any(node not in dbNodes for node in nodes): raise RuntimeError( "Some of the requested nodes are not in the source database.\n" "Requested: {}\n" "Present: {}".format(nodes, dbNodes) ) # Making the bold assumption that we are working with HDF5 h5In = _getH5File(dbIn) h5Out = _getH5File(dbOut) dbOut.writeInputsToDB(None, *dbIn.readInputsFromDB()) for cycle, timeNode in dbNodes: if nodes is not None and (cycle, timeNode) not in nodes: continue runLog.extra(f"Converting cycle={cycle}, timeNode={timeNode}") timeStepsInOutDB = set(dbOut.genTimeSteps()) r = dbIn.load(cycle, timeNode) if (r.p.cycle, r.p.timeNode) in timeStepsInOutDB: runLog.warning( "Time step ({}, {}) is already in the output DB. This " "is probably due to repeated cycle/timeNode in the source DB; " "deleting the existing time step and re-writing".format( r.p.cycle, r.p.timeNode ) ) del dbOut[r.p.cycle, r.p.timeNode, None] runLog.setVerbosity(conversionVerbosity) dbOut.writeToDB(r) for auxPath in dbIn.genAuxiliaryData((cycle, timeNode)): name = next(reversed(auxPath.split("/"))) auxOutPath = dbOut.getAuxiliaryDataPath((cycle, timeNode), name) runLog.important( "Copying auxiliary data for time ({}, {}): {} -> {}".format( cycle, timeNode, auxPath, auxOutPath ) ) h5In.copy(auxPath, h5Out, name=auxOutPath)
def test_setVerbosityFromString(self): """Test that the log verbosity can be set with a string.""" expectedStrVerbosity = runLog.getLogVerbosityLevels()[0] verbosityRank = runLog.getLogVerbosityRank(expectedStrVerbosity) runLog.setVerbosity(expectedStrVerbosity) self.assertEqual(verbosityRank, runLog.getVerbosity())