def test_setVerbosityFromString(self):
"""Test that the log verbosity can be set with a string."""
log = runLog._RunLog(1)
expectedStrVerbosity = "error"
verbosityRank = log.getLogVerbosityRank(expectedStrVerbosity)
runLog.setVerbosity(expectedStrVerbosity)
self.assertEqual(verbosityRank, runLog.getVerbosity())
self.assertEqual(verbosityRank, logging.ERROR)
def test_setVerbosityFromInteger(self):
"""Test that the log verbosity can be set with an integer."""
log = runLog._RunLog(1)
expectedStrVerbosity = "debug"
verbosityRank = log.getLogVerbosityRank(expectedStrVerbosity)
runLog.setVerbosity(verbosityRank)
self.assertEqual(verbosityRank, runLog.getVerbosity())
self.assertEqual(verbosityRank, logging.DEBUG)
def cleanTempDirs(olderThanDays=None):
"""
Clean up temporary files after a run.
The Windows HPC system sends a SIGBREAK signal when the user cancels a job, which
is NOT handled by ``atexit``. Notably SIGBREAK doesn't exist off Windows.
For the SIGBREAK signal to work with a Microsoft HPC, the ``TaskCancelGracePeriod``
option must be configured to be non-zero. This sets the period between SIGBREAK
and SIGTERM/SIGINT. To do cleanups in this case, we must use the ``signal`` module.
Actually, even then it does not work because MS ``mpiexec`` does not pass signals
through.
Parameters
----------
olderThanDays: int, optional
If provided, deletes other ARMI directories if they are older than the requested
time.
"""
disconnectAllHdfDBs()
if os.path.exists(FAST_PATH):
if runLog.getVerbosity() <= runLog.getLogVerbosityRank("extra"):
print(
"Cleaning up temporary files in: {}".format(FAST_PATH), file=sys.stdout
)
try:
shutil.rmtree(FAST_PATH)
except Exception as error: # pylint: disable=broad-except
for outputStream in (sys.stderr, sys.stdout):
print(
"Failed to delete temporary files in: {}\n"
" error: {}".format(FAST_PATH, error),
file=outputStream,
)
# Also delete anything still here after `olderThanDays` days (in case this failed on
# earlier runs)
if olderThanDays is not None:
gracePeriod = datetime.timedelta(days=olderThanDays)
now = datetime.datetime.now()
thisRunFolder = os.path.basename(FAST_PATH)
for dirname in os.listdir(APP_DATA):
dirPath = os.path.join(APP_DATA, dirname)
if not os.path.isdir(dirPath):
continue
try:
fromThisRun = dirname == thisRunFolder # second chance to delete
_rank, dateString = dirname.split("-")
dateOfFolder = datetime.datetime.strptime(dateString, "%Y%m%d%H%M%S%f")
runIsOldAndLikleyComplete = (now - dateOfFolder) > gracePeriod
if runIsOldAndLikleyComplete or fromThisRun:
# Delete old files
shutil.rmtree(dirPath)
except: # pylint: disable=bare-except
pass
def convertDatabase(
inputDBName: str,
outputDBName: Optional[str] = None,
outputVersion: Optional[str] = None,
):
"""
Convert database files between different versions.
Parameters
----------
inputDB
name of the complete hierarchy database
outputDB
name of the output database that should be consistent with XTView
outputVersion
version of the database to convert to. Defaults to latest version
"""
dbIn = databaseFactory(inputDBName, permission=Permissions.READ_ONLY_FME)
if dbIn.version == outputVersion:
runLog.important(
"The input database ({}) appears to already be in the desired "
"format ({})".format(inputDBName, dbIn.version)
)
return
outputDBName = outputDBName or "-converted".join(os.path.splitext(inputDBName))
dbOut = databaseFactory(
outputDBName, permission=Permissions.CREATE_FILE_TIE, version=outputVersion
)
# each DB load resets the verbosity to that of the run. Here we allow
# conversion users to overpower it.
conversionVerbosity = runLog.getVerbosity()
runLog.extra(f"Converting {dbIn} to DB version {outputVersion}")
with dbIn, dbOut:
# Making the bold assumption that we are working with HDF5
h5In = _getH5File(dbIn)
h5Out = _getH5File(dbOut)
dbOut.writeInputsToDB(None, *dbIn.readInputsFromDB())
for cycle, timeNode in dbIn.genTimeSteps():
runLog.extra(f"Converting cycle={cycle}, timeNode={timeNode}")
r = dbIn.load(cycle, timeNode)
runLog.setVerbosity(conversionVerbosity)
dbOut.writeToDB(r)
for auxPath in dbIn.genAuxiliaryData((cycle, timeNode)):
name = next(reversed(auxPath.split("/")))
auxOutPath = dbOut.getAuxiliaryDataPath((cycle, timeNode), name)
runLog.important(
"Copying auxiliary data for time ({}, {}): {} -> {}".format(
cycle, timeNode, auxPath, auxOutPath
)
)
h5In.copy(auxPath, h5Out, name=auxOutPath)
def cleanTempDirs(olderThanDays=None):
"""
Clean up temporary files after a run.
The Windows HPC system sends a SIGBREAK signal when the user cancels a job, which
is NOT handled by ``atexit``. Notably SIGBREAK doesn't exist off Windows.
For the SIGBREAK signal to work with a Microsoft HPC, the ``TaskCancelGracePeriod``
option must be configured to be non-zero. This sets the period between SIGBREAK
and SIGTERM/SIGINT. To do cleanups in this case, we must use the ``signal`` module.
Actually, even then it does not work because MS ``mpiexec`` does not pass signals
through.
Parameters
----------
olderThanDays: int, optional
If provided, deletes other ARMI directories if they are older than the requested
time.
"""
# pylint: disable=import-outside-toplevel # avoid cyclic import
from armi import runLog
from armi.utils.pathTools import cleanPath
disconnectAllHdfDBs()
printMsg = runLog.getVerbosity() <= DEBUG
if _FAST_PATH_IS_TEMPORARY and os.path.exists(_FAST_PATH):
if printMsg:
print(
"Cleaning up temporary files in: {}".format(_FAST_PATH),
file=sys.stdout,
)
try:
cleanPath(_FAST_PATH)
except Exception as error: # pylint: disable=broad-except
for outputStream in (sys.stderr, sys.stdout):
if printMsg:
print(
"Failed to delete temporary files in: {}\n"
" error: {}".format(_FAST_PATH, error),
file=outputStream,
)
if olderThanDays is not None:
cleanAllArmiTempDirs(olderThanDays)
def convertDatabase(
inputDBName: str,
outputDBName: Optional[str] = None,
outputVersion: Optional[str] = None,
nodes: Optional[List[Tuple[int, int]]] = None,
):
"""
Convert database files between different versions.
Parameters
----------
inputDB
name of the complete hierarchy database
outputDB
name of the output database that should be consistent with XTView
outputVersion
version of the database to convert to. Defaults to latest version
nodes
optional list of specific (cycle,node)s to convert
"""
dbIn = databaseFactory(inputDBName, permission=Permissions.READ_ONLY_FME)
if dbIn.version == outputVersion:
runLog.important(
"The input database ({}) appears to already be in the desired "
"format ({})".format(inputDBName, dbIn.version)
)
return
outputDBName = outputDBName or "-converted".join(os.path.splitext(inputDBName))
dbOut = databaseFactory(
outputDBName, permission=Permissions.CREATE_FILE_TIE, version=outputVersion
)
# each DB load resets the verbosity to that of the run. Here we allow
# conversion users to overpower it.
conversionVerbosity = runLog.getVerbosity()
runLog.extra(f"Converting {dbIn} to DB version {outputVersion}")
with dbIn, dbOut:
dbNodes = list(dbIn.genTimeSteps())
if nodes is not None and any(node not in dbNodes for node in nodes):
raise RuntimeError(
"Some of the requested nodes are not in the source database.\n"
"Requested: {}\n"
"Present: {}".format(nodes, dbNodes)
)
# Making the bold assumption that we are working with HDF5
h5In = _getH5File(dbIn)
h5Out = _getH5File(dbOut)
dbOut.writeInputsToDB(None, *dbIn.readInputsFromDB())
for cycle, timeNode in dbNodes:
if nodes is not None and (cycle, timeNode) not in nodes:
continue
runLog.extra(f"Converting cycle={cycle}, timeNode={timeNode}")
timeStepsInOutDB = set(dbOut.genTimeSteps())
r = dbIn.load(cycle, timeNode)
if (r.p.cycle, r.p.timeNode) in timeStepsInOutDB:
runLog.warning(
"Time step ({}, {}) is already in the output DB. This "
"is probably due to repeated cycle/timeNode in the source DB; "
"deleting the existing time step and re-writing".format(
r.p.cycle, r.p.timeNode
)
)
del dbOut[r.p.cycle, r.p.timeNode, None]
runLog.setVerbosity(conversionVerbosity)
dbOut.writeToDB(r)
for auxPath in dbIn.genAuxiliaryData((cycle, timeNode)):
name = next(reversed(auxPath.split("/")))
auxOutPath = dbOut.getAuxiliaryDataPath((cycle, timeNode), name)
runLog.important(
"Copying auxiliary data for time ({}, {}): {} -> {}".format(
cycle, timeNode, auxPath, auxOutPath
)
)
h5In.copy(auxPath, h5Out, name=auxOutPath)
def test_setVerbosityFromString(self):
"""Test that the log verbosity can be set with a string."""
expectedStrVerbosity = runLog.getLogVerbosityLevels()[0]
verbosityRank = runLog.getLogVerbosityRank(expectedStrVerbosity)
runLog.setVerbosity(expectedStrVerbosity)
self.assertEqual(verbosityRank, runLog.getVerbosity())
