def _stopParasol(self):
self.worker.popen.kill()
self.worker.join()
self.leader.popen.kill()
self.leader.join()
for path in ('para.results', 'parasol.jid'):
rm_f(path)
def _stopParasol(self):
self.worker.popen.kill()
self.worker.join()
self.leader.popen.kill()
self.leader.join()
for path in ('para.results', 'parasol.jid'):
rm_f(path)
def _recursiveDelete(self, path, maxTries=3):
"""
Delete a directory recursively, retrying after a delay on error, to
deal with filesystems that present a bad FS abstraction.
Detect and warn about .nfs files.
If the directory cannot ultiumately be removed after maxTries
attempts, raise an error.
We want to clean up after ourselves. But some filesystem backends (e.g.
NFS, GPFS) either create new files in the hierarchy when deleted files
are still open ("silly rename"), or else sometimes can't keep up with
filesystem operations and lack perfect read-your-own-writes
consistency. This code works around that. It can be deleted if and when
we check the mount type of the filesystem we are using to make sure it
is not flaky/fake.
"""
for tryNumber in range(maxTries):
# Try a few times to delete everything
try:
for baseName in os.listdir(path):
# For each file in this directory
fullName = os.path.join(path, baseName)
if os.path.isdir(fullName):
# Recurse into directories
self._recursiveDelete(fullName, maxTries)
else:
try:
# Try to delete files
rm_f(fullName)
except OSError:
if baseName.startswith(".nfs"):
# If we can't delete NFS files, skip them for now
if not self._warnedOnNfs:
# But warn the user they exist
self._warnedOnNfs = True
logger.warning(
"Open NFS files like {} may prevent job or job store deletion"
.format(fullName))
# Note that if the file deosn't go away soon we will fail deleting the parent directory.
continue
else:
raise
# Now we have deleted all the files in the directory (or skipped out on NFS files).
# Try to remove the directory.
os.rmdir(path)
# If it worked, we are done
return
except:
# Something went wrong
if tryNumber + 1 >= maxTries:
# This was our last attempt
raise
else:
# Wait and try again
timeToSleep = 2**tryNumber
logger.warning(
"Cannot delete {}; retrying in {} seconds".format(
path, timeToSleep))
time.sleep(timeToSleep)