def open(self, job: Job) -> Generator[None, None, None]:
jobReqs = job.disk
startingDir = os.getcwd()
self.localTempDir = make_public_dir(in_directory=self.localTempDir)
self._removeDeadJobs(self.workDir)
self.jobStateFile = self._createJobStateFile()
freeSpace, diskSize = getFileSystemSize(self.localTempDir)
if freeSpace <= 0.1 * diskSize:
logger.warning(f'Starting job {self.jobName} with less than 10%% of disk space remaining.')
try:
os.chdir(self.localTempDir)
with super().open(job):
yield
finally:
disk = getDirSizeRecursively(self.localTempDir)
percent = float(disk) / jobReqs * 100 if jobReqs > 0 else 0.0
disk_usage = (f"Job {self.jobName} used {percent:.2f}% disk ({bytes2human(disk)}B [{disk}B] used, "
f"{bytes2human(jobReqs)}B [{jobReqs}B] requested).")
if disk > jobReqs:
self.logToMaster("Job used more disk than requested. For CWL, consider increasing the outdirMin "
f"requirement, otherwise, consider increasing the disk requirement. {disk_usage}",
level=logging.WARNING)
else:
self.logToMaster(disk_usage, level=logging.DEBUG)
os.chdir(startingDir)
# Finally delete the job from the worker
os.remove(self.jobStateFile)
def open(self, job):
jobReqs = job.disk
startingDir = os.getcwd()
self.localTempDir = makePublicDir(os.path.join(self.localTempDir, str(uuid.uuid4())))
self._removeDeadJobs(self.workDir)
self.jobStateFile = self._createJobStateFile()
freeSpace, diskSize = getFileSystemSize(self.localTempDir)
if freeSpace <= 0.1 * diskSize:
logger.warning('Starting job %s with less than 10%% of disk space remaining.',
self.jobName)
try:
os.chdir(self.localTempDir)
with super().open(job):
yield
finally:
diskUsed = getDirSizeRecursively(self.localTempDir)
logString = ("Job {jobName} used {percent:.2f}% ({humanDisk}B [{disk}B] used, "
"{humanRequestedDisk}B [{requestedDisk}B] requested) at the end of "
"its run.".format(jobName=self.jobName,
percent=(float(diskUsed) / jobReqs * 100 if
jobReqs > 0 else 0.0),
humanDisk=bytes2human(diskUsed),
disk=diskUsed,
humanRequestedDisk=bytes2human(jobReqs),
requestedDisk=jobReqs))
self.logToMaster(logString, level=logging.DEBUG)
if diskUsed > jobReqs:
self.logToMaster("Job used more disk than requested. Consider modifying the user "
"script to avoid the chance of failure due to incorrectly "
"requested resources. " + logString, level=logging.WARNING)
os.chdir(startingDir)
# Finally delete the job from the worker
os.remove(self.jobStateFile)
def testGetSizeOfDirectoryWorks(self):
from toil.common import getDirSizeRecursively
# os.stat lists the number of 512-byte blocks used, but really, the file system is using
# a blocksize specific to itself. For instance, my machine uses 4096-byte blocks.
#
# >>> with open('/tmp/temp', 'w') as fileHandle:
# ... fileHandle.write(os.urandom(512*3))
# ...
# >>> os.stat('/tmp/temp').st_blksize
# 4096
# >>> os.stat('/tmp/temp')
# posix.stat_result(st_mode=33188, st_ino=2630547, st_dev=16777220, st_nlink=1, st_uid=501,
# st_gid=0, st_size=1536, st_atime=1475620048, st_mtime=1475628030,
# st_ctime=1475628030)
# >>> os.stat('/tmp/temp').st_blocks
# 8
#
# Even though the file is just 3 * 512 bytes, the file system uses one 4096-byte block and
# os.stat says it is using eight 512-byte blocks.
systemBlockSize = os.stat('.').st_blksize
# a list of the directories used in the test
directories = [self.testDir]
# A dict of {FILENAME: FILESIZE} for all files used in the test
files = {}
# Create a random directory structure
for i in range(0, 10):
directories.append(
tempfile.mkdtemp(dir=random.choice(directories),
prefix='test'))
# Create 50 random file entries in different locations in the directories. 75% of the time
# these are fresh files of sixe [1, 10] MB and 25% of the time they are hard links to old
# files.
while len(files) <= 50:
fileName = os.path.join(random.choice(directories),
self._getRandomName())
if random.randint(0, 100) < 75:
# Create a fresh file in the range of 1-10 MB
fileSize = int(round(random.random(), 2) * 10 * 1024 * 1024)
with open(fileName, 'w') as fileHandle:
fileHandle.write(os.urandom(fileSize))
files[fileName] = int(
math.ceil(fileSize * 1.0 / systemBlockSize) *
systemBlockSize)
else:
# Link to one of the previous files
if len(files) == 0:
continue
linkSrc = random.choice(list(files.keys()))
os.link(linkSrc, fileName)
files[fileName] = 'Link to %s' % linkSrc
computedDirectorySize = getDirSizeRecursively(self.testDir)
totalExpectedSize = sum(
[x for x in list(files.values()) if isinstance(x, int)])
self.assertEqual(computedDirectorySize, totalExpectedSize)
def testGetSizeOfDirectoryWorks(self):
from toil.common import getDirSizeRecursively
# os.stat lists the number of 512-byte blocks used, but really, the file system is using
# a blocksize specific to itself. For instance, my machine uses 4096-byte blocks.
#
# >>> with open('/tmp/temp', 'w') as fileHandle:
# ... fileHandle.write(os.urandom(512*3))
# ...
# >>> os.stat('/tmp/temp').st_blksize
# 4096
# >>> os.stat('/tmp/temp')
# posix.stat_result(st_mode=33188, st_ino=2630547, st_dev=16777220, st_nlink=1, st_uid=501,
# st_gid=0, st_size=1536, st_atime=1475620048, st_mtime=1475628030,
# st_ctime=1475628030)
# >>> os.stat('/tmp/temp').st_blocks
# 8
#
# Even though the file is just 3 * 512 bytes, the file system uses one 4096-byte block and
# os.stat says it is using eight 512-byte blocks.
systemBlockSize = os.stat('.').st_blksize
# a list of the directories used in the test
directories = [self.testDir]
# A dict of {FILENAME: FILESIZE} for all files used in the test
files = {}
# Create a random directory structure
for i in xrange(0,10):
directories.append(tempfile.mkdtemp(dir=random.choice(directories), prefix='test'))
# Create 50 random file entries in different locations in the directories. 75% of the time
# these are fresh files of sixe [1, 10] MB and 25% of the time they are hard links to old
# files.
while len(files) <= 50:
fileName = os.path.join(random.choice(directories), self._getRandomName())
if random.randint(0,100) < 75:
# Create a fresh file in the range of 1-10 MB
fileSize = int(round(random.random(), 2) * 10 * 1024 * 1024)
with open(fileName, 'w') as fileHandle:
fileHandle.write(os.urandom(fileSize))
files[fileName] = int(math.ceil(fileSize * 1.0 / systemBlockSize) * systemBlockSize)
else:
# Link to one of the previous files
if len(files) == 0:
continue
linkSrc = random.choice(files.keys())
os.link(linkSrc, fileName)
files[fileName] = 'Link to %s' % linkSrc
computedDirectorySize = getDirSizeRecursively(self.testDir)
totalExpectedSize = sum([x for x in files.values() if isinstance(x, int)])
self.assertEqual(computedDirectorySize, totalExpectedSize)
def testGetSizeOfDirectoryWorks(self):
'''A test to make sure toil.common.getDirSizeRecursively does not
underestimate the amount of disk space needed.
Disk space allocation varies from system to system. The computed value
should always be equal to or slightly greater than the creation value.
This test generates a number of random directories and randomly sized
files to test this using getDirSizeRecursively.
'''
from toil.common import getDirSizeRecursively
# a list of the directories used in the test
directories = [self.testDir]
# A dict of {FILENAME: FILESIZE} for all files used in the test
files = {}
# Create a random directory structure
for i in range(0, 10):
directories.append(
tempfile.mkdtemp(dir=random.choice(directories),
prefix='test'))
# Create 50 random file entries in different locations in the directories. 75% of the time
# these are fresh files of size [1, 10] MB and 25% of the time they are hard links to old
# files.
while len(files) <= 50:
fileName = os.path.join(random.choice(directories),
self._getRandomName())
if random.randint(0, 100) < 75:
# Create a fresh file in the range of 1-10 MB
fileSize = int(round(random.random(), 2) * 10 * 1024 * 1024)
with open(fileName, 'wb') as fileHandle:
fileHandle.write(os.urandom(fileSize))
files[fileName] = fileSize
else:
# Link to one of the previous files
if len(files) == 0:
continue
linkSrc = random.choice(list(files.keys()))
os.link(linkSrc, fileName)
files[fileName] = 'Link to %s' % linkSrc
computedDirectorySize = getDirSizeRecursively(self.testDir)
totalExpectedSize = sum(
[x for x in list(files.values()) if isinstance(x, int)])
self.assertGreaterEqual(computedDirectorySize, totalExpectedSize)
