def check_availablespace(self, files):
"""
Verify that enough local space is available to stage in and run the job
:param files: list of FileSpec objects.
:raise: PilotException in case of not enough space or total input size too large
"""
for f in files:
self.logger.debug('lfn=%s filesize=%d accessmode=%s' %
(f.lfn, f.filesize, f.accessmode))
maxinputsize = convert_mb_to_b(get_maximum_input_sizes())
totalsize = reduce(lambda x, y: x + y.filesize, files, 0)
# verify total filesize
if maxinputsize and totalsize > maxinputsize:
error = "too many/too large input files (%s). total file size=%s B > maxinputsize=%s B" % \
(len(files), totalsize, maxinputsize)
raise SizeTooLarge(error)
self.logger.info(
"total input file size=%s B within allowed limit=%s B (zero value means unlimited)"
% (totalsize, maxinputsize))
# get available space
available_space = convert_mb_to_b(get_local_disk_space(os.getcwd()))
self.logger.info("locally available space: %d B" % available_space)
# are we within the limit?
if totalsize > available_space:
error = "not enough local space for staging input files and run the job (need %d B, but only have %d B)" % \
(totalsize, available_space)
raise NoLocalSpace(error)
def get_max_allowed_work_dir_size(queuedata):
"""
Return the maximum allowed size of the work directory.
:param queuedata: job.infosys.queuedata object.
:return: max allowed work dir size in Bytes (int).
"""
try:
maxwdirsize = convert_mb_to_b(get_maximum_input_sizes(
)) # from MB to B, e.g. 16336 MB -> 17,129,537,536 B
except Exception as e:
max_input_size = get_max_input_size()
maxwdirsize = max_input_size + config.Pilot.local_size_limit_stdout * 1024
logger.info(
"work directory size check will use %d B as a max limit (maxinputsize [%d B] + local size limit for"
" stdout [%d B])" % (maxwdirsize, max_input_size,
config.Pilot.local_size_limit_stdout * 1024))
logger.warning('conversion caught exception: %s' % e)
else:
# grace margin, as discussed in https://its.cern.ch/jira/browse/ATLASPANDA-482
margin = 10.0 # percent, read later from somewhere
maxwdirsize = int(maxwdirsize * (1 + margin / 100.0))
logger.info(
"work directory size check will use %d B as a max limit (10%% grace limit added)"
% maxwdirsize)
return maxwdirsize