def get_disk_space(queuedata):
"""
Get the disk space from the queuedata that should be available for running the job;
either what is actually locally available or the allowed size determined by the site (value from queuedata). This
value is only to be used internally by the job dispatcher.
:param queuedata: infosys object.
:return: disk space that should be available for running the job (int).
"""
# --- non Job related queue data
# jobinfo provider is required to consider overwriteAGIS data coming from Job
_maxinputsize = infosys.queuedata.maxwdir
logger.debug(
"resolved value from global infosys.queuedata instance: infosys.queuedata.maxwdir=%s B"
% _maxinputsize)
_maxinputsize = queuedata.maxwdir
logger.debug("resolved value: queuedata.maxwdir=%s B" % _maxinputsize)
try:
du = disk_usage(os.path.abspath("."))
_diskspace = int(du[2] / (1024 * 1024)) # need to convert from B to MB
except ValueError as e:
logger.warning(
"failed to extract disk space: %s (will use schedconfig default)" %
e)
_diskspace = _maxinputsize
else:
logger.info("available WN disk space: %d MB" % (_diskspace))
_diskspace = min(_diskspace, _maxinputsize)
logger.info("sending disk space %d MB to dispatcher" % (_diskspace))
return _diskspace
def set_scratch_workdir(job, work_dir, args):
"""
Copy input files and some db files to RAM disk.
:param job: job object.
:param work_dir: job working directory (permanent FS) (string).
:param args: args dictionary to collect timing metrics.
:return: job working directory in scratch (string).
"""
scratch_path = config.HPC.scratch
du = disk_usage(scratch_path)
logger.info("Scratch dir available space: {0} used: {1}".format(
du.free, du.used))
job_scratch_dir = os.path.join(scratch_path, str(job.jobid))
for inp_file in job.input_files:
job.input_files[inp_file]["scratch_path"] = job_scratch_dir
logger.debug("Job scratch path: {0}".format(job_scratch_dir))
# special data, that should be preplaced in RAM disk
dst_db_path = 'sqlite200/'
dst_db_filename = 'ALLP200.db'
dst_db_path_2 = 'geomDB/'
dst_db_filename_2 = 'geomDB_sqlite'
tmp_path = 'tmp/'
src_file = '/ccs/proj/csc108/AtlasReleases/21.0.15/DBRelease/current/sqlite200/ALLP200.db'
src_file_2 = '/ccs/proj/csc108/AtlasReleases/21.0.15/DBRelease/current/geomDB/geomDB_sqlite'
if os.path.exists(scratch_path):
try:
add_to_pilot_timing(job.jobid, PILOT_PRE_STAGEIN, time.time(),
args)
logger.debug("Prepare \'tmp\' dir in scratch ")
if not os.path.exists(scratch_path + tmp_path):
os.makedirs(scratch_path + tmp_path)
logger.debug("Prepare dst and copy sqlite db files")
t0 = time.time()
if not os.path.exists(scratch_path + dst_db_path):
os.makedirs(scratch_path + dst_db_path)
shutil.copyfile(src_file,
scratch_path + dst_db_path + dst_db_filename)
logger.debug("")
sql_cp_time = time.time() - t0
logger.debug("Copy of sqlite files took: {0}".format(sql_cp_time))
logger.debug("Prepare dst and copy geomDB files")
t0 = time.time()
if not os.path.exists(scratch_path + dst_db_path_2):
os.makedirs(scratch_path + dst_db_path_2)
shutil.copyfile(src_file_2,
scratch_path + dst_db_path_2 + dst_db_filename_2)
geomdb_cp_time = time.time() - t0
logger.debug(
"Copy of geomDB files took: {0} s".format(geomdb_cp_time))
logger.debug("Prepare job scratch dir")
t0 = time.time()
if not os.path.exists(job_scratch_dir):
os.makedirs(job_scratch_dir)
logger.debug("Copy input file")
for inp_file in job.input_files:
logger.debug("Copy: {0} to {1}".format(
os.path.join(work_dir, inp_file),
job.input_files[inp_file]["scratch_path"]))
shutil.copyfile(
os.path.join(work_dir, inp_file),
os.path.join(job.input_files[inp_file]["scratch_path"],
inp_file))
input_cp_time = time.time() - t0
logger.debug(
"Copy of input files took: {0} s".format(input_cp_time))
except IOError as e:
logger.error("I/O error({0}): {1}".format(e.errno, e.strerror))
logger.error(
"Copy to scratch failed, execution terminated': \n %s " %
(sys.exc_info()[1]))
raise FileHandlingFailure("Copy to RAM disk failed")
finally:
add_to_pilot_timing(job.jobid, PILOT_POST_STAGEIN, time.time(),
args)
else:
logger.info('Scratch directory (%s) dos not exist' % scratch_path)
return work_dir
os.chdir(job_scratch_dir)
logger.debug("Current directory: {0}".format(os.getcwd()))
true_dir = '/ccs/proj/csc108/AtlasReleases/21.0.15/nfs_db_files'
pseudo_dir = "./poolcond"
os.symlink(true_dir, pseudo_dir)
du = disk_usage(scratch_path)
logger.info("Scratch dir available space for job: {0} used: {1}".format(
du.free, du.used))
return job_scratch_dir