def getInputFileDir(job):
"""
Return the LFN remote dirname for this job
"""
return os.path.join(
DiracFile.diracLFNBase(job.backend.credential_requirements),
'GangaJob_%s/InputFiles' % job.fqid)
def getDiracFiles():
import os
from GangaDirac.Lib.Files.DiracFile import DiracFile
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
filename = DiracFile.diracLFNBase().replace('/', '-') + '.lfns'
logger.info('Creating list, this can take a while if you have a large number of SE files, please wait...')
execute('dirac-dms-user-lfns &> /dev/null', shell=True, timeout=None)
g = GangaList()
with open(filename[1:], 'r') as lfnlist:
lfnlist.seek(0)
g.extend((DiracFile(lfn='%s' % lfn.strip()) for lfn in lfnlist.readlines()))
return addProxy(g)
def getDiracFiles():
import os
from GangaDirac.Lib.Files.DiracFile import DiracFile
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
filename = DiracFile.diracLFNBase().replace('/', '-') + '.lfns'
logger.info(
'Creating list, this can take a while if you have a large number of SE files, please wait...'
)
execute('dirac-dms-user-lfns &> /dev/null', shell=True, timeout=None)
g = GangaList()
with open(filename[1:], 'r') as lfnlist:
lfnlist.seek(0)
g.extend(
(DiracFile(lfn='%s' % lfn.strip()) for lfn in lfnlist.readlines()))
return addProxy(g)
def generateDiracInput(app):
"""
Construct a DIRAC input which must be unique to each job to have unique checksum.
This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput
Args:
app (GaudiRun): This expects a GaudiRun app to be passed so that the constructed
"""
input_files, input_folders = collectPreparedFiles(app)
job = app.getJobObject()
if input_folders:
raise ApplicationConfigurationError(None, 'Prepared folders not supported yet, please fix this in future')
else:
prep_dir = app.getSharedPath()
add_timeStampFile(prep_dir)
prep_file = prep_dir + '.tgz'
compressed_file = os.path.join(tempfile.gettempdir(), '__'+os.path.basename(prep_file))
wnScript = generateWNScript(prepareCommand(app), job)
script_name = os.path.join(tempfile.gettempdir(), wnScript.name)
wnScript.create(script_name)
with tarfile.open(compressed_file, "w:gz") as tar_file:
for name in input_files:
tar_file.add(name, arcname=os.path.basename(name))
tar_file.add(script_name, arcname=os.path.basename(script_name))
shutil.move(compressed_file, prep_dir)
new_df = DiracFile(namePattern=os.path.basename(compressed_file), localDir=app.getSharedPath())
random_SE = random.choice(getConfig('DIRAC')['allDiracSE'])
logger.info("new File: %s" % new_df)
new_lfn = os.path.join(DiracFile.diracLFNBase(), 'GangaInputFile/Job_%s' % job.fqid, os.path.basename(compressed_file))
new_df.put(uploadSE=random_SE, lfn=new_lfn)
app.uploadedInput = new_df
def getInputFileDir(job):
"""
Return the LFN remote dirname for this job
"""
return os.path.join(DiracFile.diracLFNBase(), 'GangaInputFile/Job_%s' % job.fqid)
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
cred_req = app.getJobObject().backend.credential_requirements
check_creds(cred_req)
inputsandbox, outputsandbox = master_sandbox_prepare(
app, appmasterconfig)
# If we are getting the metadata we need to make sure the summary.xml is added to the output sandbox if not there already.
if app.getMetadata and not 'summary.xml' in outputsandbox:
outputsandbox += ['summary.xml']
# Check a previously uploaded input is there in case of a job copy
if isinstance(app.uploadedInput, DiracFile):
if app.uploadedInput.getReplicas() == {}:
app.uploadedInput = None
logger.info(
"Previously uploaded cmake target missing from Dirac. Uploading it again."
)
if not isinstance(app.uploadedInput, DiracFile):
generateDiracInput(app)
try:
assert isinstance(app.uploadedInput, DiracFile)
except AssertionError:
raise ApplicationPrepareError(
"Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n"
% DiracFile.diracLFNBase(cred_req))
rep_data = app.uploadedInput.getReplicas()
try:
assert rep_data != {}
except AssertionError:
raise ApplicationPrepareError(
"Failed to find a replica of uploaded file, aborting submit")
if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)):
app.jobScriptArchive = None
generateDiracScripts(app)
try:
assert isinstance(app.jobScriptArchive, DiracFile)
except AssertionError:
raise ApplicationPrepareError(
"Failed to upload needed file, aborting submit")
rep_data = app.jobScriptArchive.getReplicas()
try:
assert rep_data != {}
except AssertionError:
raise ApplicationPrepareError(
"Failed to find a replica, aborting submit")
#Create a replica of the job and scripts files
replicateJobFile(app.jobScriptArchive)
replicateJobFile(app.uploadedInput)
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
cred_req = app.getJobObject().backend.credential_requirements
check_creds(cred_req)
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
# If we are getting the metadata we need to make sure the summary.xml is added to the output sandbox if not there already.
if app.getMetadata and not 'summary.xml' in outputsandbox:
outputsandbox += ['summary.xml']
# Check a previously uploaded input is there in case of a job copy
if isinstance(app.uploadedInput, DiracFile):
if app.uploadedInput.getReplicas() == {}:
app.uploadedInput = None
logger.info("Previously uploaded cmake target missing from Dirac. Uploading it again.")
if not isinstance(app.uploadedInput, DiracFile):
generateDiracInput(app)
try:
assert isinstance(app.uploadedInput, DiracFile)
except AssertionError:
raise ApplicationPrepareError("Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase(cred_req))
rep_data = app.uploadedInput.getReplicas()
try:
assert rep_data != {}
except AssertionError:
raise ApplicationPrepareError("Failed to find a replica of uploaded file, aborting submit")
if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)):
app.jobScriptArchive = None
generateDiracScripts(app)
try:
assert isinstance(app.jobScriptArchive, DiracFile)
except AssertionError:
raise ApplicationPrepareError("Failed to upload needed file, aborting submit")
rep_data = app.jobScriptArchive.getReplicas()
try:
assert rep_data != {}
except AssertionError:
raise ApplicationPrepareError("Failed to find a replica, aborting submit")
#Create a replica of the job and scripts files
replicateJobFile(app.jobScriptArchive)
replicateJobFile(app.uploadedInput)
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def getInputFileDir(job):
"""
Return the LFN remote dirname for this job
"""
return os.path.join(DiracFile.diracLFNBase(job.backend.credential_requirements), 'GangaJob_%s/InputFiles' % job.fqid)
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
# If we are getting the metadata we need to make sure the summary.xml is added to the output sandbox if not there already.
if app.getMetadata and not 'summary.xml' in outputsandbox:
outputsandbox += ['summary.xml']
if not isinstance(app.uploadedInput, DiracFile):
generateDiracInput(app)
assert isinstance(app.uploadedInput, DiracFile), "Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase()
rep_data = app.uploadedInput.getReplicas()
assert rep_data != {}, "Failed to find a replica, aborting submit"
if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)):
app.jobScriptArchive = None
generateDiracScripts(app)
assert isinstance(app.jobScriptArchive, DiracFile), "Failed to upload needed file, aborting submit"
rep_data = app.jobScriptArchive.getReplicas()
assert rep_data != {}, "Failed to find a replica, aborting submit"
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def getInputFileDir(job):
"""
Return the LFN remote dirname for this job
"""
return os.path.join(DiracFile.diracLFNBase(), 'GangaInputFile/Job_%s' % job.fqid)
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
if not isinstance(app.uploadedInput, DiracFile):
generateDiracInput(app)
assert isinstance(app.uploadedInput, DiracFile), "Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase()
rep_data = app.uploadedInput.getReplicas()
assert rep_data != {}, "Failed to find a replica, aborting submit"
if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)):
app.jobScriptArchive = None
generateDiracScripts(app)
assert isinstance(app.jobScriptArchive, DiracFile), "Failed to upload needed file, aborting submit"
rep_data = app.jobScriptArchive.getReplicas()
assert rep_data != {}, "Failed to find a replica, aborting submit"
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
