def uploadLocalFile(job, namePattern, localDir, should_del=True):
"""
Upload a locally available file to the grid as a DiracFile.
Randomly chooses an SE.
Args:
namePattern (str): name of the file
localDir (str): localDir of the file
should_del = (bool): should we delete the local file?
Return
DiracFile: a DiracFile of the uploaded LFN on the grid
"""
new_df = DiracFile(namePattern, localDir=localDir)
new_df.credential_requirements=job.backend.credential_requirements
trySEs = getConfig('DIRAC')['allDiracSE']
random.shuffle(trySEs)
new_lfn = os.path.join(getInputFileDir(job), namePattern)
returnable = None
for SE in trySEs:
#Check that the SE is writable
if execute('checkSEStatus("%s", "%s")' % (SE, 'Write')):
try:
returnable = new_df.put(force=True, uploadSE=SE, lfn=new_lfn)[0]
break
except GangaDiracError as err:
raise GangaException("Upload of input file as LFN %s to SE %s failed" % (new_lfn, SE))
if not returnable:
raise GangaException("Failed to upload input file to any SE")
if should_del:
os.unlink(os.path.join(localDir, namePattern))
return returnable
def upload(self, lfn, diracSE, guid=None):
from GangaDirac.Lib.Files.DiracFile import DiracFile
diracFile = DiracFile(namePattern=self.name, lfn=lfn)
diracFile.put(force=True)
return diracFile
def df():
load_config_files()
from GangaDirac.Lib.Files.DiracFile import DiracFile
f = DiracFile('np', 'ld', 'lfn')
f.locations = ['location']
f.guid = 'guid'
yield f
clear_config()
def uploadLocalFile(job, namePattern, localDir, should_del=True):
"""
Upload a locally available file to the grid as a DiracFile
Args:
namePattern (str): name of the file
localDir (str): localDir of the file
should_del = (bool): should we delete the local file?
Return
DiracFile: a DiracFile of the uploaded LFN on the grid
"""
new_df = DiracFile(namePattern, localDir=localDir)
random_SE = random.choice(getConfig('DIRAC')['allDiracSE'])
new_lfn = os.path.join(getInputFileDir(job), namePattern)
returnable = new_df.put(force=True, uploadSE=random_SE, lfn=new_lfn)[0]
if should_del:
os.unlink(os.path.join(localDir, namePattern))
return returnable
def getDiracFiles():
import os
from GangaDirac.Lib.Files.DiracFile import DiracFile
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
filename = DiracFile.diracLFNBase().replace('/', '-') + '.lfns'
logger.info('Creating list, this can take a while if you have a large number of SE files, please wait...')
execute('dirac-dms-user-lfns &> /dev/null', shell=True, timeout=None)
g = GangaList()
with open(filename[1:], 'r') as lfnlist:
lfnlist.seek(0)
g.extend((DiracFile(lfn='%s' % lfn.strip()) for lfn in lfnlist.readlines()))
return addProxy(g)
def generateDiracInput(app):
"""
Construct a DIRAC input which must be unique to each job to have unique checksum.
This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput
Args:
app (GaudiRun): This expects a GaudiRun app to be passed so that the constructed
"""
input_files, input_folders = collectPreparedFiles(app)
job = app.getJobObject()
if input_folders:
raise ApplicationConfigurationError(None, 'Prepared folders not supported yet, please fix this in future')
else:
prep_dir = app.getSharedPath()
add_timeStampFile(prep_dir)
prep_file = prep_dir + '.tgz'
compressed_file = os.path.join(tempfile.gettempdir(), '__'+os.path.basename(prep_file))
wnScript = generateWNScript(prepareCommand(app), job)
script_name = os.path.join(tempfile.gettempdir(), wnScript.name)
wnScript.create(script_name)
with tarfile.open(compressed_file, "w:gz") as tar_file:
for name in input_files:
tar_file.add(name, arcname=os.path.basename(name))
tar_file.add(script_name, arcname=os.path.basename(script_name))
shutil.move(compressed_file, prep_dir)
new_df = DiracFile(namePattern=os.path.basename(compressed_file), localDir=app.getSharedPath())
random_SE = random.choice(getConfig('DIRAC')['allDiracSE'])
logger.info("new File: %s" % new_df)
new_lfn = os.path.join(DiracFile.diracLFNBase(), 'GangaInputFile/Job_%s' % job.fqid, os.path.basename(compressed_file))
new_df.put(uploadSE=random_SE, lfn=new_lfn)
app.uploadedInput = new_df
def string_file_shortcut(v, item):
if isinstance(v, str):
# use proxy class to enable all user conversions on the value itself
# but return the implementation object (not proxy)
key = findOutputFileTypeByFileName(v)
if key is not None:
if key == 'MassStorageFile':
from .MassStorageFile import MassStorageFile
return stripProxy(MassStorageFile._proxyClass(v))
elif key == 'LCGSEFile':
from .LCGSEFile import LCGSEFile
return stripProxy(LCGSEFile._proxyClass(v))
elif key == 'DiracFile':
try:
from GangaDirac.Lib.Files.DiracFile import DiracFile
return stripProxy(DiracFile._proxyClass(v))
except:
Ganga.Utility.logging.log_unknown_exception()
pass
return stripProxy(LocalFile._proxyClass(v))
return None
def getInputFileDir(job):
"""
Return the LFN remote dirname for this job
"""
return os.path.join(DiracFile.diracLFNBase(), 'GangaInputFile/Job_%s' % job.fqid)
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
cred_req = app.getJobObject().backend.credential_requirements
check_creds(cred_req)
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
# If we are getting the metadata we need to make sure the summary.xml is added to the output sandbox if not there already.
if app.getMetadata and not 'summary.xml' in outputsandbox:
outputsandbox += ['summary.xml']
# Check a previously uploaded input is there in case of a job copy
if isinstance(app.uploadedInput, DiracFile):
if app.uploadedInput.getReplicas() == {}:
app.uploadedInput = None
logger.info("Previously uploaded cmake target missing from Dirac. Uploading it again.")
if not isinstance(app.uploadedInput, DiracFile):
generateDiracInput(app)
try:
assert isinstance(app.uploadedInput, DiracFile)
except AssertionError:
raise ApplicationPrepareError("Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase(cred_req))
rep_data = app.uploadedInput.getReplicas()
try:
assert rep_data != {}
except AssertionError:
raise ApplicationPrepareError("Failed to find a replica of uploaded file, aborting submit")
if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)):
app.jobScriptArchive = None
generateDiracScripts(app)
try:
assert isinstance(app.jobScriptArchive, DiracFile)
except AssertionError:
raise ApplicationPrepareError("Failed to upload needed file, aborting submit")
rep_data = app.jobScriptArchive.getReplicas()
try:
assert rep_data != {}
except AssertionError:
raise ApplicationPrepareError("Failed to find a replica, aborting submit")
#Create a replica of the job and scripts files
replicateJobFile(app.jobScriptArchive)
replicateJobFile(app.uploadedInput)
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def getInputFileDir(job):
"""
Return the LFN remote dirname for this job
"""
return os.path.join(DiracFile.diracLFNBase(job.backend.credential_requirements), 'GangaJob_%s/InputFiles' % job.fqid)
def df():
f = DiracFile('np', 'ld', 'lfn')
f.locations = ['location']
f.guid = 'guid'
return f
def master_prepare(self, app, appmasterconfig):
"""
Prepare the RTHandler for the master job so that applications to be submitted
Args:
app (GaudiExec): This application is only expected to handle GaudiExec Applications here
appmasterconfig (unknown): Output passed from the application master configuration call
"""
inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)
if not isinstance(app.uploadedInput, DiracFile):
generateDiracInput(app)
assert isinstance(app.uploadedInput, DiracFile), "Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase()
rep_data = app.uploadedInput.getReplicas()
assert rep_data != {}, "Failed to find a replica, aborting submit"
if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)):
app.jobScriptArchive = None
generateDiracScripts(app)
assert isinstance(app.jobScriptArchive, DiracFile), "Failed to upload needed file, aborting submit"
rep_data = app.jobScriptArchive.getReplicas()
assert rep_data != {}, "Failed to find a replica, aborting submit"
return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def getDataset(self):
'''Gets the dataset from the bookkeeping for current path, etc.'''
if not self.path:
return None
if not self.type in ['Path', 'RunsByDate', 'Run', 'Production']:
raise GangaException('Type="%s" is not valid.' % self.type)
if not self.type is 'RunsByDate':
if self.startDate:
msg = 'startDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.endDate:
msg = 'endDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.selection:
msg = 'selection not supported for type="%s".' % self.type
raise GangaException(msg)
cmd = "getDataset('%s','%s','%s','%s','%s','%s')" % (
self.path, self.dqflag, self.type, self.startDate, self.endDate,
self.selection)
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
knownLists = [tuple, list, GangaList]
if isType(self.dqflag, knownLists):
cmd = "getDataset('%s',%s,'%s','%s','%s','%s')" % (
self.path, self.dqflag, self.type, self.startDate,
self.endDate, self.selection)
result = get_result(cmd, 'BK query error.', 'BK query error.')
logger.debug("Finished Running Command")
files = []
value = result['Value']
if 'LFNs' in value:
files = value['LFNs']
if not type(files) is list: # i.e. a dict of LFN:Metadata
# if 'LFNs' in files: # i.e. a dict of LFN:Metadata
files = files.keys()
logger.debug("Creating DiracFile objects")
## Doesn't work not clear why
from GangaDirac.Lib.Files.DiracFile import DiracFile
#new_files = []
#def _createDiracLFN(this_file):
# return DiracFile(lfn = this_file)
#GangaObject.__createNewList(new_files, files, _createDiracLFN)
new_files = [DiracFile(lfn=_file) for _file in files]
#new_files = [DiracFile(lfn=_file) for _file in files]
#for f in files:
# new_files.append(DiracFile(lfn=f))
#ds.extend([DiracFile(lfn = f)])
logger.debug("Creating Dataset")
from GangaLHCb.Lib.LHCbDataset import LHCbDataset
ds = LHCbDataset(new_files)
logger.debug("Returning Dataset")
return addProxy(ds)