def upload(self, lfn, diracSE, guid=None):
from GangaDirac.Lib.Files.DiracFile import DiracFile
diracFile = DiracFile(namePattern=self.name, lfn=lfn)
diracFile.put(force=True)
return diracFile
def upload(self, lfn, diracSE, guid=None):
from GangaDirac.Lib.Files.DiracFile import DiracFile
diracFile = DiracFile(namePattern=self.name, lfn=lfn)
diracFile.put(force=True)
return diracFile
def uploadLocalFile(job, namePattern, localDir, should_del=True):
"""
Upload a locally available file to the grid as a DiracFile.
Randomly chooses an SE.
Args:
namePattern (str): name of the file
localDir (str): localDir of the file
should_del = (bool): should we delete the local file?
Return
DiracFile: a DiracFile of the uploaded LFN on the grid
"""
new_df = DiracFile(namePattern, localDir=localDir)
new_df.credential_requirements=job.backend.credential_requirements
trySEs = getConfig('DIRAC')['allDiracSE']
random.shuffle(trySEs)
new_lfn = os.path.join(getInputFileDir(job), namePattern)
returnable = None
for SE in trySEs:
#Check that the SE is writable
if execute('checkSEStatus("%s", "%s")' % (SE, 'Write')):
try:
returnable = new_df.put(force=True, uploadSE=SE, lfn=new_lfn)[0]
break
except GangaDiracError as err:
raise GangaException("Upload of input file as LFN %s to SE %s failed" % (new_lfn, SE))
if not returnable:
raise GangaException("Failed to upload input file to any SE")
if should_del:
os.unlink(os.path.join(localDir, namePattern))
return returnable
def uploadLocalFile(job, namePattern, localDir, should_del=True):
"""
Upload a locally available file to the grid as a DiracFile.
Randomly chooses an SE.
Args:
namePattern (str): name of the file
localDir (str): localDir of the file
should_del = (bool): should we delete the local file?
Return
DiracFile: a DiracFile of the uploaded LFN on the grid
"""
new_df = DiracFile(namePattern, localDir=localDir)
trySEs = getConfig('DIRAC')['allDiracSE']
random.shuffle(trySEs)
new_lfn = os.path.join(getInputFileDir(job), namePattern)
returnable = None
for SE in trySEs:
#Check that the SE is writable
if execute('checkSEStatus("%s", "%s")' % (SE, 'Write')):
try:
returnable = new_df.put(force=True, uploadSE=SE, lfn=new_lfn)[0]
break
except GangaDiracError as err:
raise GangaException("Upload of input file as LFN %s to SE %s failed" % (new_lfn, SE))
if not returnable:
raise GangaException("Failed to upload input file to any SE")
if should_del:
os.unlink(os.path.join(localDir, namePattern))
return returnable
def generateDiracInput(app):
"""
Construct a DIRAC input which must be unique to each job to have unique checksum.
This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput
Args:
app (GaudiRun): This expects a GaudiRun app to be passed so that the constructed
"""
input_files, input_folders = collectPreparedFiles(app)
job = app.getJobObject()
if input_folders:
raise ApplicationConfigurationError(None, 'Prepared folders not supported yet, please fix this in future')
else:
prep_dir = app.getSharedPath()
add_timeStampFile(prep_dir)
prep_file = prep_dir + '.tgz'
compressed_file = os.path.join(tempfile.gettempdir(), '__'+os.path.basename(prep_file))
wnScript = generateWNScript(prepareCommand(app), job)
script_name = os.path.join(tempfile.gettempdir(), wnScript.name)
wnScript.create(script_name)
with tarfile.open(compressed_file, "w:gz") as tar_file:
for name in input_files:
tar_file.add(name, arcname=os.path.basename(name))
tar_file.add(script_name, arcname=os.path.basename(script_name))
shutil.move(compressed_file, prep_dir)
new_df = DiracFile(namePattern=os.path.basename(compressed_file), localDir=app.getSharedPath())
random_SE = random.choice(getConfig('DIRAC')['allDiracSE'])
logger.info("new File: %s" % new_df)
new_lfn = os.path.join(DiracFile.diracLFNBase(), 'GangaInputFile/Job_%s' % job.fqid, os.path.basename(compressed_file))
new_df.put(uploadSE=random_SE, lfn=new_lfn)
app.uploadedInput = new_df
def uploadLocalFile(job, namePattern, localDir, should_del=True):
"""
Upload a locally available file to the grid as a DiracFile
Args:
namePattern (str): name of the file
localDir (str): localDir of the file
should_del = (bool): should we delete the local file?
Return
DiracFile: a DiracFile of the uploaded LFN on the grid
"""
new_df = DiracFile(namePattern, localDir=localDir)
random_SE = random.choice(getConfig('DIRAC')['allDiracSE'])
new_lfn = os.path.join(getInputFileDir(job), namePattern)
returnable = new_df.put(force=True, uploadSE=random_SE, lfn=new_lfn)[0]
if should_del:
os.unlink(os.path.join(localDir, namePattern))
return returnable
def uploadLocalFile(job, namePattern, localDir, should_del=True):
"""
Upload a locally available file to the grid as a DiracFile
Args:
namePattern (str): name of the file
localDir (str): localDir of the file
should_del = (bool): should we delete the local file?
Return
DiracFile: a DiracFile of the uploaded LFN on the grid
"""
new_df = DiracFile(namePattern, localDir=localDir)
random_SE = random.choice(getConfig('DIRAC')['allDiracSE'])
new_lfn = os.path.join(getInputFileDir(job), namePattern)
returnable = new_df.put(force=True, uploadSE=random_SE, lfn=new_lfn)[0]
if should_del:
os.unlink(os.path.join(localDir, namePattern))
return returnable