def execute(self, runid):
logger.info("Starting publication")
from Ganga.Utility.files import expandfilename
config = getConfig('TestRobot')
topdir=expandfilename(config['JobDir'])
publishPath=expandfilename(config['PublishPath'])
version = config['VersionNumber']
site = config['Site']
for i in range(len(config['TestPairs'])):
jobdir = join(topdir, version+"_Job_"+str(i))
import tarfile
test=config['TestPairs'][i][0].replace('/','.')
option=config['TestPairs'][i][2]
fname = version+'_'+test+'_'+option+'-'+site+'.tgz'
archive = tarfile.open(join(publishPath,fname),'w:gz')
archive.posix=False
for file in [f for f in os.listdir(join(jobdir,'output'))
if os.path.isfile(join(jobdir,'output',f))]:
toname= file.replace(option,option+'-'+site)
archive.add(join(jobdir,'output',file),join('output',toname))
for file in [f for f in os.listdir(join(jobdir,'reports','latest'))
if os.path.isfile(join(jobdir,'reports','latest',f))]:
toname = file.replace(option,option+'-'+site)
archive.add(join(jobdir,'reports','latest',file),join(toname))
archive.close()
def _setLFNnamePattern(self, lfn="", namePattern=""):
if self.defaultSE != "":
## TODO REPLACE THIS WITH IN LIST OF VONAMES KNOWN
# Check for /lhcb/some/path or /gridpp/some/path
if namePattern.split(os.pathsep)[0] == self.defaultSE \
or (len(namePattern) > 3 and namePattern[0:4].upper() == "LFN:"\
or len(namePattern.split(os.pathsep)) > 1 and namePattern.split(os.pathsep)[1] == self.defaultSE):
# Check for LFN:/gridpp/some/path or others...
lfn = namePattern
namePattern = ""
if lfn:
if len(lfn) > 3 and lfn[0:4].upper() == "LFN:":
lfn = lfn[4:]
if lfn != "" and namePattern != "":
self.lfn = lfn
self.remoteDir = os.path.dirname(lfn)
self.namePattern = os.path.basename(namePattern)
self.localDir = os.path.dirname(expandfilename(namePattern))
elif lfn != "" and namePattern == "":
self.lfn = lfn
self.remoteDir = os.path.dirname(self.lfn)
self.namePattern = os.path.basename(self.lfn)
self.localDir = ""
elif namePattern != "" and lfn == "":
self.namePattern = os.path.basename(namePattern)
self.localDir = os.path.dirname(expandfilename(namePattern))
self.remoteDir = ""
self.lfn = ""
def __construct__(self, args):
self.locations = []
if len(args) == 1 and isType(args[0], DiracFile):
self.lfn = args[0].lfn
self.namePattern = args[0].namePattern
self.remoteDir = args[0].remoteDir
self.localDir = args[0].localDir
self.guid = args[0].guid
self.compressed = args[0].compressed
self._storedReplicas = args[0]._storedReplicas
self._remoteURLs = args[0]._remoteURLs
self.failureReason = args[0].failureReason
self._have_copied = True
self.subfiles = copy.deepcopy(args[0].subfiles)
return
# LFN ONLY
if len(args) == 1 and type(args[0]) == type(""):
if str(str(args[0]).upper()[0:4]) == str("LFN:"):
self._setLFNnamePattern(lfn=args[0][4:], namePattern="")
else:
self._setLFNnamePattern(lfn="", namePattern=args[0])
# NAMEPATTERN AND LFN
elif len(args) == 2 and type(args[0]) == type("") and type(args[1]) == type(""):
self.namePattern = args[0]
self._setLFNnamePattern(lfn="", namePattern=self.namePattern)
self.localDir = expandfilename(args[1])
# NAMEPATTERN AND LFN AND LOCALDIR
elif len(args) == 3 and type(args[0]) == type("") and type(args[1]) == type("") and type(args[2]) == type(""):
self.namePattern = args[0]
self.lfn = args[2]
self._setLFNnamePattern(lfn=self.lfn, namePattern=self.namePattern)
self.localDir = expandfilename(args[1])
# NAMEPATTERN AND LFN AND LOCALDIR AND REMOTEDIR
elif (
len(args) == 4
and type(args[0]) == type("")
and type(args[1]) == type("")
and type(args[2]) == type("")
and type(args[3]) == type("")
):
self.namePattern = args[0]
self.lfn = args[2]
self._setLFNnamePattern(lfn=lfn, namePattern=namePattern)
self.localDir = expandfilename(args[1])
self.remoteDir = args[3]
# OTHER
else:
super(DiracFile, self).__construct__(args)
return
def get_share_path(app=None):
if app is None or not isType(app.is_prepared, ShareDir):
return os.path.join(
expandfilename(getConfig('Configuration')['gangadir']), 'shared',
getConfig('Configuration')['user'])
#elif isType(app, ShareDir):
else:
return os.path.join(
expandfilename(getConfig('Configuration')['gangadir']), 'shared',
getConfig('Configuration')['user'], app.is_prepared.name)
def __construct__(self, args):
self.locations = []
if len(args) == 1 and isType(args[0], DiracFile):
self.lfn = args[0].lfn
self.namePattern = args[0].namePattern
self.remoteDir = args[0].remoteDir
self.localDir = args[0].localDir
self.guid = args[0].guid
self.compressed = args[0].compressed
self._storedReplicas = args[0]._storedReplicas
self._remoteURLs = args[0]._remoteURLs
self.failureReason = args[0].failureReason
self._have_copied = True
self.subfiles = copy.deepcopy(args[0].subfiles)
return
# LFN ONLY
if len(args) == 1 and type(args[0]) == type(''):
if str(str(args[0]).upper()[0:4]) == str("LFN:"):
self._setLFNnamePattern(_lfn=args[0][4:], _namePattern="")
else:
self._setLFNnamePattern(_lfn="", _namePattern=args[0])
# NAMEPATTERN AND LFN
elif len(args) == 2 and type(args[0]) == type('') and type(
args[1]) == type(''):
self.namePattern = args[0]
self._setLFNnamePattern(_lfn='', _namePattern=self.namePattern)
self.localDir = expandfilename(args[1])
# NAMEPATTERN AND LFN AND LOCALDIR
elif len(args) == 3 and type(args[0]) == type('') and type(
args[1]) == type('') and type(args[2]) == type(''):
self.namePattern = args[0]
self.lfn = args[2]
self._setLFNnamePattern(_lfn=self.lfn,
_namePattern=self.namePattern)
self.localDir = expandfilename(args[1])
# NAMEPATTERN AND LFN AND LOCALDIR AND REMOTEDIR
elif len(args) == 4 and type(args[0]) == type('') and type(args[1]) == type('')\
and type(args[2]) == type('') and type(args[3]) == type(''):
self.namePattern = args[0]
self.lfn = args[2]
self._setLFNnamePattern(_lfn=lfn, _namePattern=namePattern)
self.localDir = expandfilename(args[1])
self.remoteDir = args[3]
# OTHER
else:
super(DiracFile, self).__construct__(args)
return
def get_share_path(app=None):
if app is None or app == "":
return os.path.join(
expandfilename(getConfig("Configuration")["gangadir"]), "shared", getConfig("Configuration")["user"]
)
return os.path.join(
expandfilename(getConfig("Configuration")["gangadir"]),
"shared",
getConfig("Configuration")["user"],
app.is_prepared.name,
)
def getPath(self,filename=None):
subpath = self.subpath
if filename is None: filename = ''
if not self.jobid is None:
jobdir = str(self.jobid)
else:
jobdir = ''
# do not use subpath if no tree splitting applies
if not self.splittree: subpath = ''
if self.splittree:
return expandfilename(os.path.join(self.top,subpath,jobdir,filename))
else:
return expandfilename(os.path.join(self.top,jobdir,subpath,filename))
def _attribute_filter__set__(self, name, value):
if value != "" and value is not None:
# Do some checking of the filenames in a subprocess
if name == 'lfn':
self.namePattern = os.path.basename(value)
if os.path.dirname(value):
self.remoteDir = os.path.dirname(value)
return value
elif name == 'remoteDir':
if self.lfn != os.path.join(value, self.namePattern):
self.lfn = os.path.join(value, self.namePattern)
elif name == 'namePattern':
self.localDir = os.path.dirname(value)
return os.path.basename(value)
elif name == 'localDir':
if value:
return expandfilename(value)
else:
return value
return value
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
from Ganga.Lib.gLite import gLiteJobConfig
prepared_exe = app.exe
if app.is_prepared is not None:
shared_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']),
'shared', getConfig('Configuration')['user'])
if isinstance(app.exe, str):
# we have a file. is it an absolute path?
if os.path.abspath(app.exe) == app.exe:
logger.info("Submitting a prepared application; taking any input files from %s" % (
app.is_prepared.name))
prepared_exe = File(os.path.join(os.path.join(
shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name)))
# else assume it's a system binary, so we don't need to
# transport anything to the sharedir
else:
prepared_exe = app.exe
elif isinstance(app.exe, File):
logger.info("Submitting a prepared application; taking any input files from %s" % (
app.is_prepared.name))
prepared_exe = File(os.path.join(os.path.join(
shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name)))
return gLiteJobConfig(prepared_exe, app._getParent().inputsandbox, convertIntToStringArgs(app.args), app._getParent().outputsandbox, app.env)
def __app_set__(obj, val):
if not hasattr(obj, 'application') or obj.application is None:
return
if hasattr(obj.application, '_is_prepared'):
#a=Job(); a.prepare(); a.application=Executable()
if obj.application.is_prepared not in [None, True] and\
hasattr(val, 'is_prepared') and val.is_prepared is None:
logger.debug('Overwriting a prepared application with one that is unprepared')
obj.application.unprepare()
#a=Job(); b=Executable(); b.prepare(); a.application=b
elif obj.application.is_prepared is not True:
if hasattr(val, 'is_prepared'):
if val.is_prepared not in [None, True]:
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
logger.debug('Overwriting application with a prepared one')
if stripProxy(obj.application) != val:
stripProxy(obj.application).unprepare()
shareref.increase(val.is_prepared.name)
# check that the shared directory actually exists before
# assigning the (prepared) application to a job
if hasattr(val, 'is_prepared'):
if val.is_prepared not in [None, True]:
if hasattr(val.is_prepared, 'name'):
from Ganga.Utility.files import expandfilename
Config_conf = getConfig('Configuration')
shared_path = os.path.join(expandfilename(Config_conf['gangadir']), 'shared', Config_conf['user'])
if not os.path.isdir(os.path.join(shared_path, val.is_prepared.name)):
logger.error('ShareDir directory not found: %s' % val.is_prepared.name)
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
from Ganga.Lib.gLite import gLiteJobConfig
prepared_exe = app.exe
if app.is_prepared is not None:
shared_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']),
'shared', getConfig('Configuration')['user'])
if isinstance(app.exe, str):
# we have a file. is it an absolute path?
if os.path.abspath(app.exe) == app.exe:
logger.info("Submitting a prepared application; taking any input files from %s" % (
app.is_prepared.name))
prepared_exe = File(os.path.join(os.path.join(
shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name)))
# else assume it's a system binary, so we don't need to
# transport anything to the sharedir
else:
prepared_exe = app.exe
elif isinstance(app.exe, File):
logger.info("Submitting a prepared application; taking any input files from %s" % (
app.is_prepared.name))
prepared_exe = File(os.path.join(os.path.join(
shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name)))
return gLiteJobConfig(prepared_exe, app._getParent().inputsandbox, convertIntToStringArgs(app.args), app._getParent().outputsandbox, app.env)
def _join_opts_files(self):
'''Create a single options file from all supplied options.'''
joined_py_opts = ''
for name in self.optsfiles:
with open(expandfilename(name), 'r') as this_file:
joined_py_opts += this_file.read()
return joined_py_opts
def getSharedPath():
# Required to be initialized for ShareDir object
from Ganga.Utility.files import expandfilename
import os.path
Conf_config = getConfig('Configuration')
root_default = os.path.join(expandfilename(Conf_config['gangadir']), 'shared', Conf_config['user'])
return root_default
def _attribute_filter__set__(self, name, value):
if value != "" and value is not None:
# Do some checking of the filenames in a subprocess
if name == 'lfn':
self.namePattern = os.path.basename(value)
if os.path.dirname(value):
self.remoteDir = os.path.dirname(value)
return value
elif name == 'remoteDir':
if self.lfn != os.path.join(value, self.namePattern):
self.lfn = os.path.join(value, self.namePattern)
elif name == 'namePattern':
self.localDir = os.path.dirname(value)
return os.path.basename(value)
elif name == 'localDir':
if value:
return expandfilename(value)
else:
return value
return value
def writefile(self, fileobj, executable=None):
from Ganga.GPIDev.Lib.File import FileBuffer
if not isType(fileobj, FileBuffer):
raise GangaTypeError('Usage of tuples is not allowed, use FileBuffer instead')
# output file name
# Added a subdir to files, (see Ganga/GPIDev/Lib/File/File.py) This allows
# to copy files into the a subdirectory of the workspace
# FIXME: make a helper method for os.makedirs
path_to_build = os.path.join(self.getPath(), fileobj.subdir)
if not os.path.isdir(path_to_build):
os.makedirs(path_to_build)
logger.debug('created %s', self.getPath())
else:
logger.debug('already exists: %s', self.getPath())
outname = expandfilename(self.getPath(fileobj.getPathInSandbox()), True)
fileobj.create(outname)
if executable:
chmod_executable(outname)
return outname
def _join_opts_files(self):
'''Create a single options file from all supplied options.'''
joined_py_opts = ''
for name in self.optsfiles:
with open(expandfilename(name), 'r') as this_file:
joined_py_opts += this_file.read()
return joined_py_opts
def writefile(self, fileobj, executable=None):
from Ganga.GPIDev.Lib.File import FileBuffer
if not isType(fileobj, FileBuffer):
raise GangaTypeError(
'Usage of tuples is not allowed, use FileBuffer instead')
# output file name
# Added a subdir to files, (see Ganga/GPIDev/Lib/File/File.py) This allows
# to copy files into the a subdirectory of the workspace
# FIXME: make a helper method for os.makedirs
path_to_build = os.path.join(self.getPath(), fileobj.subdir)
if not os.path.isdir(path_to_build):
os.makedirs(path_to_build)
logger.debug('created %s', self.getPath())
else:
logger.debug('already exists: %s', self.getPath())
outname = expandfilename(self.getPath(fileobj.getPathInSandbox()),
True)
fileobj.create(outname)
if executable:
chmod_executable(outname)
return outname
def prepare(self, force=False):
super(Ostap, self).prepare(force)
self._check_inputs()
share_dir = os.path.join (
expandfilename ( getConfig('Configuration')['gangadir'] ) ,
'shared' ,
getConfig('Configuration')['user'] ,
self.is_prepared.name )
input_sandbox_tar = os.path.join ( share_dir , 'inputsandbox',
'_input_sandbox_%s.tar' % self.is_prepared.name )
input_sandbox_tgz = os.path.join ( share_dir , 'inputsandbox',
'_input_sandbox_%s.tgz' % self.is_prepared.name )
fillPackedSandbox ( self.scripts , input_sandbox_tar )
gzipFile ( input_sandbox_tar , input_sandbox_tgz , True )
# add the newly created shared directory into the metadata system if
# the app is associated with a persisted object
self.checkPreparedHasParent(self)
self.post_prepare()
logger.debug("Finished Preparing Application in %s" % share_dir)
def __setattr__(self, attr, value):
"""
This is an overloaded setter method to make sure that we're auto-expanding the filenames of files which exist.
In the case we're assigning any other attributes the value is simply passed through
Args:
attr (str): This is the name of the attribute which we're assigning
value (unknown): This is the value being assigned.
"""
actual_value = value
if attr == "namePattern":
actual_value = os.path.basename(value)
this_dir = os.path.dirname(value)
if this_dir:
super(DiracFile, self).__setattr__('localDir', this_dir)
elif attr == 'localDir':
if value:
new_value = os.path.abspath(expandfilename(value))
if os.path.exists(new_value):
actual_value = new_value
elif attr == 'lfn':
if value:
this_name = os.path.basename(value)
super(DiracFile, self).__setattr__('namePattern', this_name)
super(DiracFile, self).__setattr__('remoteDir', os.path.dirname(value))
elif attr == 'remoteDir':
if value:
this_lfn = os.path.join(value, self.namePattern)
super(DiracFile, self).__setattr__('lfn', this_lfn)
super(DiracFile, self).__setattr__(attr, actual_value)
def prepare(self, force=False):
super(BenderScript, self).prepare(force)
self._check_inputs()
share_dir = os.path.join (
expandfilename ( getConfig('Configuration')['gangadir'] ) ,
'shared' ,
getConfig('Configuration')['user'] ,
self.is_prepared.name )
input_sandbox_tar = os.path.join ( share_dir , 'inputsandbox',
'_input_sandbox_%s.tar' % self.is_prepared.name )
input_sandbox_tgz = os.path.join ( share_dir , 'inputsandbox',
'_input_sandbox_%s.tgz' % self.is_prepared.name )
fillPackedSandbox ( self.scripts + self.imports , input_sandbox_tar )
gzipFile ( input_sandbox_tar , input_sandbox_tgz , True )
# add the newly created shared directory into the metadata system if
# the app is associated with a persisted object
self.checkPreparedHasParent(self)
self.post_prepare()
logger.debug("Finished Preparing Application in %s" % share_dir)
def split(self, job):
"""
Method to do the splitting work
Args:
job (Job): master job to be used as a template to split subjobs
"""
from GangaLHCb.Lib.Applications import Gauss
if not isinstance(job.application, (Gauss, GaudiExec)):
logger.warning(
"This application is of type: '%s', be careful how you use it with the GaussSplitter!"
% type(job.application))
subjobs = []
inputdata = job.inputdata
if not isinstance(job.application, GaudiExec):
# I'm assuming this file is created by the Gauss Application at some stage?
if not job.inputdata:
share_path = os.path.join(
expandfilename(getConfig('Configuration')['gangadir']),
'shared',
getConfig('Configuration')['user'],
job.application.is_prepared.name, 'inputdata',
'options_data.pkl')
if os.path.exists(share_path):
f = open(share_path, 'r+b')
#FIXME should this have been an addition?
inputdata = pickle.load(f)
f.close()
for i in range(self.numberOfJobs):
j = self._create_subjob(job, inputdata)
# FIXME this starts from the 1st event and not zero, is it clear why?
first = self.firstEventNumber + i * self.eventsPerJob + 1
opts = 'from Configurables import GenInit \n'
if isinstance(job.application, Gauss):
opts += 'from Gaudi.Configuration import * \n'
opts += 'ApplicationMgr().EvtMax = %d\n' % self.eventsPerJob
else:
opts += 'from Configurables import LHCbApp \n'
opts += 'LHCbApp().EvtMax = %d\n' % self.eventsPerJob
opts += 'GenInit("GaussGen").FirstEventNumber = %d\n' % first
spillOver = ["GaussGenPrev", "GaussGenPrevPrev", "GaussGenNext"]
for s in spillOver:
opts += 'GenInit("%s").FirstEventNumber = %d\n' % (s, first)
#j.application.extra.input_buffers['data.py'] += opts
if isinstance(job.application, GaudiExec):
j.application.extraOpts = j.application.extraOpts + '\n' + opts
else:
j._splitter_data = opts
# j.inputsandbox.append(File(FileBuffer(path,opts).create().name))
logger.debug("Creating job %d w/ FirstEventNumber = %d" %
(i, first))
subjobs.append(j)
return subjobs
def create(self, outname):
"""create a file in a local filesystem as 'outname', maintain
the original permissions """
import shutil
shutil.copy(expandfilename(self.name), outname)
if self.executable:
chmod_executable(outname)
def download(self, dir='.'):
'Downloads the LFN to dir (default is current directory).'
dir = expandfilename(dir)
dir = os.path.abspath(dir)
cmd = 'result = DiracCommands.getFile("%s","%s")' % (self.name, dir)
result = get_result(cmd, 'Problem during download', 'Download error.')
from PhysicalFile import PhysicalFile
return GPIProxyObjectFactory(PhysicalFile(name=result['Value']))
def getSharedPath():
# Required to be initialized for ShareDir object
from Ganga.Utility.files import expandfilename
import os.path
Conf_config = getConfig('Configuration')
root_default = os.path.join(expandfilename(Conf_config['gangadir']),
'shared', Conf_config['user'])
return root_default
def create(self, outname):
"""create a file in a local filesystem as 'outname', maintain
the original permissions """
import shutil
shutil.copy(expandfilename(self.name), outname)
if self.executable:
chmod_executable(outname)
def getDir(self):
"""
This function returns a sanitized absolute path of the self.directory method from user input
"""
if self.directory:
return path.abspath(expandfilename(self.directory))
else:
raise ApplicationConfigurationError(None, "No Opts File has been specified, please provide one!")
def getSharedPath(self):
"""
Return the full path of the shared directory where files are placed
"""
share_folder = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'])
if isinstance(self.is_prepared, ShareDir):
return os.path.join(share_folder, self.is_prepared.name)
else:
return share_folder
def retrieve(self, type=None, name=None, **options):
"""Retieve files listed in outputdata and registered in output from
remote SE to local filesystem in background thread"""
from Ganga.GPIDev.Lib.Job import Job
import os
job = self._getParent()
try:
logger.info("ATLASOutputDataset.retrieve() called.")
logger.debug('job.id: %d, Job.subjobs: %d', job.id,
len(job.subjobs))
logger.debug('job.outputdir: %s', job.outputdir)
logger.debug('job.outputsandbox: %s', job.outputsandbox)
logger.debug('job.outputdata.outputsandbox: %s', job.outputdata)
logger.debug('job.outputdata.outputdata: %s',
job.outputdata.outputdata)
logger.debug('job.outputdata.output: %s', job.outputdata.output)
logger.debug('job.outputdata.location: %s',
job.outputdata.location)
logger.debug('job.outputdata.local_location: %s',
job.outputdata.local_location)
except AttributeError:
logger.error('job.outputdata error')
return 1
local_location = options.get('local_location')
# Determine local output path to store files
outputlocation = ''
if job.outputdata.location and (job.backend._name == 'Local'):
outputlocation = expandfilename(job.outputdata.location)
elif local_location:
outputlocation = expandfilename(local_location)
elif job.outputdata.local_location:
outputlocation = expandfilename(job.outputdata.local_location)
else:
# User job repository location
#outputlocation = job.outputdir+'/../'
pass
# Output files
outputfiles = job.outputdata.outputdata
def repairJobRepository(jobId):
""" Repairs job repository for the comment attribute (migration from Comment object to string) """
def repairFilePath(filePath):
fileRead = open(filePath, 'r')
index = -1
found = False
lines = fileRead.readlines()
fileRead.close()
for line in lines:
index += 1
if line.find('<class name="Comment"') > -1:
found = True
break
if found:
newLines = []
for i in range(index):
newLines.append(lines[i])
newLines.append(lines[index+1].replace('<attribute name="comment"> ', ''))
for i in range(index+4, len(lines)):
newLines.append(lines[i])
fileWrite = open(filePath, 'w')
fileWrite.write(''.join(newLines))
fileWrite.close()
from Ganga.Utility.Config import getConfig
from Ganga.Utility.files import expandfilename
import os
if not isinstance(jobId,int):
return
repositoryPath = "repository/$usr/LocalXML/6.0/jobs/$thousandsNumxxx"
repositoryPath = repositoryPath.replace('$usr', getConfig('Configuration')['user'])
repositoryPath = repositoryPath.replace('$thousandsNum', str(jobId/1000))
repositoryFullPath = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), repositoryPath, str(jobId))
#repair also the subjobs data files
for subjobDir in os.listdir(repositoryFullPath):
repositorySubJobFullPath = os.path.join(repositoryFullPath, subjobDir)
if os.path.isdir(repositorySubJobFullPath):
repairFilePath(os.path.join(repositorySubJobFullPath, 'data'))
repairFilePath(os.path.join(repositoryFullPath, 'data'))
def split(self, job):
"""
Method to do the splitting work
Args:
job (Job): master job to be used as a template to split subjobs
"""
from GangaLHCb.Lib.Applications import Gauss
if not isinstance(job.application, (Gauss, GaudiExec)):
logger.warning("This application is of type: '%s', be careful how you use it with the GaussSplitter!" % type(job.application))
subjobs = []
inputdata = job.inputdata
if not isinstance(job.application, GaudiExec):
# I'm assuming this file is created by the Gauss Application at some stage?
if not job.inputdata:
share_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']),
'shared',
getConfig('Configuration')['user'],
job.application.is_prepared.name,
'inputdata',
'options_data.pkl')
if os.path.exists(share_path):
f = open(share_path, 'r+b')
#FIXME should this have been an addition?
inputdata = pickle.load(f)
f.close()
for i in range(self.numberOfJobs):
j = self._create_subjob(job, inputdata)
# FIXME this starts from the 1st event and not zero, is it clear why?
first = self.firstEventNumber + i * self.eventsPerJob + 1
opts = 'from Configurables import GenInit \n'
if isinstance(job.application, Gauss):
opts += 'from Gaudi.Configuration import * \n'
opts += 'ApplicationMgr().EvtMax = %d\n' % self.eventsPerJob
else:
opts += 'from Configurables import LHCbApp \n'
opts += 'LHCbApp().EvtMax = %d\n' % self.eventsPerJob
opts += 'GenInit("GaussGen").FirstEventNumber = %d\n' % first
spillOver = ["GaussGenPrev", "GaussGenPrevPrev", "GaussGenNext"]
for s in spillOver:
opts += 'GenInit("%s").FirstEventNumber = %d\n' % (s, first)
#j.application.extra.input_buffers['data.py'] += opts
if isinstance(job.application, GaudiExec):
j.application.extraOpts = j.application.extraOpts + '\n' + opts
else:
j._splitter_data = opts
# j.inputsandbox.append(File(FileBuffer(path,opts).create().name))
logger.debug("Creating job %d w/ FirstEventNumber = %d" % (i, first))
subjobs.append(j)
return subjobs
def full_expand_filename(name):
if len(name) >= 4 and name[0:4].upper() == 'LFN:':
msg = 'Can not create PhysicalFile from string that begins w/ "LFN:".'\
' You probably want to create a LogicalFile.'
raise GangaException(msg)
urlprefix=re.compile('^(([a-zA-Z_][\w]*:)+/?)?/')
if len(name) >= 4 and name[0:4].upper() == 'PFN:': name = name[4:]
expanded_name = expandfilename(name)
if urlprefix.match(expanded_name): return expanded_name
return os.path.abspath(expanded_name)
def getLocalWorkspace():
# Get the local top level dirtectory for the Workspace
if config['repositorytype'] in [
'LocalXML', 'LocalAMGA', 'LocalPickle', 'SQLite'
]:
return os.path.join(expandfilename(config['gangadir'],
True), 'workspace', config['user'],
config['repositorytype'])
else:
return ''
def full_expand_filename(name):
if len(name) >= 4 and name[0:4].upper() == 'LFN:':
msg = 'Can not create PhysicalFile from string that begins w/ "LFN:".'\
' You probably want to create a LogicalFile.'
raise GangaException(msg)
urlprefix = re.compile('^(([a-zA-Z_][\w]*:)+/?)?/')
if len(name) >= 4 and name[0:4].upper() == 'PFN:': name = name[4:]
expanded_name = expandfilename(name)
if urlprefix.match(expanded_name): return expanded_name
return os.path.abspath(expanded_name)
def getSharedPath(self):
"""
Return the full path of the shared directory where files are placed
"""
share_folder = os.path.join(
expandfilename(getConfig('Configuration')['gangadir']), 'shared',
getConfig('Configuration')['user'])
if isinstance(self.is_prepared, ShareDir):
return os.path.join(share_folder, self.is_prepared.name)
else:
return share_folder
def getPath(self, filename=None):
subpath = self.subpath
if filename is None:
filename = ''
if self.jobid is not None:
jobdir = str(self.jobid)
else:
jobdir = ''
subpath = ''
return expandfilename(os.path.join(self.top, jobdir, subpath, filename), True)
def __construct__(self,args):
if len(args) == 1 and type(args[0]) == type(''):
v = args[0]
import os.path
expanded = expandfilename(v)
if not urlprefix.match(expanded): # if it is not already an absolute filename
self.name = os.path.abspath(expanded)
else: #bugfix #20545
self.name = expanded
else:
super(File,self).__construct__(args)
def getPath(self, filename=None):
subpath = self.subpath
if filename is None:
filename = ''
if self.jobid is not None:
jobdir = str(self.jobid)
else:
jobdir = ''
subpath = ''
return expandfilename(os.path.join(self.top, jobdir, subpath, filename), True)
def __setattr__(self, attr, value):
"""
This is an overloaded setter method to make sure that we're auto-expanding the filenames of files which exist.
In the case we're assigning any other attributes the value is simply passed through
Args:
attr (str): This is the name of the attribute which we're assigning
value (unknown): This is the value being assigned.
"""
actual_value = value
if attr == "name":
actual_value = expandfilename(value)
super(File, self).__setattr__(attr, actual_value)
def __setattr__(self, attr, value):
"""
This is an overloaded setter method to make sure that we're auto-expanding the filenames of files which exist.
In the case we're assigning any other attributes the value is simply passed through
Args:
attr (str): This is the name of the attribute which we're assigning
value (unknown): This is the value being assigned.
"""
actual_value = value
if attr == "name":
actual_value = expandfilename(value)
super(File, self).__setattr__(attr, actual_value)
def __construct__(self, args):
if len(args) == 1 and isType(args[0], DiracFile):
self.lfn = args[0].lfn
self.namePattern = args[0].namePattern
self.remoteDir = args[0].remoteDir
self.localDir = args[0].localDir
self.guid = args[0].guid
self.compressed = args[0].compressed
self._storedReplicas = args[0]._storedReplicas
self._remoteURLs = args[0]._remoteURLs
self.failureReason = args[0].failureReason
self._have_copied = True
self.subfiles = copy.deepcopy(args[0].subfiles)
return
if len(args) == 1 and type(args[0]) == type(''):
if str(str(args[0]).upper()[0:4]) == str("LFN:"):
self._setLFNnamePattern(_lfn=args[0][4:], _namePattern="")
else:
self._setLFNnamePattern(_lfn="", _namePattern=args[0])
elif len(args) == 2 and type(args[0]) == type('') and type(args[1]) == type(''):
self.namePattern = args[0]
self._setLFNnamePattern(_lfn='', _namePattern=self.namePattern)
self.localDir = expandfilename(args[1])
elif len(args) == 3 and type(args[0]) == type('') and type(args[1]) == type('') and type(args[2]) == type(''):
self.namePattern = args[0]
self.lfn = args[2]
self._setLFNnamePattern(_lfn=self.lfn, _namePattern=self.namePattern)
self.localDir = expandfilename(args[1])
elif len(args) == 4 and type(args[0]) == type('') and type(args[1]) == type('')\
and type(args[2]) == type('') and type(args[3]) == type(''):
self.namePattern = args[0]
self.lfn = args[2]
self._setLFNnamePattern(_lfn=lfn, _namePattern=namePattern)
self.localDir = expandfilename(args[1])
self.remoteDir = args[3]
else:
super(DiracFile, self).__construct__(args)
return
def retrieve(self, type=None, name=None, **options ):
"""Retieve files listed in outputdata and registered in output from
remote SE to local filesystem in background thread"""
from Ganga.GPIDev.Lib.Job import Job
import os
job = self._getParent()
try:
logger.info("ATLASOutputDataset.retrieve() called.")
logger.debug('job.id: %d, Job.subjobs: %d',job.id,len(job.subjobs))
logger.debug('job.outputdir: %s',job.outputdir)
logger.debug('job.outputsandbox: %s',job.outputsandbox)
logger.debug('job.outputdata.outputsandbox: %s',job.outputdata)
logger.debug('job.outputdata.outputdata: %s',job.outputdata.outputdata)
logger.debug('job.outputdata.output: %s',job.outputdata.output)
logger.debug('job.outputdata.location: %s',job.outputdata.location)
logger.debug('job.outputdata.local_location: %s',job.outputdata.local_location)
except AttributeError:
logger.error('job.outputdata error')
return 1
local_location = options.get('local_location')
# Determine local output path to store files
outputlocation = ''
if job.outputdata.location and (job.backend._name == 'Local'):
outputlocation = expandfilename(job.outputdata.location)
elif local_location:
outputlocation = expandfilename(local_location)
elif job.outputdata.local_location:
outputlocation = expandfilename(job.outputdata.local_location)
else:
# User job repository location
#outputlocation = job.outputdir+'/../'
pass
# Output files
outputfiles = job.outputdata.outputdata
def add(self, input):
from Ganga.Core.GangaRepository import getRegistry
if not isType(input, list):
input = [input]
for item in input:
if isType(item, str):
if os.path.isfile(expandfilename(item)):
logger.info('Copying file %s to shared directory %s' %
(item, self.name))
shutil.copy2(expandfilename(item),
os.path.join(getSharedPath(), self.name))
shareref = getRegistry("prep").getShareRef()
shareref.increase(self.name)
shareref.decrease(self.name)
else:
logger.error('File %s not found' % expandfilename(item))
elif isType(item, File) and item.name is not '' and os.path.isfile(
expandfilename(item.name)):
logger.info('Copying file object %s to shared directory %s' %
(item.name, self.name))
shutil.copy2(expandfilename(item.name),
os.path.join(getSharedPath(), self.name))
shareref = getRegistry("prep").getShareRef()
shareref.increase(self.name)
shareref.decrease(self.name)
else:
logger.error('File %s not found' % expandfilename(item.name))
def readfile(path):
"""Return the content of the specified file.
Keyword arguments:
path -- The path to the file (variables and tilde are expanded).
"""
path = files.expandfilename(path)
f = open(path)
try:
return f.read()
finally:
f.close()
def readfile(path):
"""Return the content of the specified file.
Keyword arguments:
path -- The path to the file (variables and tilde are expanded).
"""
path = files.expandfilename(path)
f = open(path)
try:
return f.read()
finally:
f.close()
def copyIntoPrepDir(self, obj2copy):
"""
Method for actually copying the "obj2copy" object to the prepared state dir of this application
Args:
obj2copy (bool): is a string (local) address of a file to be copied as it's passed to shutil.copy2
"""
shared_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'])
shr_dir = os.path.join(shared_path, self.is_prepared.name)
if not os.path.isdir(shr_dir):
os.makedirs(shr_dir)
shutil.copy2(obj2copy, shr_dir)
logger.debug("Copying %s into: %s" % (obj2copy, shr_dir))
def prepare(self, force=False):
super(GaudiPython, self).prepare(force)
self._check_inputs()
share_dir = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'], self.is_prepared.name)
fillPackedSandbox(self.script, os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name))
gzipFile(os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name),
os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tgz' % self.is_prepared.name), True)
# add the newly created shared directory into the metadata system if
# the app is associated with a persisted object
self.checkPreparedHasParent(self)
self.post_prepare()
def test_GaudiPython_prepare(self):
g = self.job.application
g.prepare()
assert g.is_prepared is not None, 'is_prepared not correctly set'
g.is_prepared.ls()
share_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']),
'shared',
getConfig('Configuration')['user'],
g.is_prepared.name,
'inputsandbox')
os.system(
'cd ' + share_path + ';tar -xzvf _input_sandbox_' + g.is_prepared.name + '.tgz')
assert os.path.exists(os.path.join(share_path, 'dummy.script'))
def __init__(self, namePattern="", localDir=None, lfn="", remoteDir=None, **kwds):
"""
name is the name of the output file that has to be written ...
"""
super(DiracFile, self).__init__()
self.locations = []
self._setLFNnamePattern(lfn, namePattern)
if localDir is not None:
self.localDir = expandfilename(localDir)
if remoteDir is not None:
self.remoteDir = remoteDir
def _setLFNnamePattern(self, _lfn="", _namePattern=""):
if _lfn != "" and _lfn is not None:
if len(_lfn) > 3 and _lfn[0:4] == "LFN:":
_lfn = _lfn[4:]
if _lfn != "" and _namePattern != "":
self.lfn = _lfn
self.remoteDir = os.path.dirname(_lfn)
self.namePattern = os.path.basename(_namePattern)
self.localDir = os.path.dirname(expandfilename(_namePattern))
elif _lfn != "" and _namePattern == "":
self.lfn = _lfn
self.remoteDir = os.path.dirname(self.lfn)
if self.namePattern != "":
self.namePattern = os.path.basename(self.lfn)
self.localDir = ""
elif _namePattern != "" and _lfn == "":
self.namePattern = os.path.basename(_namePattern)
self.localDir = os.path.dirname(expandfilename(_namePattern))
self.remoteDir = ""
self.lfn = ""
def _setLFNnamePattern(self, _lfn="", _namePattern=""):
if _lfn != "" and _lfn is not None:
if len(_lfn) > 3 and _lfn[0:4] == "LFN:":
_lfn = _lfn[4:]
if _lfn != "" and _namePattern != "":
self.lfn = _lfn
self.remoteDir = os.path.dirname(_lfn)
self.namePattern = os.path.basename(_namePattern)
self.localDir = os.path.dirname(expandfilename(_namePattern))
elif _lfn != "" and _namePattern == "":
self.lfn = _lfn
self.remoteDir = os.path.dirname(self.lfn)
if self.namePattern != "":
self.namePattern = os.path.basename(self.lfn)
self.localDir = ""
elif _namePattern != "" and _lfn == "":
self.namePattern = os.path.basename(_namePattern)
self.localDir = os.path.dirname(expandfilename(_namePattern))
self.remoteDir = ""
self.lfn = ""
def __construct__(self, args):
if len(args) == 1 and isinstance(args[0], str):
v = args[0]
import os.path
expanded = expandfilename(v)
# if it is not already an absolute filename
if not urlprefix.match(expanded):
if os.path.exists(os.path.abspath(expanded)):
self.name = os.path.abspath(expanded)
else:
self.name = v
else: # bugfix #20545
self.name = expanded
else:
super(File, self).__construct__(args)
def writefile(path, content):
"""Write the content to the specified file, creating directories if necessary.
Keyword arguments:
path -- The path to the file (variables and tilde are expanded).
context -- The file contexts as a string.
"""
path = files.expandfilename(path)
directory = os.path.dirname(path)
if not os.path.exists(directory):
os.makedirs(directory)
f = open(path, 'w')
try:
f.write(content)
finally:
f.close()
def __setattr__(self, attr, value):
"""
This is an overloaded setter method to make sure that we're auto-expanding the filenames of files which exist.
In the case we're assigning any other attributes the value is simply passed through
Args:
attr (str): This is the name of the attribute which we're assigning
value (unknown): This is the value being assigned.
"""
actual_value = value
if attr == "namePattern":
actual_value = os.path.basename(value)
this_localDir = os.path.dirname(value)
super(MassStorageFile, self).__setattr__('localDir', this_localDir)
if attr == "localDir":
actual_value = os.path.abspath(expandfilename(value))
super(MassStorageFile, self).__setattr__(attr, actual_value)
def prepare_cmake_app(myApp, myVer, myPath='$HOME/cmtuser', myGetpack=None):
"""
Short helper function for setting up minimal application environments on disk for job submission
Args:
myApp (str): This is the name of the app to pass to lb-dev
myVer (str): This is the version of 'myApp' to pass tp lb-dev
myPath (str): This is where lb-dev will be run
myGepPack (str): This is a getpack which will be run once the lb-dev has executed
"""
full_path = expandfilename(myPath, True)
if not path.exists(full_path):
makedirs(full_path)
chdir(full_path)
_exec_cmd('lb-dev %s %s' % (myApp, myVer), full_path)
dev_dir = path.join(full_path, myApp + 'Dev_' + myVer)
logger.info("Set up App Env at: %s" % dev_dir)
if myGetpack:
_exec_cmd('getpack %s' % myGetpack, dev_dir)
logger.info("Ran Getpack: %s" % myGetpack)
return dev_dir
def __setattr__(self, attr, value):
"""
This is an overloaded setter method to make sure that we're auto-expanding the filenames of files which exist.
In the case we're assigning any other attributes the value is simply passed through
Args:
attr (str): This is the name of the attribute which we're assigning
value (unknown): This is the value being assigned.
"""
actual_value = value
if attr == 'namePattern':
if len(value.split(os.sep)) > 1:
this_dir = path.dirname(value)
super(LocalFile, self).__setattr__('localDir', this_dir)
actual_value = path.basename(value)
elif attr == 'localDir':
if value:
new_value = path.abspath(expandfilename(value))
if path.exists(new_value):
actual_value = new_value
super(LocalFile, self).__setattr__(attr, actual_value)
def _attribute_filter__set__(self, name, value):
if value != "" and value is not None:
# Do some checking of the filenames in a subprocess
if name == 'lfn':
this_dir, self.namePattern = os.path.split(value)
if this_dir:
self.remoteDir = this_dir
return value
elif name == 'namePattern':
self.localDir, this_name = os.path.split(value)
return this_name
elif name == 'localDir':
if value:
return expandfilename(value)
else:
return value
return value
def test_Gaudi_prepare(self):
# Test standalone preparation
d = DaVinci()
d.prepare()
assert d.is_prepared is not None, 'is_prepared not correctly set'
#assert d._impl.prep_inputbox, 'inputbox empty'
# Now test as part of job
j = Job(application=DaVinci())
j.prepare()
assert j.application.is_prepared is not None, 'is_prepared not correctly set'
#assert j.application._impl.prep_inputbox, 'inputbox empty'
job = Job(application=Gauss(optsfile='./Gauss-Job.py'))
job.application.platform = 'x86_64-slc6-gcc48-opt'
gauss = job.application
job.inputdata = ['pfn:dummy1.in', 'pfn:dummy2.in']
job.outputfiles = [
'Gauss.sim', 'GaussHistos.root', 'GaussMonitor.root']
#job.outputsandbox = ['GaussHistos.root','GaussMonitor.root']
#inputs,extra = gauss._impl.master_configure()
# provide basic test of where output goes - a more complete test is
# run on the PythonOptionsParser methods.
job.prepare()
# ok = job.application._impl.prep_outputbox.count('GaussHistos.root') > 0 and \
## job.application._impl.prep_outputbox.count('GaussMonitor.root') > 0
share_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']),
'shared',
getConfig('Configuration')['user'],
job.application.is_prepared.name)
# ok = os.path.exists(os.path.join(job.application._impl.is_prepared.name,'GaussHistos.root')) and \
# os.path.exists(os.path.join(job.application._impl.is_prepared.name,'GaussMonitor.root'))
assert os.path.exists(
os.path.join(share_path, 'output', 'options_parser.pkl')), 'outputsandbox error'
os.system('cd ' + share_path + '/inputsandbox/' +
';tar -xzvf _input_sandbox_' + job.application.is_prepared.name + '.tgz')
assert os.path.exists(os.path.join(
share_path, 'inputsandbox', 'options.pkl')), 'pickled options file error'
assert os.path.exists(
os.path.join(share_path, 'debug', 'gaudi-env.py.gz')), 'zipped env file error'
