def execute(self, runid): logger.info("Starting publication") from Ganga.Utility.files import expandfilename config = getConfig('TestRobot') topdir=expandfilename(config['JobDir']) publishPath=expandfilename(config['PublishPath']) version = config['VersionNumber'] site = config['Site'] for i in range(len(config['TestPairs'])): jobdir = join(topdir, version+"_Job_"+str(i)) import tarfile test=config['TestPairs'][i][0].replace('/','.') option=config['TestPairs'][i][2] fname = version+'_'+test+'_'+option+'-'+site+'.tgz' archive = tarfile.open(join(publishPath,fname),'w:gz') archive.posix=False for file in [f for f in os.listdir(join(jobdir,'output')) if os.path.isfile(join(jobdir,'output',f))]: toname= file.replace(option,option+'-'+site) archive.add(join(jobdir,'output',file),join('output',toname)) for file in [f for f in os.listdir(join(jobdir,'reports','latest')) if os.path.isfile(join(jobdir,'reports','latest',f))]: toname = file.replace(option,option+'-'+site) archive.add(join(jobdir,'reports','latest',file),join(toname)) archive.close()
def _setLFNnamePattern(self, lfn="", namePattern=""): if self.defaultSE != "": ## TODO REPLACE THIS WITH IN LIST OF VONAMES KNOWN # Check for /lhcb/some/path or /gridpp/some/path if namePattern.split(os.pathsep)[0] == self.defaultSE \ or (len(namePattern) > 3 and namePattern[0:4].upper() == "LFN:"\ or len(namePattern.split(os.pathsep)) > 1 and namePattern.split(os.pathsep)[1] == self.defaultSE): # Check for LFN:/gridpp/some/path or others... lfn = namePattern namePattern = "" if lfn: if len(lfn) > 3 and lfn[0:4].upper() == "LFN:": lfn = lfn[4:] if lfn != "" and namePattern != "": self.lfn = lfn self.remoteDir = os.path.dirname(lfn) self.namePattern = os.path.basename(namePattern) self.localDir = os.path.dirname(expandfilename(namePattern)) elif lfn != "" and namePattern == "": self.lfn = lfn self.remoteDir = os.path.dirname(self.lfn) self.namePattern = os.path.basename(self.lfn) self.localDir = "" elif namePattern != "" and lfn == "": self.namePattern = os.path.basename(namePattern) self.localDir = os.path.dirname(expandfilename(namePattern)) self.remoteDir = "" self.lfn = ""
def __construct__(self, args): self.locations = [] if len(args) == 1 and isType(args[0], DiracFile): self.lfn = args[0].lfn self.namePattern = args[0].namePattern self.remoteDir = args[0].remoteDir self.localDir = args[0].localDir self.guid = args[0].guid self.compressed = args[0].compressed self._storedReplicas = args[0]._storedReplicas self._remoteURLs = args[0]._remoteURLs self.failureReason = args[0].failureReason self._have_copied = True self.subfiles = copy.deepcopy(args[0].subfiles) return # LFN ONLY if len(args) == 1 and type(args[0]) == type(""): if str(str(args[0]).upper()[0:4]) == str("LFN:"): self._setLFNnamePattern(lfn=args[0][4:], namePattern="") else: self._setLFNnamePattern(lfn="", namePattern=args[0]) # NAMEPATTERN AND LFN elif len(args) == 2 and type(args[0]) == type("") and type(args[1]) == type(""): self.namePattern = args[0] self._setLFNnamePattern(lfn="", namePattern=self.namePattern) self.localDir = expandfilename(args[1]) # NAMEPATTERN AND LFN AND LOCALDIR elif len(args) == 3 and type(args[0]) == type("") and type(args[1]) == type("") and type(args[2]) == type(""): self.namePattern = args[0] self.lfn = args[2] self._setLFNnamePattern(lfn=self.lfn, namePattern=self.namePattern) self.localDir = expandfilename(args[1]) # NAMEPATTERN AND LFN AND LOCALDIR AND REMOTEDIR elif ( len(args) == 4 and type(args[0]) == type("") and type(args[1]) == type("") and type(args[2]) == type("") and type(args[3]) == type("") ): self.namePattern = args[0] self.lfn = args[2] self._setLFNnamePattern(lfn=lfn, namePattern=namePattern) self.localDir = expandfilename(args[1]) self.remoteDir = args[3] # OTHER else: super(DiracFile, self).__construct__(args) return
def get_share_path(app=None): if app is None or not isType(app.is_prepared, ShareDir): return os.path.join( expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user']) #elif isType(app, ShareDir): else: return os.path.join( expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'], app.is_prepared.name)
def __construct__(self, args): self.locations = [] if len(args) == 1 and isType(args[0], DiracFile): self.lfn = args[0].lfn self.namePattern = args[0].namePattern self.remoteDir = args[0].remoteDir self.localDir = args[0].localDir self.guid = args[0].guid self.compressed = args[0].compressed self._storedReplicas = args[0]._storedReplicas self._remoteURLs = args[0]._remoteURLs self.failureReason = args[0].failureReason self._have_copied = True self.subfiles = copy.deepcopy(args[0].subfiles) return # LFN ONLY if len(args) == 1 and type(args[0]) == type(''): if str(str(args[0]).upper()[0:4]) == str("LFN:"): self._setLFNnamePattern(_lfn=args[0][4:], _namePattern="") else: self._setLFNnamePattern(_lfn="", _namePattern=args[0]) # NAMEPATTERN AND LFN elif len(args) == 2 and type(args[0]) == type('') and type( args[1]) == type(''): self.namePattern = args[0] self._setLFNnamePattern(_lfn='', _namePattern=self.namePattern) self.localDir = expandfilename(args[1]) # NAMEPATTERN AND LFN AND LOCALDIR elif len(args) == 3 and type(args[0]) == type('') and type( args[1]) == type('') and type(args[2]) == type(''): self.namePattern = args[0] self.lfn = args[2] self._setLFNnamePattern(_lfn=self.lfn, _namePattern=self.namePattern) self.localDir = expandfilename(args[1]) # NAMEPATTERN AND LFN AND LOCALDIR AND REMOTEDIR elif len(args) == 4 and type(args[0]) == type('') and type(args[1]) == type('')\ and type(args[2]) == type('') and type(args[3]) == type(''): self.namePattern = args[0] self.lfn = args[2] self._setLFNnamePattern(_lfn=lfn, _namePattern=namePattern) self.localDir = expandfilename(args[1]) self.remoteDir = args[3] # OTHER else: super(DiracFile, self).__construct__(args) return
def get_share_path(app=None): if app is None or app == "": return os.path.join( expandfilename(getConfig("Configuration")["gangadir"]), "shared", getConfig("Configuration")["user"] ) return os.path.join( expandfilename(getConfig("Configuration")["gangadir"]), "shared", getConfig("Configuration")["user"], app.is_prepared.name, )
def getPath(self,filename=None): subpath = self.subpath if filename is None: filename = '' if not self.jobid is None: jobdir = str(self.jobid) else: jobdir = '' # do not use subpath if no tree splitting applies if not self.splittree: subpath = '' if self.splittree: return expandfilename(os.path.join(self.top,subpath,jobdir,filename)) else: return expandfilename(os.path.join(self.top,jobdir,subpath,filename))
def _attribute_filter__set__(self, name, value): if value != "" and value is not None: # Do some checking of the filenames in a subprocess if name == 'lfn': self.namePattern = os.path.basename(value) if os.path.dirname(value): self.remoteDir = os.path.dirname(value) return value elif name == 'remoteDir': if self.lfn != os.path.join(value, self.namePattern): self.lfn = os.path.join(value, self.namePattern) elif name == 'namePattern': self.localDir = os.path.dirname(value) return os.path.basename(value) elif name == 'localDir': if value: return expandfilename(value) else: return value return value
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): from Ganga.Lib.gLite import gLiteJobConfig prepared_exe = app.exe if app.is_prepared is not None: shared_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user']) if isinstance(app.exe, str): # we have a file. is it an absolute path? if os.path.abspath(app.exe) == app.exe: logger.info("Submitting a prepared application; taking any input files from %s" % ( app.is_prepared.name)) prepared_exe = File(os.path.join(os.path.join( shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name))) # else assume it's a system binary, so we don't need to # transport anything to the sharedir else: prepared_exe = app.exe elif isinstance(app.exe, File): logger.info("Submitting a prepared application; taking any input files from %s" % ( app.is_prepared.name)) prepared_exe = File(os.path.join(os.path.join( shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name))) return gLiteJobConfig(prepared_exe, app._getParent().inputsandbox, convertIntToStringArgs(app.args), app._getParent().outputsandbox, app.env)
def __app_set__(obj, val): if not hasattr(obj, 'application') or obj.application is None: return if hasattr(obj.application, '_is_prepared'): #a=Job(); a.prepare(); a.application=Executable() if obj.application.is_prepared not in [None, True] and\ hasattr(val, 'is_prepared') and val.is_prepared is None: logger.debug('Overwriting a prepared application with one that is unprepared') obj.application.unprepare() #a=Job(); b=Executable(); b.prepare(); a.application=b elif obj.application.is_prepared is not True: if hasattr(val, 'is_prepared'): if val.is_prepared not in [None, True]: from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) logger.debug('Overwriting application with a prepared one') if stripProxy(obj.application) != val: stripProxy(obj.application).unprepare() shareref.increase(val.is_prepared.name) # check that the shared directory actually exists before # assigning the (prepared) application to a job if hasattr(val, 'is_prepared'): if val.is_prepared not in [None, True]: if hasattr(val.is_prepared, 'name'): from Ganga.Utility.files import expandfilename Config_conf = getConfig('Configuration') shared_path = os.path.join(expandfilename(Config_conf['gangadir']), 'shared', Config_conf['user']) if not os.path.isdir(os.path.join(shared_path, val.is_prepared.name)): logger.error('ShareDir directory not found: %s' % val.is_prepared.name)
def _join_opts_files(self): '''Create a single options file from all supplied options.''' joined_py_opts = '' for name in self.optsfiles: with open(expandfilename(name), 'r') as this_file: joined_py_opts += this_file.read() return joined_py_opts
def getSharedPath(): # Required to be initialized for ShareDir object from Ganga.Utility.files import expandfilename import os.path Conf_config = getConfig('Configuration') root_default = os.path.join(expandfilename(Conf_config['gangadir']), 'shared', Conf_config['user']) return root_default
def writefile(self, fileobj, executable=None): from Ganga.GPIDev.Lib.File import FileBuffer if not isType(fileobj, FileBuffer): raise GangaTypeError('Usage of tuples is not allowed, use FileBuffer instead') # output file name # Added a subdir to files, (see Ganga/GPIDev/Lib/File/File.py) This allows # to copy files into the a subdirectory of the workspace # FIXME: make a helper method for os.makedirs path_to_build = os.path.join(self.getPath(), fileobj.subdir) if not os.path.isdir(path_to_build): os.makedirs(path_to_build) logger.debug('created %s', self.getPath()) else: logger.debug('already exists: %s', self.getPath()) outname = expandfilename(self.getPath(fileobj.getPathInSandbox()), True) fileobj.create(outname) if executable: chmod_executable(outname) return outname
def writefile(self, fileobj, executable=None): from Ganga.GPIDev.Lib.File import FileBuffer if not isType(fileobj, FileBuffer): raise GangaTypeError( 'Usage of tuples is not allowed, use FileBuffer instead') # output file name # Added a subdir to files, (see Ganga/GPIDev/Lib/File/File.py) This allows # to copy files into the a subdirectory of the workspace # FIXME: make a helper method for os.makedirs path_to_build = os.path.join(self.getPath(), fileobj.subdir) if not os.path.isdir(path_to_build): os.makedirs(path_to_build) logger.debug('created %s', self.getPath()) else: logger.debug('already exists: %s', self.getPath()) outname = expandfilename(self.getPath(fileobj.getPathInSandbox()), True) fileobj.create(outname) if executable: chmod_executable(outname) return outname
def prepare(self, force=False): super(Ostap, self).prepare(force) self._check_inputs() share_dir = os.path.join ( expandfilename ( getConfig('Configuration')['gangadir'] ) , 'shared' , getConfig('Configuration')['user'] , self.is_prepared.name ) input_sandbox_tar = os.path.join ( share_dir , 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name ) input_sandbox_tgz = os.path.join ( share_dir , 'inputsandbox', '_input_sandbox_%s.tgz' % self.is_prepared.name ) fillPackedSandbox ( self.scripts , input_sandbox_tar ) gzipFile ( input_sandbox_tar , input_sandbox_tgz , True ) # add the newly created shared directory into the metadata system if # the app is associated with a persisted object self.checkPreparedHasParent(self) self.post_prepare() logger.debug("Finished Preparing Application in %s" % share_dir)
def __setattr__(self, attr, value): """ This is an overloaded setter method to make sure that we're auto-expanding the filenames of files which exist. In the case we're assigning any other attributes the value is simply passed through Args: attr (str): This is the name of the attribute which we're assigning value (unknown): This is the value being assigned. """ actual_value = value if attr == "namePattern": actual_value = os.path.basename(value) this_dir = os.path.dirname(value) if this_dir: super(DiracFile, self).__setattr__('localDir', this_dir) elif attr == 'localDir': if value: new_value = os.path.abspath(expandfilename(value)) if os.path.exists(new_value): actual_value = new_value elif attr == 'lfn': if value: this_name = os.path.basename(value) super(DiracFile, self).__setattr__('namePattern', this_name) super(DiracFile, self).__setattr__('remoteDir', os.path.dirname(value)) elif attr == 'remoteDir': if value: this_lfn = os.path.join(value, self.namePattern) super(DiracFile, self).__setattr__('lfn', this_lfn) super(DiracFile, self).__setattr__(attr, actual_value)
def prepare(self, force=False): super(BenderScript, self).prepare(force) self._check_inputs() share_dir = os.path.join ( expandfilename ( getConfig('Configuration')['gangadir'] ) , 'shared' , getConfig('Configuration')['user'] , self.is_prepared.name ) input_sandbox_tar = os.path.join ( share_dir , 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name ) input_sandbox_tgz = os.path.join ( share_dir , 'inputsandbox', '_input_sandbox_%s.tgz' % self.is_prepared.name ) fillPackedSandbox ( self.scripts + self.imports , input_sandbox_tar ) gzipFile ( input_sandbox_tar , input_sandbox_tgz , True ) # add the newly created shared directory into the metadata system if # the app is associated with a persisted object self.checkPreparedHasParent(self) self.post_prepare() logger.debug("Finished Preparing Application in %s" % share_dir)
def split(self, job): """ Method to do the splitting work Args: job (Job): master job to be used as a template to split subjobs """ from GangaLHCb.Lib.Applications import Gauss if not isinstance(job.application, (Gauss, GaudiExec)): logger.warning( "This application is of type: '%s', be careful how you use it with the GaussSplitter!" % type(job.application)) subjobs = [] inputdata = job.inputdata if not isinstance(job.application, GaudiExec): # I'm assuming this file is created by the Gauss Application at some stage? if not job.inputdata: share_path = os.path.join( expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'], job.application.is_prepared.name, 'inputdata', 'options_data.pkl') if os.path.exists(share_path): f = open(share_path, 'r+b') #FIXME should this have been an addition? inputdata = pickle.load(f) f.close() for i in range(self.numberOfJobs): j = self._create_subjob(job, inputdata) # FIXME this starts from the 1st event and not zero, is it clear why? first = self.firstEventNumber + i * self.eventsPerJob + 1 opts = 'from Configurables import GenInit \n' if isinstance(job.application, Gauss): opts += 'from Gaudi.Configuration import * \n' opts += 'ApplicationMgr().EvtMax = %d\n' % self.eventsPerJob else: opts += 'from Configurables import LHCbApp \n' opts += 'LHCbApp().EvtMax = %d\n' % self.eventsPerJob opts += 'GenInit("GaussGen").FirstEventNumber = %d\n' % first spillOver = ["GaussGenPrev", "GaussGenPrevPrev", "GaussGenNext"] for s in spillOver: opts += 'GenInit("%s").FirstEventNumber = %d\n' % (s, first) #j.application.extra.input_buffers['data.py'] += opts if isinstance(job.application, GaudiExec): j.application.extraOpts = j.application.extraOpts + '\n' + opts else: j._splitter_data = opts # j.inputsandbox.append(File(FileBuffer(path,opts).create().name)) logger.debug("Creating job %d w/ FirstEventNumber = %d" % (i, first)) subjobs.append(j) return subjobs
def create(self, outname): """create a file in a local filesystem as 'outname', maintain the original permissions """ import shutil shutil.copy(expandfilename(self.name), outname) if self.executable: chmod_executable(outname)
def download(self, dir='.'): 'Downloads the LFN to dir (default is current directory).' dir = expandfilename(dir) dir = os.path.abspath(dir) cmd = 'result = DiracCommands.getFile("%s","%s")' % (self.name, dir) result = get_result(cmd, 'Problem during download', 'Download error.') from PhysicalFile import PhysicalFile return GPIProxyObjectFactory(PhysicalFile(name=result['Value']))
def getDir(self): """ This function returns a sanitized absolute path of the self.directory method from user input """ if self.directory: return path.abspath(expandfilename(self.directory)) else: raise ApplicationConfigurationError(None, "No Opts File has been specified, please provide one!")
def getSharedPath(self): """ Return the full path of the shared directory where files are placed """ share_folder = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user']) if isinstance(self.is_prepared, ShareDir): return os.path.join(share_folder, self.is_prepared.name) else: return share_folder
def retrieve(self, type=None, name=None, **options): """Retieve files listed in outputdata and registered in output from remote SE to local filesystem in background thread""" from Ganga.GPIDev.Lib.Job import Job import os job = self._getParent() try: logger.info("ATLASOutputDataset.retrieve() called.") logger.debug('job.id: %d, Job.subjobs: %d', job.id, len(job.subjobs)) logger.debug('job.outputdir: %s', job.outputdir) logger.debug('job.outputsandbox: %s', job.outputsandbox) logger.debug('job.outputdata.outputsandbox: %s', job.outputdata) logger.debug('job.outputdata.outputdata: %s', job.outputdata.outputdata) logger.debug('job.outputdata.output: %s', job.outputdata.output) logger.debug('job.outputdata.location: %s', job.outputdata.location) logger.debug('job.outputdata.local_location: %s', job.outputdata.local_location) except AttributeError: logger.error('job.outputdata error') return 1 local_location = options.get('local_location') # Determine local output path to store files outputlocation = '' if job.outputdata.location and (job.backend._name == 'Local'): outputlocation = expandfilename(job.outputdata.location) elif local_location: outputlocation = expandfilename(local_location) elif job.outputdata.local_location: outputlocation = expandfilename(job.outputdata.local_location) else: # User job repository location #outputlocation = job.outputdir+'/../' pass # Output files outputfiles = job.outputdata.outputdata
def repairJobRepository(jobId): """ Repairs job repository for the comment attribute (migration from Comment object to string) """ def repairFilePath(filePath): fileRead = open(filePath, 'r') index = -1 found = False lines = fileRead.readlines() fileRead.close() for line in lines: index += 1 if line.find('<class name="Comment"') > -1: found = True break if found: newLines = [] for i in range(index): newLines.append(lines[i]) newLines.append(lines[index+1].replace('<attribute name="comment"> ', '')) for i in range(index+4, len(lines)): newLines.append(lines[i]) fileWrite = open(filePath, 'w') fileWrite.write(''.join(newLines)) fileWrite.close() from Ganga.Utility.Config import getConfig from Ganga.Utility.files import expandfilename import os if not isinstance(jobId,int): return repositoryPath = "repository/$usr/LocalXML/6.0/jobs/$thousandsNumxxx" repositoryPath = repositoryPath.replace('$usr', getConfig('Configuration')['user']) repositoryPath = repositoryPath.replace('$thousandsNum', str(jobId/1000)) repositoryFullPath = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), repositoryPath, str(jobId)) #repair also the subjobs data files for subjobDir in os.listdir(repositoryFullPath): repositorySubJobFullPath = os.path.join(repositoryFullPath, subjobDir) if os.path.isdir(repositorySubJobFullPath): repairFilePath(os.path.join(repositorySubJobFullPath, 'data')) repairFilePath(os.path.join(repositoryFullPath, 'data'))
def split(self, job): """ Method to do the splitting work Args: job (Job): master job to be used as a template to split subjobs """ from GangaLHCb.Lib.Applications import Gauss if not isinstance(job.application, (Gauss, GaudiExec)): logger.warning("This application is of type: '%s', be careful how you use it with the GaussSplitter!" % type(job.application)) subjobs = [] inputdata = job.inputdata if not isinstance(job.application, GaudiExec): # I'm assuming this file is created by the Gauss Application at some stage? if not job.inputdata: share_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'], job.application.is_prepared.name, 'inputdata', 'options_data.pkl') if os.path.exists(share_path): f = open(share_path, 'r+b') #FIXME should this have been an addition? inputdata = pickle.load(f) f.close() for i in range(self.numberOfJobs): j = self._create_subjob(job, inputdata) # FIXME this starts from the 1st event and not zero, is it clear why? first = self.firstEventNumber + i * self.eventsPerJob + 1 opts = 'from Configurables import GenInit \n' if isinstance(job.application, Gauss): opts += 'from Gaudi.Configuration import * \n' opts += 'ApplicationMgr().EvtMax = %d\n' % self.eventsPerJob else: opts += 'from Configurables import LHCbApp \n' opts += 'LHCbApp().EvtMax = %d\n' % self.eventsPerJob opts += 'GenInit("GaussGen").FirstEventNumber = %d\n' % first spillOver = ["GaussGenPrev", "GaussGenPrevPrev", "GaussGenNext"] for s in spillOver: opts += 'GenInit("%s").FirstEventNumber = %d\n' % (s, first) #j.application.extra.input_buffers['data.py'] += opts if isinstance(job.application, GaudiExec): j.application.extraOpts = j.application.extraOpts + '\n' + opts else: j._splitter_data = opts # j.inputsandbox.append(File(FileBuffer(path,opts).create().name)) logger.debug("Creating job %d w/ FirstEventNumber = %d" % (i, first)) subjobs.append(j) return subjobs
def full_expand_filename(name): if len(name) >= 4 and name[0:4].upper() == 'LFN:': msg = 'Can not create PhysicalFile from string that begins w/ "LFN:".'\ ' You probably want to create a LogicalFile.' raise GangaException(msg) urlprefix=re.compile('^(([a-zA-Z_][\w]*:)+/?)?/') if len(name) >= 4 and name[0:4].upper() == 'PFN:': name = name[4:] expanded_name = expandfilename(name) if urlprefix.match(expanded_name): return expanded_name return os.path.abspath(expanded_name)
def getLocalWorkspace(): # Get the local top level dirtectory for the Workspace if config['repositorytype'] in [ 'LocalXML', 'LocalAMGA', 'LocalPickle', 'SQLite' ]: return os.path.join(expandfilename(config['gangadir'], True), 'workspace', config['user'], config['repositorytype']) else: return ''
def full_expand_filename(name): if len(name) >= 4 and name[0:4].upper() == 'LFN:': msg = 'Can not create PhysicalFile from string that begins w/ "LFN:".'\ ' You probably want to create a LogicalFile.' raise GangaException(msg) urlprefix = re.compile('^(([a-zA-Z_][\w]*:)+/?)?/') if len(name) >= 4 and name[0:4].upper() == 'PFN:': name = name[4:] expanded_name = expandfilename(name) if urlprefix.match(expanded_name): return expanded_name return os.path.abspath(expanded_name)
def getSharedPath(self): """ Return the full path of the shared directory where files are placed """ share_folder = os.path.join( expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user']) if isinstance(self.is_prepared, ShareDir): return os.path.join(share_folder, self.is_prepared.name) else: return share_folder
def getPath(self, filename=None): subpath = self.subpath if filename is None: filename = '' if self.jobid is not None: jobdir = str(self.jobid) else: jobdir = '' subpath = '' return expandfilename(os.path.join(self.top, jobdir, subpath, filename), True)
def __construct__(self,args): if len(args) == 1 and type(args[0]) == type(''): v = args[0] import os.path expanded = expandfilename(v) if not urlprefix.match(expanded): # if it is not already an absolute filename self.name = os.path.abspath(expanded) else: #bugfix #20545 self.name = expanded else: super(File,self).__construct__(args)
def __setattr__(self, attr, value): """ This is an overloaded setter method to make sure that we're auto-expanding the filenames of files which exist. In the case we're assigning any other attributes the value is simply passed through Args: attr (str): This is the name of the attribute which we're assigning value (unknown): This is the value being assigned. """ actual_value = value if attr == "name": actual_value = expandfilename(value) super(File, self).__setattr__(attr, actual_value)
def __construct__(self, args): if len(args) == 1 and isType(args[0], DiracFile): self.lfn = args[0].lfn self.namePattern = args[0].namePattern self.remoteDir = args[0].remoteDir self.localDir = args[0].localDir self.guid = args[0].guid self.compressed = args[0].compressed self._storedReplicas = args[0]._storedReplicas self._remoteURLs = args[0]._remoteURLs self.failureReason = args[0].failureReason self._have_copied = True self.subfiles = copy.deepcopy(args[0].subfiles) return if len(args) == 1 and type(args[0]) == type(''): if str(str(args[0]).upper()[0:4]) == str("LFN:"): self._setLFNnamePattern(_lfn=args[0][4:], _namePattern="") else: self._setLFNnamePattern(_lfn="", _namePattern=args[0]) elif len(args) == 2 and type(args[0]) == type('') and type(args[1]) == type(''): self.namePattern = args[0] self._setLFNnamePattern(_lfn='', _namePattern=self.namePattern) self.localDir = expandfilename(args[1]) elif len(args) == 3 and type(args[0]) == type('') and type(args[1]) == type('') and type(args[2]) == type(''): self.namePattern = args[0] self.lfn = args[2] self._setLFNnamePattern(_lfn=self.lfn, _namePattern=self.namePattern) self.localDir = expandfilename(args[1]) elif len(args) == 4 and type(args[0]) == type('') and type(args[1]) == type('')\ and type(args[2]) == type('') and type(args[3]) == type(''): self.namePattern = args[0] self.lfn = args[2] self._setLFNnamePattern(_lfn=lfn, _namePattern=namePattern) self.localDir = expandfilename(args[1]) self.remoteDir = args[3] else: super(DiracFile, self).__construct__(args) return
def retrieve(self, type=None, name=None, **options ): """Retieve files listed in outputdata and registered in output from remote SE to local filesystem in background thread""" from Ganga.GPIDev.Lib.Job import Job import os job = self._getParent() try: logger.info("ATLASOutputDataset.retrieve() called.") logger.debug('job.id: %d, Job.subjobs: %d',job.id,len(job.subjobs)) logger.debug('job.outputdir: %s',job.outputdir) logger.debug('job.outputsandbox: %s',job.outputsandbox) logger.debug('job.outputdata.outputsandbox: %s',job.outputdata) logger.debug('job.outputdata.outputdata: %s',job.outputdata.outputdata) logger.debug('job.outputdata.output: %s',job.outputdata.output) logger.debug('job.outputdata.location: %s',job.outputdata.location) logger.debug('job.outputdata.local_location: %s',job.outputdata.local_location) except AttributeError: logger.error('job.outputdata error') return 1 local_location = options.get('local_location') # Determine local output path to store files outputlocation = '' if job.outputdata.location and (job.backend._name == 'Local'): outputlocation = expandfilename(job.outputdata.location) elif local_location: outputlocation = expandfilename(local_location) elif job.outputdata.local_location: outputlocation = expandfilename(job.outputdata.local_location) else: # User job repository location #outputlocation = job.outputdir+'/../' pass # Output files outputfiles = job.outputdata.outputdata
def add(self, input): from Ganga.Core.GangaRepository import getRegistry if not isType(input, list): input = [input] for item in input: if isType(item, str): if os.path.isfile(expandfilename(item)): logger.info('Copying file %s to shared directory %s' % (item, self.name)) shutil.copy2(expandfilename(item), os.path.join(getSharedPath(), self.name)) shareref = getRegistry("prep").getShareRef() shareref.increase(self.name) shareref.decrease(self.name) else: logger.error('File %s not found' % expandfilename(item)) elif isType(item, File) and item.name is not '' and os.path.isfile( expandfilename(item.name)): logger.info('Copying file object %s to shared directory %s' % (item.name, self.name)) shutil.copy2(expandfilename(item.name), os.path.join(getSharedPath(), self.name)) shareref = getRegistry("prep").getShareRef() shareref.increase(self.name) shareref.decrease(self.name) else: logger.error('File %s not found' % expandfilename(item.name))
def readfile(path): """Return the content of the specified file. Keyword arguments: path -- The path to the file (variables and tilde are expanded). """ path = files.expandfilename(path) f = open(path) try: return f.read() finally: f.close()
def copyIntoPrepDir(self, obj2copy): """ Method for actually copying the "obj2copy" object to the prepared state dir of this application Args: obj2copy (bool): is a string (local) address of a file to be copied as it's passed to shutil.copy2 """ shared_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user']) shr_dir = os.path.join(shared_path, self.is_prepared.name) if not os.path.isdir(shr_dir): os.makedirs(shr_dir) shutil.copy2(obj2copy, shr_dir) logger.debug("Copying %s into: %s" % (obj2copy, shr_dir))
def prepare(self, force=False): super(GaudiPython, self).prepare(force) self._check_inputs() share_dir = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'], self.is_prepared.name) fillPackedSandbox(self.script, os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name)) gzipFile(os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name), os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tgz' % self.is_prepared.name), True) # add the newly created shared directory into the metadata system if # the app is associated with a persisted object self.checkPreparedHasParent(self) self.post_prepare()
def test_GaudiPython_prepare(self): g = self.job.application g.prepare() assert g.is_prepared is not None, 'is_prepared not correctly set' g.is_prepared.ls() share_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'], g.is_prepared.name, 'inputsandbox') os.system( 'cd ' + share_path + ';tar -xzvf _input_sandbox_' + g.is_prepared.name + '.tgz') assert os.path.exists(os.path.join(share_path, 'dummy.script'))
def __init__(self, namePattern="", localDir=None, lfn="", remoteDir=None, **kwds): """ name is the name of the output file that has to be written ... """ super(DiracFile, self).__init__() self.locations = [] self._setLFNnamePattern(lfn, namePattern) if localDir is not None: self.localDir = expandfilename(localDir) if remoteDir is not None: self.remoteDir = remoteDir
def _setLFNnamePattern(self, _lfn="", _namePattern=""): if _lfn != "" and _lfn is not None: if len(_lfn) > 3 and _lfn[0:4] == "LFN:": _lfn = _lfn[4:] if _lfn != "" and _namePattern != "": self.lfn = _lfn self.remoteDir = os.path.dirname(_lfn) self.namePattern = os.path.basename(_namePattern) self.localDir = os.path.dirname(expandfilename(_namePattern)) elif _lfn != "" and _namePattern == "": self.lfn = _lfn self.remoteDir = os.path.dirname(self.lfn) if self.namePattern != "": self.namePattern = os.path.basename(self.lfn) self.localDir = "" elif _namePattern != "" and _lfn == "": self.namePattern = os.path.basename(_namePattern) self.localDir = os.path.dirname(expandfilename(_namePattern)) self.remoteDir = "" self.lfn = ""
def __construct__(self, args): if len(args) == 1 and isinstance(args[0], str): v = args[0] import os.path expanded = expandfilename(v) # if it is not already an absolute filename if not urlprefix.match(expanded): if os.path.exists(os.path.abspath(expanded)): self.name = os.path.abspath(expanded) else: self.name = v else: # bugfix #20545 self.name = expanded else: super(File, self).__construct__(args)
def writefile(path, content): """Write the content to the specified file, creating directories if necessary. Keyword arguments: path -- The path to the file (variables and tilde are expanded). context -- The file contexts as a string. """ path = files.expandfilename(path) directory = os.path.dirname(path) if not os.path.exists(directory): os.makedirs(directory) f = open(path, 'w') try: f.write(content) finally: f.close()
def __setattr__(self, attr, value): """ This is an overloaded setter method to make sure that we're auto-expanding the filenames of files which exist. In the case we're assigning any other attributes the value is simply passed through Args: attr (str): This is the name of the attribute which we're assigning value (unknown): This is the value being assigned. """ actual_value = value if attr == "namePattern": actual_value = os.path.basename(value) this_localDir = os.path.dirname(value) super(MassStorageFile, self).__setattr__('localDir', this_localDir) if attr == "localDir": actual_value = os.path.abspath(expandfilename(value)) super(MassStorageFile, self).__setattr__(attr, actual_value)
def prepare_cmake_app(myApp, myVer, myPath='$HOME/cmtuser', myGetpack=None): """ Short helper function for setting up minimal application environments on disk for job submission Args: myApp (str): This is the name of the app to pass to lb-dev myVer (str): This is the version of 'myApp' to pass tp lb-dev myPath (str): This is where lb-dev will be run myGepPack (str): This is a getpack which will be run once the lb-dev has executed """ full_path = expandfilename(myPath, True) if not path.exists(full_path): makedirs(full_path) chdir(full_path) _exec_cmd('lb-dev %s %s' % (myApp, myVer), full_path) dev_dir = path.join(full_path, myApp + 'Dev_' + myVer) logger.info("Set up App Env at: %s" % dev_dir) if myGetpack: _exec_cmd('getpack %s' % myGetpack, dev_dir) logger.info("Ran Getpack: %s" % myGetpack) return dev_dir
def __setattr__(self, attr, value): """ This is an overloaded setter method to make sure that we're auto-expanding the filenames of files which exist. In the case we're assigning any other attributes the value is simply passed through Args: attr (str): This is the name of the attribute which we're assigning value (unknown): This is the value being assigned. """ actual_value = value if attr == 'namePattern': if len(value.split(os.sep)) > 1: this_dir = path.dirname(value) super(LocalFile, self).__setattr__('localDir', this_dir) actual_value = path.basename(value) elif attr == 'localDir': if value: new_value = path.abspath(expandfilename(value)) if path.exists(new_value): actual_value = new_value super(LocalFile, self).__setattr__(attr, actual_value)
def _attribute_filter__set__(self, name, value): if value != "" and value is not None: # Do some checking of the filenames in a subprocess if name == 'lfn': this_dir, self.namePattern = os.path.split(value) if this_dir: self.remoteDir = this_dir return value elif name == 'namePattern': self.localDir, this_name = os.path.split(value) return this_name elif name == 'localDir': if value: return expandfilename(value) else: return value return value
def test_Gaudi_prepare(self): # Test standalone preparation d = DaVinci() d.prepare() assert d.is_prepared is not None, 'is_prepared not correctly set' #assert d._impl.prep_inputbox, 'inputbox empty' # Now test as part of job j = Job(application=DaVinci()) j.prepare() assert j.application.is_prepared is not None, 'is_prepared not correctly set' #assert j.application._impl.prep_inputbox, 'inputbox empty' job = Job(application=Gauss(optsfile='./Gauss-Job.py')) job.application.platform = 'x86_64-slc6-gcc48-opt' gauss = job.application job.inputdata = ['pfn:dummy1.in', 'pfn:dummy2.in'] job.outputfiles = [ 'Gauss.sim', 'GaussHistos.root', 'GaussMonitor.root'] #job.outputsandbox = ['GaussHistos.root','GaussMonitor.root'] #inputs,extra = gauss._impl.master_configure() # provide basic test of where output goes - a more complete test is # run on the PythonOptionsParser methods. job.prepare() # ok = job.application._impl.prep_outputbox.count('GaussHistos.root') > 0 and \ ## job.application._impl.prep_outputbox.count('GaussMonitor.root') > 0 share_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'], job.application.is_prepared.name) # ok = os.path.exists(os.path.join(job.application._impl.is_prepared.name,'GaussHistos.root')) and \ # os.path.exists(os.path.join(job.application._impl.is_prepared.name,'GaussMonitor.root')) assert os.path.exists( os.path.join(share_path, 'output', 'options_parser.pkl')), 'outputsandbox error' os.system('cd ' + share_path + '/inputsandbox/' + ';tar -xzvf _input_sandbox_' + job.application.is_prepared.name + '.tgz') assert os.path.exists(os.path.join( share_path, 'inputsandbox', 'options.pkl')), 'pickled options file error' assert os.path.exists( os.path.join(share_path, 'debug', 'gaudi-env.py.gz')), 'zipped env file error'