def allSampleInfo(sampleName, listLevel): if listLevel == 3: contents = castortools.ls(castorDir) for c in contents: print c #os.system("rfdir %s | awk '{print \"%s/\"$9}'" % (castorDir,castorDir) ) return print sampleName if listLevel > 0: print '------------------------------------------------------------------------------------------------------------' print 'PFN:' print castorDir print 'LFN:' print castortools.castorToLFN(castorDir) if listLevel > 1: contents = castortools.ls(castorDir) for c in contents: print c if listLevel > 0 and localDir != None: print 'local:' print localDir if os.path.isdir(localDir): if listLevel > 1: os.system('ls -l ' + localDir) # print localDir + '*.root' else: if listLevel > 0: print 'TO BE IMPORTED' if listLevel > 0: print print
def __init__(self, dirLocalOrTgzDirOnCastor, castorTgz, dbsAPI): #self.dbAPI = DatabaseAPI.DatabaseAPI('/afs/cern.ch/user/p/pmeckiff/public/bookkeeping.db') self.dirLocal = None self.tgzDirOnCastor = None self.dirOnCastor = None self.setName = dirLocalOrTgzDirOnCastor self.dbsAPI = dbsAPI # Set Directory name if local local = dirLocalOrTgzDirOnCastor.rstrip('/') castorTgz = castortools.castorToLFN(castorTgz) print castorTgz # Check if local first (obviously) if self.isDirLocal(local): print "File is on local machine: " + local self.dirLocal = local #if found set class attribute # Check if on castor next elif self.isTgzDirOnCastor(castorTgz): print "File is directory on Castor" self.tgzDirOnCastor = castorTgz # if found set class attribute for i in castortools.matchingFiles(castorTgz.rstrip("/Logger.tgz"), ".*tgz"): print i # If logger is not present but directory exists elif self.isDirOnCastor(castorTgz.rstrip("/Logger.tgz")): print "Directory is valid on Castor, but no logger file is present." self.dirOnCastor = castorTgz.rstrip("/Logger.tgz") # If neither then raise an exception else: raise ValueError( dirLocalOrTgzDirOnCastor + ' is neither a tgz directory on castor (provide a LFN!) nor a local directory' )
def datasetString(self, details, files, tags, castorDir, comment): detailArray = [] # Print name and primary set at top detailArray.append( "*%s* " % details['PathList'][0]) detailArray.append("\t*Status*: " + details['Status']) castor = "/store/cmst3/user/" + details['CreatedBy'] + "/" + details['PhysicsGroup'] + details['PathList'][0] detailArray.append("\t*LFN Castor Directory*:\n\t" + castortools.castorToLFN(castorDir)) detailArray.append("\t*Castor Directory*:\n\t" + castorDir) if details['ParentList']: detailArray.append("\t*Parent Dataset*: \n\t" + details['ParentList'][0]) else: detailArray.append("\t*Parent Dataset*: \n\t No Parent") # Print all other data for element in details: if element != "DateCreated" and element != "PhysicsGroup" and element != 'CreatedBy': pass # If no entry found: elif not details[element]: detailArray.append("\t*%s*: No entry found" % element) # If item is a list elif type(details[element]) == list: # Create human readable name name = "\t*%s*:" % re.sub( r"([A-Z])", r" \1", element).lstrip(" ") # If list is empty: if not details[element]: detailArray.append(name + ": No entry found") else: detailArray.append(name) for i in details[element]: detailArray.append("\t\t%s"% i) # If item is a date attribute elif element == 'LastModificationDate' or element == 'DateCreated': # Takes timestamp in dataset object and converts to a human readable date detailArray.append("\t*%s*:" % re.sub( r"([A-Z])", r" \1", element).lstrip(" ") + " " + datetime.date.fromtimestamp(int(details[element])).strftime('%d-%m-%Y')) # Otherwise just print the value else: detailArray.append("\t*%s*: " % re.sub( r"([A-Z])", r" \1", element).lstrip(" ") + details[element]) # Cycle through array of details and add them to a string detailString = "" for i in detailArray: detailString += i + "\n" detailString+= "\n" if tags!=None: detailString+="*Tags*: \n" for i in tags: tag = i.split(" ")[0] package = i.split(" ")[1] detailString +="_"+tag+"_ "+package +"\n" if files!=None: detailString+="\n*Root files*:\n" for i in files: detailString +="\t"+ i + "\n\n" if comment!= None: detailString+="\n*Additional Comments*\n"+ comment return detailString
def isTgzDirOnCastor(self, file): if not castortools.isCastorDir(file): file = castortools.castorToLFN(file) if castortools.isLFN(file): tgzPattern = re.compile('.*\.tgz$') m = tgzPattern.match(file) if m: return True else: return False else: return False
def run(self, input): """Check that the directory is writable""" dir = input['FindOnCastor']['Directory'] if self.options.check: _, name = tempfile.mkstemp('.txt',text=True) testFile = file(name,'w') testFile.write('Test file') testFile.close() store = castortools.castorToLFN(dir) #this is bad, but castortools is giving me problems if not os.system('cmsStage %s %s' % (name,store)): fname = '%s/%s' % (dir,os.path.basename(name)) write = castortools.fileExists(fname) if write: castortools.rm(fname) else: raise Exception("Failed to write to directory '%s'" % dir) os.remove(name) return {'Directory':dir,'WriteAccess':True}
def run(self, input): find = FindOnCastor(self.dataset,self.options.batch_user,self.options) find.create = True out = find.run({}) full = input['ExpandConfig']['ExpandedFullCFG'] jobdir = input['CreateJobDirectory']['JobDir'] sampleDir = os.path.join(out['Directory'],self.options.tier) sampleDir = castortools.castorToLFN(sampleDir) cmd = ['cmsBatch.py',str(self.options.nInput),os.path.basename(full),'-o','%s_Jobs' % self.options.tier,'--force'] cmd.extend(['-r',sampleDir]) if self.options.run_batch: jname = "%s/%s" % (self.dataset,self.options.tier) jname = jname.replace("//","/") cmd.extend(['-b',"'bsub -q %s -J %s < ./batchScript.sh | tee job_id.txt'" % (self.options.queue,jname)]) print " ".join(cmd) pwd = os.getcwd() error = None try: os.chdir(jobdir) returncode = os.system(" ".join(cmd)) if returncode != 0: error = "Running cmsBatch failed. Return code was %i." % returncode finally: os.chdir(pwd) if error is not None: raise Exception(error) return {'SampleDataset':"%s/%s" % (self.dataset,self.options.tier),'BatchUser':self.options.batch_user, 'SampleOutputDir':sampleDir,'LSFJobsTopDir':os.path.join(jobdir,'%s_Jobs' % self.options.tier)}