def allSampleInfo(sampleName, listLevel):
if listLevel == 3:
contents = castortools.ls(castorDir)
for c in contents:
print c
#os.system("rfdir %s | awk '{print \"%s/\"$9}'" % (castorDir,castorDir) )
return
print sampleName
if listLevel > 0:
print '------------------------------------------------------------------------------------------------------------'
print 'PFN:'
print castorDir
print 'LFN:'
print castortools.castorToLFN(castorDir)
if listLevel > 1:
contents = castortools.ls(castorDir)
for c in contents:
print c
if listLevel > 0 and localDir != None:
print 'local:'
print localDir
if os.path.isdir(localDir):
if listLevel > 1:
os.system('ls -l ' + localDir)
# print localDir + '*.root'
else:
if listLevel > 0:
print 'TO BE IMPORTED'
if listLevel > 0:
print
print
def __init__(self, dirLocalOrTgzDirOnCastor, castorTgz, dbsAPI):
#self.dbAPI = DatabaseAPI.DatabaseAPI('/afs/cern.ch/user/p/pmeckiff/public/bookkeeping.db')
self.dirLocal = None
self.tgzDirOnCastor = None
self.dirOnCastor = None
self.setName = dirLocalOrTgzDirOnCastor
self.dbsAPI = dbsAPI
# Set Directory name if local
local = dirLocalOrTgzDirOnCastor.rstrip('/')
castorTgz = castortools.castorToLFN(castorTgz)
print castorTgz
# Check if local first (obviously)
if self.isDirLocal(local):
print "File is on local machine: " + local
self.dirLocal = local #if found set class attribute
# Check if on castor next
elif self.isTgzDirOnCastor(castorTgz):
print "File is directory on Castor"
self.tgzDirOnCastor = castorTgz # if found set class attribute
for i in castortools.matchingFiles(castorTgz.rstrip("/Logger.tgz"),
".*tgz"):
print i
# If logger is not present but directory exists
elif self.isDirOnCastor(castorTgz.rstrip("/Logger.tgz")):
print "Directory is valid on Castor, but no logger file is present."
self.dirOnCastor = castorTgz.rstrip("/Logger.tgz")
# If neither then raise an exception
else:
raise ValueError(
dirLocalOrTgzDirOnCastor +
' is neither a tgz directory on castor (provide a LFN!) nor a local directory'
)
def datasetString(self, details, files, tags, castorDir, comment):
detailArray = []
# Print name and primary set at top
detailArray.append( "*%s* " % details['PathList'][0])
detailArray.append("\t*Status*: " + details['Status'])
castor = "/store/cmst3/user/" + details['CreatedBy'] + "/" + details['PhysicsGroup'] + details['PathList'][0]
detailArray.append("\t*LFN Castor Directory*:\n\t" + castortools.castorToLFN(castorDir))
detailArray.append("\t*Castor Directory*:\n\t" + castorDir)
if details['ParentList']: detailArray.append("\t*Parent Dataset*: \n\t" + details['ParentList'][0])
else: detailArray.append("\t*Parent Dataset*: \n\t No Parent")
# Print all other data
for element in details:
if element != "DateCreated" and element != "PhysicsGroup" and element != 'CreatedBy':
pass
# If no entry found:
elif not details[element]:
detailArray.append("\t*%s*: No entry found" % element)
# If item is a list
elif type(details[element]) == list:
# Create human readable name
name = "\t*%s*:" % re.sub( r"([A-Z])", r" \1", element).lstrip(" ")
# If list is empty:
if not details[element]: detailArray.append(name + ": No entry found")
else:
detailArray.append(name)
for i in details[element]: detailArray.append("\t\t%s"% i)
# If item is a date attribute
elif element == 'LastModificationDate' or element == 'DateCreated':
# Takes timestamp in dataset object and converts to a human readable date
detailArray.append("\t*%s*:" % re.sub( r"([A-Z])", r" \1", element).lstrip(" ") + " " + datetime.date.fromtimestamp(int(details[element])).strftime('%d-%m-%Y'))
# Otherwise just print the value
else:
detailArray.append("\t*%s*: " % re.sub( r"([A-Z])", r" \1", element).lstrip(" ") + details[element])
# Cycle through array of details and add them to a string
detailString = ""
for i in detailArray:
detailString += i + "\n"
detailString+= "\n"
if tags!=None:
detailString+="*Tags*: \n"
for i in tags:
tag = i.split(" ")[0]
package = i.split(" ")[1]
detailString +="_"+tag+"_ "+package +"\n"
if files!=None:
detailString+="\n*Root files*:\n"
for i in files:
detailString +="\t"+ i + "\n\n"
if comment!= None:
detailString+="\n*Additional Comments*\n"+ comment
return detailString
def isTgzDirOnCastor(self, file):
if not castortools.isCastorDir(file):
file = castortools.castorToLFN(file)
if castortools.isLFN(file):
tgzPattern = re.compile('.*\.tgz$')
m = tgzPattern.match(file)
if m:
return True
else:
return False
else:
return False
def run(self, input):
"""Check that the directory is writable"""
dir = input['FindOnCastor']['Directory']
if self.options.check:
_, name = tempfile.mkstemp('.txt',text=True)
testFile = file(name,'w')
testFile.write('Test file')
testFile.close()
store = castortools.castorToLFN(dir)
#this is bad, but castortools is giving me problems
if not os.system('cmsStage %s %s' % (name,store)):
fname = '%s/%s' % (dir,os.path.basename(name))
write = castortools.fileExists(fname)
if write:
castortools.rm(fname)
else:
raise Exception("Failed to write to directory '%s'" % dir)
os.remove(name)
return {'Directory':dir,'WriteAccess':True}
def run(self, input):
find = FindOnCastor(self.dataset,self.options.batch_user,self.options)
find.create = True
out = find.run({})
full = input['ExpandConfig']['ExpandedFullCFG']
jobdir = input['CreateJobDirectory']['JobDir']
sampleDir = os.path.join(out['Directory'],self.options.tier)
sampleDir = castortools.castorToLFN(sampleDir)
cmd = ['cmsBatch.py',str(self.options.nInput),os.path.basename(full),'-o','%s_Jobs' % self.options.tier,'--force']
cmd.extend(['-r',sampleDir])
if self.options.run_batch:
jname = "%s/%s" % (self.dataset,self.options.tier)
jname = jname.replace("//","/")
cmd.extend(['-b',"'bsub -q %s -J %s < ./batchScript.sh | tee job_id.txt'" % (self.options.queue,jname)])
print " ".join(cmd)
pwd = os.getcwd()
error = None
try:
os.chdir(jobdir)
returncode = os.system(" ".join(cmd))
if returncode != 0:
error = "Running cmsBatch failed. Return code was %i." % returncode
finally:
os.chdir(pwd)
if error is not None:
raise Exception(error)
return {'SampleDataset':"%s/%s" % (self.dataset,self.options.tier),'BatchUser':self.options.batch_user,
'SampleOutputDir':sampleDir,'LSFJobsTopDir':os.path.join(jobdir,'%s_Jobs' % self.options.tier)}