def test(self, previous=None, timeout=-1):
if not castortools.fileExists(self.directory):
raise Exception(
"The top level directory '%s' for this dataset does not exist"
% self.directory)
self.query()
test_results = {}
#support updating to speed things up
prev_results = {}
if previous is not None:
for name, status in previous['Files'].iteritems():
prev_results[name] = status
filesToTest = self.sortByBaseDir(self.listRootFiles(self.directory))
for dir, filelist in filesToTest.iteritems():
filemask = {}
#apply a UNIX wildcard if specified
filtered = filelist
if self.options.wildcard is not None:
filtered = fnmatch.filter(filelist, self.options.wildcard)
if not filtered:
print >> sys.stderr, "Warning: The wildcard '%s' does not match any files in '%s'. Please check you are using quotes." % (
self.options.wildcard, self.directory)
count = 0
for ff in filtered:
fname = os.path.join(dir, ff)
lfn = castortools.castorToLFN(fname)
#try to update from the previous result if available
if lfn in prev_results and prev_results[lfn][0]:
if self.options.printout:
print '[%i/%i]\t Skipping %s...' % (
count, len(filtered), fname),
OK, num = prev_results[lfn]
else:
if self.options.printout:
print '[%i/%i]\t Checking %s...' % (
count, len(filtered), fname),
OK, num = self.testFileTimeOut(lfn, timeout)
filemask[ff] = (OK, num)
if self.options.printout:
print(OK, num)
if OK:
self.eventsSeen += num
count += 1
test_results[castortools.castorToLFN(dir)] = filemask
self.test_result = test_results
self.duplicates, self.bad_jobs, sum_dup = self.stripDuplicates()
#remove duplicate entries from the event count
self.eventsSeen -= sum_dup
def test(self, previous=None, timeout=-1):
if not castortools.fileExists(self.directory):
raise Exception("The top level directory '%s' for this dataset does not exist" % self.directory)
self.query()
test_results = {}
# support updating to speed things up
prev_results = {}
if previous is not None:
for name, status in previous["Files"].iteritems():
prev_results[name] = status
filesToTest = self.sortByBaseDir(self.listRootFiles(self.directory))
for dir, filelist in filesToTest.iteritems():
filemask = {}
# apply a UNIX wildcard if specified
filtered = filelist
if self.options.wildcard is not None:
filtered = fnmatch.filter(filelist, self.options.wildcard)
if not filtered:
print >> sys.stderr, "Warning: The wildcard '%s' does not match any files in '%s'. Please check you are using quotes." % (
self.options.wildcard,
self.directory,
)
count = 0
for ff in filtered:
fname = os.path.join(dir, ff)
lfn = castortools.castorToLFN(fname)
# try to update from the previous result if available
if lfn in prev_results and prev_results[lfn][0]:
if self.options.printout:
print "[%i/%i]\t Skipping %s..." % (count, len(filtered), fname),
OK, num = prev_results[lfn]
else:
if self.options.printout:
print "[%i/%i]\t Checking %s..." % (count, len(filtered), fname),
OK, num = self.testFileTimeOut(lfn, timeout)
filemask[ff] = (OK, num)
if self.options.printout:
print (OK, num)
if OK:
self.eventsSeen += num
count += 1
test_results[castortools.castorToLFN(dir)] = filemask
self.test_result = test_results
self.duplicates, self.bad_jobs, sum_dup = self.stripDuplicates()
# remove duplicate entries from the event count
self.eventsSeen -= sum_dup
def publish(self, report):
"""Publish a file"""
for path in report['PathList']:
_, name = tempfile.mkstemp('.txt', text=True)
json.dump(report, file(name,'w'), sort_keys=True, indent=4)
fname = '%s_%s.txt' % (self.parent, report['DateCreated'])
#rename the file locally - TODO: This is a potential problem
nname = os.path.join(os.path.dirname(name),fname)
os.rename(name, nname)
castor_path = castortools.lfnToCastor(path)
new_name = '%s/%s' % (castor_path, fname)
castortools.xrdcp(nname,path)
time.sleep(1)
if castortools.fileExists(new_name):
#castortools.move(old_name, new_name)
#castortools.chmod(new_name, '644')
print "File published: '%s'" % castortools.castorToLFN(new_name)
os.remove(nname)
else:
pathhash = path.replace('/','.')
hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname)
shutil.move(nname, hashed_name)
print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % (castor_path, hashed_name)
def publish(self, report):
"""Publish a file"""
for path in report['PathList']:
_, name = tempfile.mkstemp('.txt', text=True)
json.dump(report, file(name, 'w'), sort_keys=True, indent=4)
fname = '%s_%s.txt' % (self.parent, report['DateCreated'])
#rename the file locally - TODO: This is a potential problem
nname = os.path.join(os.path.dirname(name), fname)
os.rename(name, nname)
castor_path = castortools.lfnToCastor(path)
new_name = '%s/%s' % (castor_path, fname)
castortools.xrdcp(nname, path)
time.sleep(1)
if castortools.fileExists(new_name):
#castortools.move(old_name, new_name)
#castortools.chmod(new_name, '644')
print "File published: '%s'" % castortools.castorToLFN(
new_name)
os.remove(nname)
else:
pathhash = path.replace('/', '.')
hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname)
shutil.move(nname, hashed_name)
print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % (
castor_path, hashed_name)
def run(self, input):
find = FindOnCastor(self.dataset, self.options.batch_user,
self.options)
find.create = True
out = find.run({})
full = input['ExpandConfig']['ExpandedFullCFG']
jobdir = input['CreateJobDirectory']['JobDir']
sampleDir = os.path.join(out['Directory'], self.options.tier)
sampleDir = castortools.castorToLFN(sampleDir)
cmd = [
'cmsBatch.py',
str(self.options.nInput),
os.path.basename(full), '-o',
'%s_Jobs' % self.options.tier, '--force'
]
cmd.extend(['-r', sampleDir])
if self.options.run_batch:
jname = "%s/%s" % (self.dataset, self.options.tier)
jname = jname.replace("//", "/")
user_group = ''
if self.options.group is not None:
user_group = '-G %s' % self.options.group
cmd.extend([
'-b',
"'bsub -q %s -J %s -u [email protected] %s < ./batchScript.sh | tee job_id.txt'"
% (self.options.queue, jname, user_group)
])
print " ".join(cmd)
pwd = os.getcwd()
error = None
try:
os.chdir(jobdir)
returncode = os.system(" ".join(cmd))
if returncode != 0:
error = "Running cmsBatch failed. Return code was %i." % returncode
finally:
os.chdir(pwd)
if error is not None:
raise Exception(error)
return {
'SampleDataset': "%s/%s" % (self.dataset, self.options.tier),
'BatchUser': self.options.batch_user,
'SampleOutputDir': sampleDir,
'LSFJobsTopDir': os.path.join(jobdir,
'%s_Jobs' % self.options.tier)
}
def isTgzDirOnEOS(self, file):
'''Checks if file is a .tgz file in an eos dir'''
if not castortools.isCastorDir(file):
file = castortools.castorToLFN(file)
if castortools.isLFN(file):
tgzPattern = re.compile('.*\.tgz$')
m = tgzPattern.match(file)
if m:
return True
else:
return False
else:
return False
def isTgzDirOnEOS(self, file ):
'''Checks if file is a .tgz file in an eos dir'''
if not castortools.isCastorDir( file ):
file = castortools.castorToLFN(file)
if castortools.isLFN( file ):
tgzPattern = re.compile('.*\.tgz$')
m = tgzPattern.match( file )
if m:
return True
else:
return False
else:
return False
def run(self, input):
find = FindOnCastor(self.dataset,self.options.batch_user,self.options)
find.create = True
out = find.run({})
full = input['ExpandConfig']['ExpandedFullCFG']
jobdir = input['CreateJobDirectory']['JobDir']
sampleDir = os.path.join(out['Directory'],self.options.tier)
sampleDir = castortools.castorToLFN(sampleDir)
cmd = ['cmsBatch.py',str(self.options.nInput),os.path.basename(full),'-o','%s_Jobs' % self.options.tier,'--force']
cmd.extend(['-r',sampleDir])
if self.options.run_batch:
jname = "%s/%s" % (self.dataset,self.options.tier)
jname = jname.replace("//","/")
user_group = ''
if self.options.group is not None:
user_group = '-G %s' % self.options.group
cmd.extend(['-b',"'bsub -q %s -J %s -u [email protected] %s < ./batchScript.sh | tee job_id.txt'" % (self.options.queue,jname,user_group)])
print " ".join(cmd)
pwd = os.getcwd()
error = None
try:
os.chdir(jobdir)
returncode = os.system(" ".join(cmd))
if returncode != 0:
error = "Running cmsBatch failed. Return code was %i." % returncode
finally:
os.chdir(pwd)
if error is not None:
raise Exception(error)
return {'SampleDataset':"%s/%s" % (self.dataset,self.options.tier),'BatchUser':self.options.batch_user,
'SampleOutputDir':sampleDir,'LSFJobsTopDir':os.path.join(jobdir,'%s_Jobs' % self.options.tier)}
def run(self, input):
"""Check that the directory is writable"""
if self.user == 'CMS':
return {'Directory':None,'WriteAccess':True}
dir = input['FindOnCastor']['Directory']
if self.options.check:
_, name = tempfile.mkstemp('.txt',text=True)
testFile = file(name,'w')
testFile.write('Test file')
testFile.close()
store = castortools.castorToLFN(dir)
#this is bad, but castortools is giving me problems
if not os.system('cmsStage %s %s' % (name,store)):
fname = '%s/%s' % (dir,os.path.basename(name))
write = castortools.fileExists(fname)
if write:
castortools.rm(fname)
else:
raise Exception("Failed to write to directory '%s'" % dir)
os.remove(name)
return {'Directory':dir,'WriteAccess':True}
def run(self, input):
"""Check that the directory is writable"""
if self.user == 'CMS':
return {'Directory': None, 'WriteAccess': True}
dir = input['FindOnCastor']['Directory']
if self.options.check:
_, name = tempfile.mkstemp('.txt', text=True)
testFile = file(name, 'w')
testFile.write('Test file')
testFile.close()
store = castortools.castorToLFN(dir)
#this is bad, but castortools is giving me problems
if not os.system('cmsStage %s %s' % (name, store)):
fname = '%s/%s' % (dir, os.path.basename(name))
write = castortools.fileExists(fname)
if write:
castortools.rm(fname)
else:
raise Exception("Failed to write to directory '%s'" % dir)
os.remove(name)
return {'Directory': dir, 'WriteAccess': True}
