def publish(self, report):
"""Publish a file"""
for path in report['PathList']:
_, name = tempfile.mkstemp('.txt', text=True)
json.dump(report, file(name,'w'), sort_keys=True, indent=4)
fname = '%s_%s.txt' % (self.parent, report['DateCreated'])
#rename the file locally - TODO: This is a potential problem
nname = os.path.join(os.path.dirname(name),fname)
os.rename(name, nname)
castor_path = castortools.lfnToCastor(path)
new_name = '%s/%s' % (castor_path, fname)
castortools.xrdcp(nname,path)
time.sleep(1)
if castortools.fileExists(new_name):
#castortools.move(old_name, new_name)
#castortools.chmod(new_name, '644')
print "File published: '%s'" % castortools.castorToLFN(new_name)
os.remove(nname)
else:
pathhash = path.replace('/','.')
hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname)
shutil.move(nname, hashed_name)
print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % (castor_path, hashed_name)
def publish(self, report):
"""Publish a file"""
for path in report['PathList']:
_, name = tempfile.mkstemp('.txt', text=True)
json.dump(report, file(name, 'w'), sort_keys=True, indent=4)
fname = '%s_%s.txt' % (self.parent, report['DateCreated'])
#rename the file locally - TODO: This is a potential problem
nname = os.path.join(os.path.dirname(name), fname)
os.rename(name, nname)
castor_path = castortools.lfnToCastor(path)
new_name = '%s/%s' % (castor_path, fname)
castortools.xrdcp(nname, path)
time.sleep(1)
if castortools.fileExists(new_name):
#castortools.move(old_name, new_name)
#castortools.chmod(new_name, '644')
print "File published: '%s'" % castortools.castorToLFN(
new_name)
os.remove(nname)
else:
pathhash = path.replace('/', '.')
hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname)
shutil.move(nname, hashed_name)
print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % (
castor_path, hashed_name)
def test(self, previous=None, timeout=-1):
if not castortools.fileExists(self.directory):
raise Exception(
"The top level directory '%s' for this dataset does not exist"
% self.directory)
self.query()
test_results = {}
#support updating to speed things up
prev_results = {}
if previous is not None:
for name, status in previous['Files'].iteritems():
prev_results[name] = status
filesToTest = self.sortByBaseDir(self.listRootFiles(self.directory))
for dir, filelist in filesToTest.iteritems():
filemask = {}
#apply a UNIX wildcard if specified
filtered = filelist
if self.options.wildcard is not None:
filtered = fnmatch.filter(filelist, self.options.wildcard)
if not filtered:
print >> sys.stderr, "Warning: The wildcard '%s' does not match any files in '%s'. Please check you are using quotes." % (
self.options.wildcard, self.directory)
count = 0
for ff in filtered:
fname = os.path.join(dir, ff)
lfn = castortools.castorToLFN(fname)
#try to update from the previous result if available
if lfn in prev_results and prev_results[lfn][0]:
if self.options.printout:
print '[%i/%i]\t Skipping %s...' % (
count, len(filtered), fname),
OK, num = prev_results[lfn]
else:
if self.options.printout:
print '[%i/%i]\t Checking %s...' % (
count, len(filtered), fname),
OK, num = self.testFileTimeOut(lfn, timeout)
filemask[ff] = (OK, num)
if self.options.printout:
print(OK, num)
if OK:
self.eventsSeen += num
count += 1
test_results[castortools.castorToLFN(dir)] = filemask
self.test_result = test_results
self.duplicates, self.bad_jobs, sum_dup = self.stripDuplicates()
#remove duplicate entries from the event count
self.eventsSeen -= sum_dup
def test(self, previous=None, timeout=-1):
if not castortools.fileExists(self.directory):
raise Exception("The top level directory '%s' for this dataset does not exist" % self.directory)
self.query()
test_results = {}
# support updating to speed things up
prev_results = {}
if previous is not None:
for name, status in previous["Files"].iteritems():
prev_results[name] = status
filesToTest = self.sortByBaseDir(self.listRootFiles(self.directory))
for dir, filelist in filesToTest.iteritems():
filemask = {}
# apply a UNIX wildcard if specified
filtered = filelist
if self.options.wildcard is not None:
filtered = fnmatch.filter(filelist, self.options.wildcard)
if not filtered:
print >> sys.stderr, "Warning: The wildcard '%s' does not match any files in '%s'. Please check you are using quotes." % (
self.options.wildcard,
self.directory,
)
count = 0
for ff in filtered:
fname = os.path.join(dir, ff)
lfn = castortools.castorToLFN(fname)
# try to update from the previous result if available
if lfn in prev_results and prev_results[lfn][0]:
if self.options.printout:
print "[%i/%i]\t Skipping %s..." % (count, len(filtered), fname),
OK, num = prev_results[lfn]
else:
if self.options.printout:
print "[%i/%i]\t Checking %s..." % (count, len(filtered), fname),
OK, num = self.testFileTimeOut(lfn, timeout)
filemask[ff] = (OK, num)
if self.options.printout:
print (OK, num)
if OK:
self.eventsSeen += num
count += 1
test_results[castortools.castorToLFN(dir)] = filemask
self.test_result = test_results
self.duplicates, self.bad_jobs, sum_dup = self.stripDuplicates()
# remove duplicate entries from the event count
self.eventsSeen -= sum_dup
def run(self, input):
if self.user == 'CMS':
return {'Topdir':None,'Directory':None}
topdir = castortools.lfnToCastor(castorBaseDir(user=self.user))
directory = '%s/%s' % (topdir,self.dataset)
# directory = directory.replace('//','/')
if not castortools.fileExists(directory):
if hasattr(self,'create') and self.create:
castortools.createCastorDir(directory)
#castortools.chmod(directory,'775')
if not castortools.isDirectory(directory):
raise Exception("Dataset directory '%s' does not exist or could not be created" % directory)
return {'Topdir':topdir,'Directory':directory}
def run(self, input):
if self.user == 'CMS':
return {'Topdir':None,'Directory':None}
topdir = castortools.lfnToCastor(castorBaseDir(user=self.user))
directory = '%s/%s' % (topdir,self.dataset)
# directory = directory.replace('//','/')
if not castortools.fileExists(directory):
if hasattr(self,'create') and self.create:
castortools.createCastorDir(directory)
#castortools.chmod(directory,'775')
if not castortools.isDirectory(directory):
raise Exception("Dataset directory '%s' does not exist or could not be created" % directory)
return {'Topdir':topdir,'Directory':directory}
def run(self, input):
"""Check that the directory is writable"""
if self.user == 'CMS':
return {'Directory':None,'WriteAccess':True}
dir = input['FindOnCastor']['Directory']
if self.options.check:
_, name = tempfile.mkstemp('.txt',text=True)
testFile = file(name,'w')
testFile.write('Test file')
testFile.close()
store = castortools.castorToLFN(dir)
#this is bad, but castortools is giving me problems
if not os.system('cmsStage %s %s' % (name,store)):
fname = '%s/%s' % (dir,os.path.basename(name))
write = castortools.fileExists(fname)
if write:
castortools.rm(fname)
else:
raise Exception("Failed to write to directory '%s'" % dir)
os.remove(name)
return {'Directory':dir,'WriteAccess':True}
def run(self, input):
"""Check that the directory is writable"""
if self.user == 'CMS':
return {'Directory': None, 'WriteAccess': True}
dir = input['FindOnCastor']['Directory']
if self.options.check:
_, name = tempfile.mkstemp('.txt', text=True)
testFile = file(name, 'w')
testFile.write('Test file')
testFile.close()
store = castortools.castorToLFN(dir)
#this is bad, but castortools is giving me problems
if not os.system('cmsStage %s %s' % (name, store)):
fname = '%s/%s' % (dir, os.path.basename(name))
write = castortools.fileExists(fname)
if write:
castortools.rm(fname)
else:
raise Exception("Failed to write to directory '%s'" % dir)
os.remove(name)
return {'Directory': dir, 'WriteAccess': True}
