def publish(self, report): """Publish a file""" for path in report['PathList']: _, name = tempfile.mkstemp('.txt', text=True) json.dump(report, file(name,'w'), sort_keys=True, indent=4) fname = '%s_%s.txt' % (self.parent, report['DateCreated']) #rename the file locally - TODO: This is a potential problem nname = os.path.join(os.path.dirname(name),fname) os.rename(name, nname) castor_path = castortools.lfnToCastor(path) new_name = '%s/%s' % (castor_path, fname) castortools.xrdcp(nname,path) time.sleep(1) if castortools.fileExists(new_name): #castortools.move(old_name, new_name) #castortools.chmod(new_name, '644') print "File published: '%s'" % castortools.castorToLFN(new_name) os.remove(nname) else: pathhash = path.replace('/','.') hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname) shutil.move(nname, hashed_name) print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % (castor_path, hashed_name)
def publish(self, report): """Publish a file""" for path in report['PathList']: _, name = tempfile.mkstemp('.txt', text=True) json.dump(report, file(name, 'w'), sort_keys=True, indent=4) fname = '%s_%s.txt' % (self.parent, report['DateCreated']) #rename the file locally - TODO: This is a potential problem nname = os.path.join(os.path.dirname(name), fname) os.rename(name, nname) castor_path = castortools.lfnToCastor(path) new_name = '%s/%s' % (castor_path, fname) castortools.xrdcp(nname, path) time.sleep(1) if castortools.fileExists(new_name): #castortools.move(old_name, new_name) #castortools.chmod(new_name, '644') print "File published: '%s'" % castortools.castorToLFN( new_name) os.remove(nname) else: pathhash = path.replace('/', '.') hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname) shutil.move(nname, hashed_name) print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % ( castor_path, hashed_name)
def test(self, previous=None, timeout=-1): if not castortools.fileExists(self.directory): raise Exception( "The top level directory '%s' for this dataset does not exist" % self.directory) self.query() test_results = {} #support updating to speed things up prev_results = {} if previous is not None: for name, status in previous['Files'].iteritems(): prev_results[name] = status filesToTest = self.sortByBaseDir(self.listRootFiles(self.directory)) for dir, filelist in filesToTest.iteritems(): filemask = {} #apply a UNIX wildcard if specified filtered = filelist if self.options.wildcard is not None: filtered = fnmatch.filter(filelist, self.options.wildcard) if not filtered: print >> sys.stderr, "Warning: The wildcard '%s' does not match any files in '%s'. Please check you are using quotes." % ( self.options.wildcard, self.directory) count = 0 for ff in filtered: fname = os.path.join(dir, ff) lfn = castortools.castorToLFN(fname) #try to update from the previous result if available if lfn in prev_results and prev_results[lfn][0]: if self.options.printout: print '[%i/%i]\t Skipping %s...' % ( count, len(filtered), fname), OK, num = prev_results[lfn] else: if self.options.printout: print '[%i/%i]\t Checking %s...' % ( count, len(filtered), fname), OK, num = self.testFileTimeOut(lfn, timeout) filemask[ff] = (OK, num) if self.options.printout: print(OK, num) if OK: self.eventsSeen += num count += 1 test_results[castortools.castorToLFN(dir)] = filemask self.test_result = test_results self.duplicates, self.bad_jobs, sum_dup = self.stripDuplicates() #remove duplicate entries from the event count self.eventsSeen -= sum_dup
def test(self, previous=None, timeout=-1): if not castortools.fileExists(self.directory): raise Exception("The top level directory '%s' for this dataset does not exist" % self.directory) self.query() test_results = {} # support updating to speed things up prev_results = {} if previous is not None: for name, status in previous["Files"].iteritems(): prev_results[name] = status filesToTest = self.sortByBaseDir(self.listRootFiles(self.directory)) for dir, filelist in filesToTest.iteritems(): filemask = {} # apply a UNIX wildcard if specified filtered = filelist if self.options.wildcard is not None: filtered = fnmatch.filter(filelist, self.options.wildcard) if not filtered: print >> sys.stderr, "Warning: The wildcard '%s' does not match any files in '%s'. Please check you are using quotes." % ( self.options.wildcard, self.directory, ) count = 0 for ff in filtered: fname = os.path.join(dir, ff) lfn = castortools.castorToLFN(fname) # try to update from the previous result if available if lfn in prev_results and prev_results[lfn][0]: if self.options.printout: print "[%i/%i]\t Skipping %s..." % (count, len(filtered), fname), OK, num = prev_results[lfn] else: if self.options.printout: print "[%i/%i]\t Checking %s..." % (count, len(filtered), fname), OK, num = self.testFileTimeOut(lfn, timeout) filemask[ff] = (OK, num) if self.options.printout: print (OK, num) if OK: self.eventsSeen += num count += 1 test_results[castortools.castorToLFN(dir)] = filemask self.test_result = test_results self.duplicates, self.bad_jobs, sum_dup = self.stripDuplicates() # remove duplicate entries from the event count self.eventsSeen -= sum_dup
def run(self, input): if self.user == 'CMS': return {'Topdir':None,'Directory':None} topdir = castortools.lfnToCastor(castorBaseDir(user=self.user)) directory = '%s/%s' % (topdir,self.dataset) # directory = directory.replace('//','/') if not castortools.fileExists(directory): if hasattr(self,'create') and self.create: castortools.createCastorDir(directory) #castortools.chmod(directory,'775') if not castortools.isDirectory(directory): raise Exception("Dataset directory '%s' does not exist or could not be created" % directory) return {'Topdir':topdir,'Directory':directory}
def run(self, input): """Check that the directory is writable""" if self.user == 'CMS': return {'Directory':None,'WriteAccess':True} dir = input['FindOnCastor']['Directory'] if self.options.check: _, name = tempfile.mkstemp('.txt',text=True) testFile = file(name,'w') testFile.write('Test file') testFile.close() store = castortools.castorToLFN(dir) #this is bad, but castortools is giving me problems if not os.system('cmsStage %s %s' % (name,store)): fname = '%s/%s' % (dir,os.path.basename(name)) write = castortools.fileExists(fname) if write: castortools.rm(fname) else: raise Exception("Failed to write to directory '%s'" % dir) os.remove(name) return {'Directory':dir,'WriteAccess':True}
def run(self, input): """Check that the directory is writable""" if self.user == 'CMS': return {'Directory': None, 'WriteAccess': True} dir = input['FindOnCastor']['Directory'] if self.options.check: _, name = tempfile.mkstemp('.txt', text=True) testFile = file(name, 'w') testFile.write('Test file') testFile.close() store = castortools.castorToLFN(dir) #this is bad, but castortools is giving me problems if not os.system('cmsStage %s %s' % (name, store)): fname = '%s/%s' % (dir, os.path.basename(name)) write = castortools.fileExists(fname) if write: castortools.rm(fname) else: raise Exception("Failed to write to directory '%s'" % dir) os.remove(name) return {'Directory': dir, 'WriteAccess': True}