def test(self, previous=None, timeout=-1): if not castortools.fileExists(self.directory): raise Exception( "The top level directory '%s' for this dataset does not exist" % self.directory) self.query() test_results = {} #support updating to speed things up prev_results = {} if previous is not None: for name, status in previous['Files'].iteritems(): prev_results[name] = status filesToTest = self.sortByBaseDir(self.listRootFiles(self.directory)) for dir, filelist in filesToTest.iteritems(): filemask = {} #apply a UNIX wildcard if specified filtered = filelist if self.options.wildcard is not None: filtered = fnmatch.filter(filelist, self.options.wildcard) if not filtered: print >> sys.stderr, "Warning: The wildcard '%s' does not match any files in '%s'. Please check you are using quotes." % ( self.options.wildcard, self.directory) count = 0 for ff in filtered: fname = os.path.join(dir, ff) lfn = castortools.castorToLFN(fname) #try to update from the previous result if available if lfn in prev_results and prev_results[lfn][0]: if self.options.printout: print '[%i/%i]\t Skipping %s...' % ( count, len(filtered), fname), OK, num = prev_results[lfn] else: if self.options.printout: print '[%i/%i]\t Checking %s...' % ( count, len(filtered), fname), OK, num = self.testFileTimeOut(lfn, timeout) filemask[ff] = (OK, num) if self.options.printout: print(OK, num) if OK: self.eventsSeen += num count += 1 test_results[castortools.castorToLFN(dir)] = filemask self.test_result = test_results self.duplicates, self.bad_jobs, sum_dup = self.stripDuplicates() #remove duplicate entries from the event count self.eventsSeen -= sum_dup
def test(self, previous=None, timeout=-1): if not castortools.fileExists(self.directory): raise Exception("The top level directory '%s' for this dataset does not exist" % self.directory) self.query() test_results = {} # support updating to speed things up prev_results = {} if previous is not None: for name, status in previous["Files"].iteritems(): prev_results[name] = status filesToTest = self.sortByBaseDir(self.listRootFiles(self.directory)) for dir, filelist in filesToTest.iteritems(): filemask = {} # apply a UNIX wildcard if specified filtered = filelist if self.options.wildcard is not None: filtered = fnmatch.filter(filelist, self.options.wildcard) if not filtered: print >> sys.stderr, "Warning: The wildcard '%s' does not match any files in '%s'. Please check you are using quotes." % ( self.options.wildcard, self.directory, ) count = 0 for ff in filtered: fname = os.path.join(dir, ff) lfn = castortools.castorToLFN(fname) # try to update from the previous result if available if lfn in prev_results and prev_results[lfn][0]: if self.options.printout: print "[%i/%i]\t Skipping %s..." % (count, len(filtered), fname), OK, num = prev_results[lfn] else: if self.options.printout: print "[%i/%i]\t Checking %s..." % (count, len(filtered), fname), OK, num = self.testFileTimeOut(lfn, timeout) filemask[ff] = (OK, num) if self.options.printout: print (OK, num) if OK: self.eventsSeen += num count += 1 test_results[castortools.castorToLFN(dir)] = filemask self.test_result = test_results self.duplicates, self.bad_jobs, sum_dup = self.stripDuplicates() # remove duplicate entries from the event count self.eventsSeen -= sum_dup
def publish(self, report): """Publish a file""" for path in report['PathList']: _, name = tempfile.mkstemp('.txt', text=True) json.dump(report, file(name,'w'), sort_keys=True, indent=4) fname = '%s_%s.txt' % (self.parent, report['DateCreated']) #rename the file locally - TODO: This is a potential problem nname = os.path.join(os.path.dirname(name),fname) os.rename(name, nname) castor_path = castortools.lfnToCastor(path) new_name = '%s/%s' % (castor_path, fname) castortools.xrdcp(nname,path) time.sleep(1) if castortools.fileExists(new_name): #castortools.move(old_name, new_name) #castortools.chmod(new_name, '644') print "File published: '%s'" % castortools.castorToLFN(new_name) os.remove(nname) else: pathhash = path.replace('/','.') hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname) shutil.move(nname, hashed_name) print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % (castor_path, hashed_name)
def publish(self, report): """Publish a file""" for path in report['PathList']: _, name = tempfile.mkstemp('.txt', text=True) json.dump(report, file(name, 'w'), sort_keys=True, indent=4) fname = '%s_%s.txt' % (self.parent, report['DateCreated']) #rename the file locally - TODO: This is a potential problem nname = os.path.join(os.path.dirname(name), fname) os.rename(name, nname) castor_path = castortools.lfnToCastor(path) new_name = '%s/%s' % (castor_path, fname) castortools.xrdcp(nname, path) time.sleep(1) if castortools.fileExists(new_name): #castortools.move(old_name, new_name) #castortools.chmod(new_name, '644') print "File published: '%s'" % castortools.castorToLFN( new_name) os.remove(nname) else: pathhash = path.replace('/', '.') hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname) shutil.move(nname, hashed_name) print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % ( castor_path, hashed_name)
def run(self, input): find = FindOnCastor(self.dataset, self.options.batch_user, self.options) find.create = True out = find.run({}) full = input['ExpandConfig']['ExpandedFullCFG'] jobdir = input['CreateJobDirectory']['JobDir'] sampleDir = os.path.join(out['Directory'], self.options.tier) sampleDir = castortools.castorToLFN(sampleDir) cmd = [ 'cmsBatch.py', str(self.options.nInput), os.path.basename(full), '-o', '%s_Jobs' % self.options.tier, '--force' ] cmd.extend(['-r', sampleDir]) if self.options.run_batch: jname = "%s/%s" % (self.dataset, self.options.tier) jname = jname.replace("//", "/") user_group = '' if self.options.group is not None: user_group = '-G %s' % self.options.group cmd.extend([ '-b', "'bsub -q %s -J %s -u [email protected] %s < ./batchScript.sh | tee job_id.txt'" % (self.options.queue, jname, user_group) ]) print " ".join(cmd) pwd = os.getcwd() error = None try: os.chdir(jobdir) returncode = os.system(" ".join(cmd)) if returncode != 0: error = "Running cmsBatch failed. Return code was %i." % returncode finally: os.chdir(pwd) if error is not None: raise Exception(error) return { 'SampleDataset': "%s/%s" % (self.dataset, self.options.tier), 'BatchUser': self.options.batch_user, 'SampleOutputDir': sampleDir, 'LSFJobsTopDir': os.path.join(jobdir, '%s_Jobs' % self.options.tier) }
def isTgzDirOnEOS(self, file): '''Checks if file is a .tgz file in an eos dir''' if not castortools.isCastorDir(file): file = castortools.castorToLFN(file) if castortools.isLFN(file): tgzPattern = re.compile('.*\.tgz$') m = tgzPattern.match(file) if m: return True else: return False else: return False
def isTgzDirOnEOS(self, file ): '''Checks if file is a .tgz file in an eos dir''' if not castortools.isCastorDir( file ): file = castortools.castorToLFN(file) if castortools.isLFN( file ): tgzPattern = re.compile('.*\.tgz$') m = tgzPattern.match( file ) if m: return True else: return False else: return False
def run(self, input): find = FindOnCastor(self.dataset,self.options.batch_user,self.options) find.create = True out = find.run({}) full = input['ExpandConfig']['ExpandedFullCFG'] jobdir = input['CreateJobDirectory']['JobDir'] sampleDir = os.path.join(out['Directory'],self.options.tier) sampleDir = castortools.castorToLFN(sampleDir) cmd = ['cmsBatch.py',str(self.options.nInput),os.path.basename(full),'-o','%s_Jobs' % self.options.tier,'--force'] cmd.extend(['-r',sampleDir]) if self.options.run_batch: jname = "%s/%s" % (self.dataset,self.options.tier) jname = jname.replace("//","/") user_group = '' if self.options.group is not None: user_group = '-G %s' % self.options.group cmd.extend(['-b',"'bsub -q %s -J %s -u [email protected] %s < ./batchScript.sh | tee job_id.txt'" % (self.options.queue,jname,user_group)]) print " ".join(cmd) pwd = os.getcwd() error = None try: os.chdir(jobdir) returncode = os.system(" ".join(cmd)) if returncode != 0: error = "Running cmsBatch failed. Return code was %i." % returncode finally: os.chdir(pwd) if error is not None: raise Exception(error) return {'SampleDataset':"%s/%s" % (self.dataset,self.options.tier),'BatchUser':self.options.batch_user, 'SampleOutputDir':sampleDir,'LSFJobsTopDir':os.path.join(jobdir,'%s_Jobs' % self.options.tier)}
def run(self, input): """Check that the directory is writable""" if self.user == 'CMS': return {'Directory':None,'WriteAccess':True} dir = input['FindOnCastor']['Directory'] if self.options.check: _, name = tempfile.mkstemp('.txt',text=True) testFile = file(name,'w') testFile.write('Test file') testFile.close() store = castortools.castorToLFN(dir) #this is bad, but castortools is giving me problems if not os.system('cmsStage %s %s' % (name,store)): fname = '%s/%s' % (dir,os.path.basename(name)) write = castortools.fileExists(fname) if write: castortools.rm(fname) else: raise Exception("Failed to write to directory '%s'" % dir) os.remove(name) return {'Directory':dir,'WriteAccess':True}
def run(self, input): """Check that the directory is writable""" if self.user == 'CMS': return {'Directory': None, 'WriteAccess': True} dir = input['FindOnCastor']['Directory'] if self.options.check: _, name = tempfile.mkstemp('.txt', text=True) testFile = file(name, 'w') testFile.write('Test file') testFile.close() store = castortools.castorToLFN(dir) #this is bad, but castortools is giving me problems if not os.system('cmsStage %s %s' % (name, store)): fname = '%s/%s' % (dir, os.path.basename(name)) write = castortools.fileExists(fname) if write: castortools.rm(fname) else: raise Exception("Failed to write to directory '%s'" % dir) os.remove(name) return {'Directory': dir, 'WriteAccess': True}