def buildListOfBadFiles(self):
'''fills the list of bad files from the IntegrityCheck log.
When the integrity check file is not available,
files are considered as good.'''
mask = "IntegrityCheck"
self.bad_files = {}
self.good_files = []
file_mask = castortools.matchingFiles(self.castorDir, '^%s_.*\.txt$' % mask)
if file_mask:
from CMGTools.Production.edmIntegrityCheck import PublishToFileSystem
p = PublishToFileSystem(mask)
report = p.get(self.castorDir)
if report is not None and report:
self.maskExists = True
self.report = report
dup = report.get('ValidDuplicates',{})
for name, status in report['Files'].iteritems():
# print name, status
if not status[0]:
self.bad_files[name] = 'MarkedBad'
elif dup.has_key(name):
self.bad_files[name] = 'ValidDup'
else:
self.good_files.append( name )
else:
raise IntegrityCheckError( "ERROR: IntegrityCheck log file IntegrityCheck_XXXXXXXXXX.txt not found" )
def buildListOfBadFiles(self):
'''fills the list of bad files from the IntegrityCheck log.
When the integrity check file is not available,
files are considered as good.'''
mask = "IntegrityCheck"
self.bad_files = {}
self.good_files = []
file_mask = castortools.matchingFiles(self.castorDir,
'^%s_.*\.txt$' % mask)
if file_mask:
from CMGTools.Production.edmIntegrityCheck import PublishToFileSystem
p = PublishToFileSystem(mask)
report = p.get(self.castorDir)
if report is not None and report:
self.maskExists = True
self.report = report
dup = report.get('ValidDuplicates', {})
for name, status in report['Files'].iteritems():
# print name, status
if not status[0]:
self.bad_files[name] = 'MarkedBad'
elif dup.has_key(name):
self.bad_files[name] = 'ValidDup'
else:
self.good_files.append(name)
else:
raise IntegrityCheckError(
"ERROR: IntegrityCheck log file IntegrityCheck_XXXXXXXXXX.txt not found"
)
def run(self, input):
#skip for DBS
if self.user == 'CMS':
return {'MaskPresent':True,'Report':'Files taken from DBS'}
dir = input['FindOnCastor']['Directory']
mask = "IntegrityCheck"
file_mask = []
report = None
if (hasattr(self.options,'check') and self.options.check) or not hasattr(self.options,'check'):
file_mask = castortools.matchingFiles(dir, '^%s_.*\.txt$' % mask)
if file_mask:
from CMGTools.Production.edmIntegrityCheck import PublishToFileSystem
p = PublishToFileSystem(mask)
report = p.get(dir)
return {'MaskPresent':report is not None,'Report':report}
def run(self, input):
#skip for DBS
if self.user == 'CMS':
return {'MaskPresent':True,'Report':'Files taken from DBS'}
dir = input['FindOnCastor']['Directory']
mask = "IntegrityCheck"
file_mask = []
report = None
if (hasattr(self.options,'check') and self.options.check) or not hasattr(self.options,'check'):
file_mask = castortools.matchingFiles(dir, '^%s_.*\.txt$' % mask)
if file_mask:
from CMGTools.Production.edmIntegrityCheck import PublishToFileSystem
p = PublishToFileSystem(mask)
report = p.get(dir)
return {'MaskPresent':report is not None,'Report':report}
def work(d,op):
tokens = d.split('%')
if len(tokens) == 2:
op.user = tokens[0]
d = tokens[1]
check = IntegrityCheck(d,op)
pub = PublishToFileSystem(check)
previous = None
if op.update:
previous = pub.get(check.directory)
check.test(previous = previous, timeout = op.timeout)
if op.printout:
check.report()
report = check.structured()
pub.publish(report)
return d
def run(self, input):
report = None
if self.options.check and not input['CheckForMask']['MaskPresent']:
from CMGTools.Production.edmIntegrityCheck import IntegrityCheck, PublishToFileSystem
options = copy.deepcopy(self.options)
options.user = self.user
if input.has_key('BaseDataset'):
options.name = input['BaseDataset']['Name']
else:
options.name = None
check = IntegrityCheck(self.dataset,options)
check.test()
report = check.structured()
pub = PublishToFileSystem(check)
pub.publish(report)
elif input['CheckForMask']['MaskPresent']:
report = input['CheckForMask']['Report']
return {'MaskPresent':report is not None,'Report':report}
def run(self, input):
report = None
if self.options.check and not input['CheckForMask']['MaskPresent']:
from CMGTools.Production.edmIntegrityCheck import IntegrityCheck, PublishToFileSystem
options = copy.deepcopy(self.options)
options.user = self.user
if input.has_key('BaseDataset'):
options.name = input['BaseDataset']['Name']
else:
options.name = None
check = IntegrityCheck(self.dataset,options)
check.test()
report = check.structured()
pub = PublishToFileSystem(check)
pub.publish(report)
elif input['CheckForMask']['MaskPresent']:
report = input['CheckForMask']['Report']
return {'MaskPresent':report is not None,'Report':report}
dir = args[0]
regexp = args[1]
exists = castortools.fileExists(dir)
if not exists:
print 'sourceFileList: directory does not exist. Exiting'
sys.exit(1)
files = castortools.matchingFiles(dir, regexp)
mask = "IntegrityCheck"
file_mask = []
if options.check:
file_mask = castortools.matchingFiles(dir, '^%s_.*\.txt$' % mask)
bad_files = {}
if options.check and file_mask:
from CMGTools.Production.edmIntegrityCheck import PublishToFileSystem
p = PublishToFileSystem(mask)
report = p.get(dir)
if report is not None and report:
dup = report.get('ValidDuplicates', {})
for name, status in report['Files'].iteritems():
if not status[0]:
bad_files[name] = 'MarkedBad'
elif dup.has_key(name):
bad_files[name] = 'ValidDup'
from CMGTools.Production.sourceFileListCff import sourceFileListCff
print sourceFileListCff(files, bad_files)
exists = castortools.fileExists( dir )
if not exists:
print 'sourceFileList: directory does not exist. Exiting'
sys.exit(1)
files = castortools.matchingFiles( dir, regexp)
mask = "IntegrityCheck"
file_mask = []
if options.check:
file_mask = castortools.matchingFiles(dir, '^%s_.*\.txt$' % mask)
bad_files = {}
if options.check and file_mask:
from CMGTools.Production.edmIntegrityCheck import PublishToFileSystem
p = PublishToFileSystem(mask)
report = p.get(dir)
if report is not None and report:
dup = report.get('ValidDuplicates',{})
for name, status in report['Files'].iteritems():
if not status[0]:
bad_files[name] = 'MarkedBad'
elif dup.has_key(name):
bad_files[name] = 'ValidDup'
from CMGTools.Production.sourceFileListCff import sourceFileListCff
print sourceFileListCff( files, bad_files)