def buildListOfBadFiles(self):
'''fills the list of bad files from the IntegrityCheck log.
When the integrity check file is not available,
files are considered as good.'''
mask = "IntegrityCheck"
self.bad_files = {}
self.good_files = []
file_mask = castortools.matchingFiles(self.castorDir,
'^%s_.*\.txt$' % mask)
if file_mask:
# here to avoid circular dependency
from edmIntegrityCheck import PublishToFileSystem
p = PublishToFileSystem(mask)
report = p.get(self.castorDir)
if report is not None and report:
self.maskExists = True
self.report = report
dup = report.get('ValidDuplicates', {})
for name, status in report['Files'].iteritems():
# print name, status
if not status[0]:
self.bad_files[name] = 'MarkedBad'
elif name in dup:
self.bad_files[name] = 'ValidDup'
else:
self.good_files.append(name)
else:
raise IntegrityCheckError(
"ERROR: IntegrityCheck log file IntegrityCheck_XXXXXXXXXX.txt not found"
)
def buildListOfBadFiles(self):
'''fills the list of bad files from the IntegrityCheck log.
When the integrity check file is not available,
files are considered as good.'''
mask = "IntegrityCheck"
self.bad_files = {}
self.good_files = []
file_mask = castortools.matchingFiles(self.castorDir, '^%s_.*\.txt$' % mask)
if file_mask:
# here to avoid circular dependency
from edmIntegrityCheck import PublishToFileSystem
p = PublishToFileSystem(mask)
report = p.get(self.castorDir)
if report is not None and report:
self.maskExists = True
self.report = report
dup = report.get('ValidDuplicates',{})
for name, status in report['Files'].iteritems():
# print name, status
if not status[0]:
self.bad_files[name] = 'MarkedBad'
elif name in dup:
self.bad_files[name] = 'ValidDup'
else:
self.good_files.append( name )
else:
raise IntegrityCheckError( "ERROR: IntegrityCheck log file IntegrityCheck_XXXXXXXXXX.txt not found" )
def run(self, input):
#skip for DBS
if self.user == 'CMS':
return {'MaskPresent':True,'Report':'Files taken from DBS'}
dir = input['FindOnCastor']['Directory']
mask = "IntegrityCheck"
file_mask = []
report = None
if (hasattr(self.options,'check') and self.options.check) or not hasattr(self.options,'check'):
file_mask = castortools.matchingFiles(dir, '^%s_.*\.txt$' % mask)
if file_mask:
p = PublishToFileSystem(mask)
report = p.get(dir)
return {'MaskPresent':report is not None,'Report':report}
def run(self, input):
#skip for DBS
if self.user == 'CMS':
return {'MaskPresent':True,'Report':'Files taken from DBS'}
dir = input['FindOnCastor']['Directory']
mask = "IntegrityCheck"
file_mask = []
report = None
if (hasattr(self.options,'check') and self.options.check) or not hasattr(self.options,'check'):
file_mask = castortools.matchingFiles(dir, '^%s_.*\.txt$' % mask)
if file_mask:
p = PublishToFileSystem(mask)
report = p.get(dir)
return {'MaskPresent':report is not None,'Report':report}
def run(self, input):
report = None
if self.options.check and not input['CheckForMask']['MaskPresent']:
options = copy.deepcopy(self.options)
options.user = self.user
if input.has_key('BaseDataset'):
options.name = input['BaseDataset']['Name']
else:
options.name = None
check = IntegrityCheck(self.dataset,options)
check.test()
report = check.structured()
pub = PublishToFileSystem(check)
pub.publish(report)
elif input['CheckForMask']['MaskPresent']:
report = input['CheckForMask']['Report']
return {'MaskPresent':report is not None,'Report':report}
def run(self, input):
report = None
if self.options.check and not input['CheckForMask']['MaskPresent']:
options = copy.deepcopy(self.options)
options.user = self.user
if input.has_key('BaseDataset'):
options.name = input['BaseDataset']['Name']
else:
options.name = None
check = IntegrityCheck(self.dataset, options)
check.test()
report = check.structured()
pub = PublishToFileSystem(check)
pub.publish(report)
elif input['CheckForMask']['MaskPresent']:
report = input['CheckForMask']['Report']
return {'MaskPresent': report is not None, 'Report': report}
