def buildListOfBadFiles(self): '''fills the list of bad files from the IntegrityCheck log. When the integrity check file is not available, files are considered as good.''' mask = "IntegrityCheck" self.bad_files = {} self.good_files = [] file_mask = castortools.matchingFiles(self.castorDir, '^%s_.*\.txt$' % mask) if file_mask: # here to avoid circular dependency from edmIntegrityCheck import PublishToFileSystem p = PublishToFileSystem(mask) report = p.get(self.castorDir) if report is not None and report: self.maskExists = True self.report = report dup = report.get('ValidDuplicates', {}) for name, status in report['Files'].iteritems(): # print name, status if not status[0]: self.bad_files[name] = 'MarkedBad' elif name in dup: self.bad_files[name] = 'ValidDup' else: self.good_files.append(name) else: raise IntegrityCheckError( "ERROR: IntegrityCheck log file IntegrityCheck_XXXXXXXXXX.txt not found" )
def buildListOfBadFiles(self): '''fills the list of bad files from the IntegrityCheck log. When the integrity check file is not available, files are considered as good.''' mask = "IntegrityCheck" self.bad_files = {} self.good_files = [] file_mask = castortools.matchingFiles(self.castorDir, '^%s_.*\.txt$' % mask) if file_mask: # here to avoid circular dependency from edmIntegrityCheck import PublishToFileSystem p = PublishToFileSystem(mask) report = p.get(self.castorDir) if report is not None and report: self.maskExists = True self.report = report dup = report.get('ValidDuplicates',{}) for name, status in report['Files'].iteritems(): # print name, status if not status[0]: self.bad_files[name] = 'MarkedBad' elif name in dup: self.bad_files[name] = 'ValidDup' else: self.good_files.append( name ) else: raise IntegrityCheckError( "ERROR: IntegrityCheck log file IntegrityCheck_XXXXXXXXXX.txt not found" )
def run(self, input): #skip for DBS if self.user == 'CMS': return {'MaskPresent':True,'Report':'Files taken from DBS'} dir = input['FindOnCastor']['Directory'] mask = "IntegrityCheck" file_mask = [] report = None if (hasattr(self.options,'check') and self.options.check) or not hasattr(self.options,'check'): file_mask = castortools.matchingFiles(dir, '^%s_.*\.txt$' % mask) if file_mask: p = PublishToFileSystem(mask) report = p.get(dir) return {'MaskPresent':report is not None,'Report':report}
def run(self, input): report = None if self.options.check and not input['CheckForMask']['MaskPresent']: options = copy.deepcopy(self.options) options.user = self.user if input.has_key('BaseDataset'): options.name = input['BaseDataset']['Name'] else: options.name = None check = IntegrityCheck(self.dataset,options) check.test() report = check.structured() pub = PublishToFileSystem(check) pub.publish(report) elif input['CheckForMask']['MaskPresent']: report = input['CheckForMask']['Report'] return {'MaskPresent':report is not None,'Report':report}
def run(self, input): report = None if self.options.check and not input['CheckForMask']['MaskPresent']: options = copy.deepcopy(self.options) options.user = self.user if input.has_key('BaseDataset'): options.name = input['BaseDataset']['Name'] else: options.name = None check = IntegrityCheck(self.dataset, options) check.test() report = check.structured() pub = PublishToFileSystem(check) pub.publish(report) elif input['CheckForMask']['MaskPresent']: report = input['CheckForMask']['Report'] return {'MaskPresent': report is not None, 'Report': report}