示例#1
0
    def buildListOfBadFiles(self):
        '''fills the list of bad files from the IntegrityCheck log.

        When the integrity check file is not available,
        files are considered as good.'''
        mask = "IntegrityCheck"
           
        self.bad_files = {}
        self.good_files = []

        file_mask = castortools.matchingFiles(self.castorDir, '^%s_.*\.txt$' % mask)
        if file_mask:
            from CMGTools.Production.edmIntegrityCheck import PublishToFileSystem
            p = PublishToFileSystem(mask)
            report = p.get(self.castorDir)
            if report is not None and report:
                self.maskExists = True
                self.report = report
                dup = report.get('ValidDuplicates',{})
                for name, status in report['Files'].iteritems():
                    # print name, status
                    if not status[0]:
                        self.bad_files[name] = 'MarkedBad'
                    elif dup.has_key(name):
                        self.bad_files[name] = 'ValidDup'
                    else:
                        self.good_files.append( name )
        else:
            raise IntegrityCheckError( "ERROR: IntegrityCheck log file IntegrityCheck_XXXXXXXXXX.txt not found" )
示例#2
0
    def buildListOfBadFiles(self):
        '''fills the list of bad files from the IntegrityCheck log.

        When the integrity check file is not available,
        files are considered as good.'''
        mask = "IntegrityCheck"

        self.bad_files = {}
        self.good_files = []

        file_mask = castortools.matchingFiles(self.castorDir,
                                              '^%s_.*\.txt$' % mask)
        if file_mask:
            from CMGTools.Production.edmIntegrityCheck import PublishToFileSystem
            p = PublishToFileSystem(mask)
            report = p.get(self.castorDir)
            if report is not None and report:
                self.maskExists = True
                self.report = report
                dup = report.get('ValidDuplicates', {})
                for name, status in report['Files'].iteritems():
                    # print name, status
                    if not status[0]:
                        self.bad_files[name] = 'MarkedBad'
                    elif dup.has_key(name):
                        self.bad_files[name] = 'ValidDup'
                    else:
                        self.good_files.append(name)
        else:
            raise IntegrityCheckError(
                "ERROR: IntegrityCheck log file IntegrityCheck_XXXXXXXXXX.txt not found"
            )
示例#3
0
    def run(self, input):
        #skip for DBS
        if self.user == 'CMS':
            return {'MaskPresent':True,'Report':'Files taken from DBS'}
        
        dir = input['FindOnCastor']['Directory']
        mask = "IntegrityCheck"
        file_mask = []  

        report = None
        if (hasattr(self.options,'check') and self.options.check) or not hasattr(self.options,'check'):
            file_mask = castortools.matchingFiles(dir, '^%s_.*\.txt$' % mask)

            if file_mask:
                from CMGTools.Production.edmIntegrityCheck import PublishToFileSystem
                p = PublishToFileSystem(mask)
                report = p.get(dir)
        return {'MaskPresent':report is not None,'Report':report}
示例#4
0
    def run(self, input):
        #skip for DBS
        if self.user == 'CMS':
            return {'MaskPresent':True,'Report':'Files taken from DBS'}
        
        dir = input['FindOnCastor']['Directory']
        mask = "IntegrityCheck"
        file_mask = []  

        report = None
        if (hasattr(self.options,'check') and self.options.check) or not hasattr(self.options,'check'):
            file_mask = castortools.matchingFiles(dir, '^%s_.*\.txt$' % mask)

            if file_mask:
                from CMGTools.Production.edmIntegrityCheck import PublishToFileSystem
                p = PublishToFileSystem(mask)
                report = p.get(dir)
        return {'MaskPresent':report is not None,'Report':report}
示例#5
0
    def work(d,op):
        tokens = d.split('%')
        if len(tokens) == 2:
            op.user = tokens[0]
            d = tokens[1]
        
        check = IntegrityCheck(d,op)
        pub = PublishToFileSystem(check)

        previous = None
        if op.update:
            previous = pub.get(check.directory)

        check.test(previous = previous, timeout = op.timeout)
        if op.printout:
            check.report()
        report = check.structured()
        pub.publish(report)

        return d
示例#6
0
    def run(self, input):
        
        report = None
        if self.options.check and not input['CheckForMask']['MaskPresent']:
            from CMGTools.Production.edmIntegrityCheck import IntegrityCheck, PublishToFileSystem
            
            options = copy.deepcopy(self.options)
            options.user = self.user

            if input.has_key('BaseDataset'):
                options.name = input['BaseDataset']['Name']
            else:
                options.name = None
            
            check = IntegrityCheck(self.dataset,options)
            check.test()
            report = check.structured()
            pub = PublishToFileSystem(check)
            pub.publish(report)
        elif input['CheckForMask']['MaskPresent']:
            report = input['CheckForMask']['Report']
        
        return {'MaskPresent':report is not None,'Report':report}
示例#7
0
    def run(self, input):
        
        report = None
        if self.options.check and not input['CheckForMask']['MaskPresent']:
            from CMGTools.Production.edmIntegrityCheck import IntegrityCheck, PublishToFileSystem
            
            options = copy.deepcopy(self.options)
            options.user = self.user

            if input.has_key('BaseDataset'):
                options.name = input['BaseDataset']['Name']
            else:
                options.name = None
            
            check = IntegrityCheck(self.dataset,options)
            check.test()
            report = check.structured()
            pub = PublishToFileSystem(check)
            pub.publish(report)
        elif input['CheckForMask']['MaskPresent']:
            report = input['CheckForMask']['Report']
        
        return {'MaskPresent':report is not None,'Report':report}
示例#8
0
dir = args[0]
regexp = args[1]

exists = castortools.fileExists(dir)
if not exists:
    print 'sourceFileList: directory does not exist. Exiting'
    sys.exit(1)

files = castortools.matchingFiles(dir, regexp)

mask = "IntegrityCheck"
file_mask = []
if options.check:
    file_mask = castortools.matchingFiles(dir, '^%s_.*\.txt$' % mask)
bad_files = {}
if options.check and file_mask:
    from CMGTools.Production.edmIntegrityCheck import PublishToFileSystem
    p = PublishToFileSystem(mask)
    report = p.get(dir)
    if report is not None and report:
        dup = report.get('ValidDuplicates', {})
        for name, status in report['Files'].iteritems():
            if not status[0]:
                bad_files[name] = 'MarkedBad'
            elif dup.has_key(name):
                bad_files[name] = 'ValidDup'

from CMGTools.Production.sourceFileListCff import sourceFileListCff

print sourceFileListCff(files, bad_files)
示例#9
0
exists = castortools.fileExists( dir )
if not exists:
    print 'sourceFileList: directory does not exist. Exiting'
    sys.exit(1)


files = castortools.matchingFiles( dir, regexp)

mask = "IntegrityCheck"
file_mask = []  
if options.check:
    file_mask = castortools.matchingFiles(dir, '^%s_.*\.txt$' % mask)
bad_files = {}    
if options.check and file_mask:
    from CMGTools.Production.edmIntegrityCheck import PublishToFileSystem
    p = PublishToFileSystem(mask)
    report = p.get(dir)
    if report is not None and report:
        dup = report.get('ValidDuplicates',{})
        for name, status in report['Files'].iteritems():
            if not status[0]:
                bad_files[name] = 'MarkedBad'
            elif dup.has_key(name):
                bad_files[name] = 'ValidDup'


from CMGTools.Production.sourceFileListCff import sourceFileListCff
print sourceFileListCff( files, bad_files)