Exemplo n.º 1
0
def getDASData (query):
    results = []
    jsondict = das_client.get_data ('https://cmsweb.cern.ch', query, 0, 0, False)
    status = jsondict['status']
    if status != 'ok':
        print "DAS query status: %s"%(status)
        return results

    mongo_query = jsondict['mongo_query']
    filters = mongo_query['filters']
    data = jsondict['data']

    for row in data:
        result = [r for r in das_client.get_value (row, filters['grep'])][0]
        if len (result) > 0 and not result in results:
            results.append (result)

    return results
Exemplo n.º 2
0
def getFileNames (event):
    files = []
    # Query DAS
    query = "file dataset=%(dataset)s run=%(run)i lumi=%(lumi)i | grep file.name" % event
    jsondict = das_client.get_data('https://cmsweb.cern.ch', query, 0, 0, False)
    status = jsondict['status']
    if status != 'ok':
        print "DAS query status: %s"%(status)
        return files

    mongo_query = jsondict['mongo_query']
    filters = mongo_query['filters']
    data = jsondict['data']

    files = []
    for row in data:
        file = [r for r in das_client.get_value(row, filters['grep'])][0]
        if len(file) > 0 and not file in files:
            files.append(file)

    return files
Exemplo n.º 3
0
def getFileNames (event):
    files = []
    # Query DAS
    query = "file dataset=%(dataset)s run=%(run)i lumi=%(lumi)i | grep file.name" % event
    jsondict = das_client.get_data('https://cmsweb.cern.ch', query, 0, 0, False)
    status = jsondict['status']
    if status != 'ok':
        print "DAS query status: %s"%(status)
        return files

    mongo_query = jsondict['mongo_query']
    filters = mongo_query['filters']
    data = jsondict['data']

    files = []
    for row in data:
        file = [r for r in das_client.get_value(row, filters['grep'])][0]
        if len(file) > 0 and not file in files:
            files.append(file)

    return files
Exemplo n.º 4
0
def getFileNames(event):
    files = []
    # Query DAS
    query = "file dataset=%(dataset)s run=%(run)i lumi=%(lumi)i | grep file.name" % event
    jsondict = das_client.get_data("https://cmsweb.cern.ch", query, 0, 0, False)
    status = jsondict["status"]
    if status != "ok":
        print "DAS query status: %s" % (status)
        return files

    mongo_query = jsondict["mongo_query"]
    filters = mongo_query["filters"]
    data = jsondict["data"]

    files = []
    for row in data:
        file = [r for r in das_client.get_value(row, filters["grep"])][0]
        if len(file) > 0 and not file in files:
            files.append(file)

    return files
    def apply( self ):
        useDAS        = self._parameters[ 'useDAS'        ].value
        cmsswVersion  = self._parameters[ 'cmsswVersion'  ].value
        formerVersion = self._parameters[ 'formerVersion' ].value
        relVal        = self._parameters[ 'relVal'        ].value
        dataTier      = self._parameters[ 'dataTier'      ].value
        condition     = self._parameters[ 'condition'     ].value # only used for GT determination in initialization, if GT not explicitly given
        globalTag     = self._parameters[ 'globalTag'     ].value
        maxVersions   = self._parameters[ 'maxVersions'   ].value
        skipFiles     = self._parameters[ 'skipFiles'     ].value
        numberOfFiles = self._parameters[ 'numberOfFiles' ].value
        debug         = self._parameters[ 'debug'         ].value

        filePaths = []

        # Determine corresponding CMSSW version for RelVals
        preId      = '_pre'
        patchId    = '_patch'    # patch releases
        hltPatchId = '_hltpatch' # HLT patch releases
        dqmPatchId = '_dqmpatch' # DQM patch releases
        slhcId     = '_SLHC'     # SLHC releases
        rootId     = '_root'     # ROOT test releases
        ibId       = '_X_'       # IBs
        if patchId in cmsswVersion:
            cmsswVersion = cmsswVersion.split( patchId )[ 0 ]
        elif hltPatchId in cmsswVersion:
            cmsswVersion = cmsswVersion.split( hltPatchId )[ 0 ]
        elif dqmPatchId in cmsswVersion:
            cmsswVersion = cmsswVersion.split( dqmPatchId )[ 0 ]
        elif rootId in cmsswVersion:
            cmsswVersion = cmsswVersion.split( rootId )[ 0 ]
        elif slhcId in cmsswVersion:
            cmsswVersion = cmsswVersion.split( slhcId )[ 0 ]
        elif ibId in cmsswVersion or formerVersion:
            outputTuple = Popen( [ 'scram', 'l -c CMSSW' ], stdout = PIPE, stderr = PIPE ).communicate()
            if len( outputTuple[ 1 ] ) != 0:
                print '%s INFO : SCRAM error'%( self._label )
                if debug:
                    print '    from trying to determine last valid releases before \'%s\''%( cmsswVersion )
                    print
                    print outputTuple[ 1 ]
                    print
                    self.messageEmptyList()
                return filePaths
            versions = { 'last'      :''
                       , 'lastToLast':''
                       }
            for line in outputTuple[ 0 ].splitlines():
                version = line.split()[ 1 ]
                if cmsswVersion.split( ibId )[ 0 ] in version or cmsswVersion.rpartition( '_' )[ 0 ] in version:
                    if not ( patchId in version or hltPatchId in version or dqmPatchId in version or slhcId in version or ibId in version or rootId in version ):
                        versions[ 'lastToLast' ] = versions[ 'last' ]
                        versions[ 'last' ]       = version
                        if version == cmsswVersion:
                            break
            # FIXME: ordering of output problematic ('XYZ_pre10' before 'XYZ_pre2', no "formerVersion" for 'XYZ_pre1')
            if formerVersion:
                # Don't use pre-releases as "former version" for other releases than CMSSW_X_Y_0
                if preId in versions[ 'lastToLast' ] and not preId in versions[ 'last' ] and not versions[ 'last' ].endswith( '_0' ):
                    versions[ 'lastToLast' ] = versions[ 'lastToLast' ].split( preId )[ 0 ] # works only, if 'CMSSW_X_Y_0' esists ;-)
                # Use pre-release as "former version" for CMSSW_X_Y_0
                elif versions[ 'last' ].endswith( '_0' ) and not ( preId in versions[ 'lastToLast' ] and versions[ 'lastToLast' ].startswith( versions[ 'last' ] ) ):
                    versions[ 'lastToLast' ] = ''
                    for line in outputTuple[ 0 ].splitlines():
                        version      = line.split()[ 1 ]
                        versionParts = version.partition( preId )
                        if versionParts[ 0 ] == versions[ 'last' ] and versionParts[ 1 ] == preId:
                            versions[ 'lastToLast' ] = version
                        elif versions[ 'lastToLast' ] != '':
                            break
                # Don't use CMSSW_X_Y_0 as "former version" for pre-releases
                elif preId in versions[ 'last' ] and not preId in versions[ 'lastToLast' ] and versions[ 'lastToLast' ].endswith( '_0' ):
                    versions[ 'lastToLast' ] = '' # no alternative :-(
                cmsswVersion = versions[ 'lastToLast' ]
            else:
                cmsswVersion = versions[ 'last' ]

        # Debugging output
        if debug:
            print '%s DEBUG: Called with...'%( self._label )
            for key in self._parameters.keys():
               print '    %s:\t'%( key ),
               print self._parameters[ key ].value,
               if self._parameters[ key ].value is self.getDefaultParameters()[ key ].value:
                   print ' (default)'
               else:
                   print
               if key == 'cmsswVersion' and cmsswVersion != self._parameters[ key ].value:
                   if formerVersion:
                       print '    ==> modified to last to last valid release %s (s. \'formerVersion\' parameter)'%( cmsswVersion )
                   else:
                       print '    ==> modified to last valid release %s'%( cmsswVersion )

        # Check domain
        domain = socket.getfqdn().split( '.' )
        domainSE = ''
        if len( domain ) == 0:
            print '%s INFO : Cannot determine domain of this computer'%( self._label )
            if debug:
                self.messageEmptyList()
            return filePaths
        elif os.uname()[0] == "Darwin":
            print '%s INFO : Running on MacOSX without direct access to RelVal files.'%( self._label )
            if debug:
                self.messageEmptyList()
            return filePaths
        elif len( domain ) == 1:
            print '%s INFO : Running on local host \'%s\' without direct access to RelVal files'%( self._label, domain[ 0 ] )
            if debug:
                self.messageEmptyList()
            return filePaths
        if not ( ( domain[ -2 ] == 'cern' and domain[ -1 ] == 'ch' ) or ( domain[ -2 ] == 'fnal' and domain[ -1 ] == 'gov' ) ):
            print '%s INFO : Running on site \'%s.%s\' without direct access to RelVal files'%( self._label, domain[ -2 ], domain[ -1 ] )
            if debug:
                self.messageEmptyList()
            return filePaths
        if domain[ -2 ] == 'cern':
            domainSE = 'T2_CH_CERN'
        elif domain[ -2 ] == 'fnal':
            domainSE = 'T1_US_FNAL_MSS'
        if debug:
            print '%s DEBUG: Running at site \'%s.%s\''%( self._label, domain[ -2 ], domain[ -1 ] )
            print '%s DEBUG: Looking for SE \'%s\''%( self._label, domainSE )

        # Find files
        validVersion = 0
        dataset    = ''
        datasetAll = '/%s/%s-%s-v*/%s'%( relVal, cmsswVersion, globalTag, dataTier )
        if useDAS:
            if debug:
                print '%s DEBUG: Using DAS query'%( self._label )
            dasLimit = numberOfFiles
            if dasLimit <= 0:
                dasLimit += 1
            for version in range( maxVersions, 0, -1 ):
                filePaths    = []
                filePathsTmp = []
                fileCount    = 0
                dataset = '/%s/%s-%s-v%i/%s'%( relVal, cmsswVersion, globalTag, version, dataTier )
                dasQuery = 'file dataset=%s | grep file.name'%( dataset )
                if debug:
                    print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
                    print '    \'%s\''%( dasQuery )
                # partially stolen from das_client.py for option '--format=plain', needs filter ("grep") in the query
                dasData     = das_client.get_data( 'https://cmsweb.cern.ch', dasQuery, 0, dasLimit, False )
                jsondict    = json.loads( dasData )
                if debug:
                    print '%s DEBUG: Received DAS data:'%( self._label )
                    print '    \'%s\''%( dasData )
                    print '%s DEBUG: Determined JSON dictionary:'%( self._label )
                    print '    \'%s\''%( jsondict )
                if jsondict[ 'status' ] != 'ok':
                    print 'There was a problem while querying DAS with query \'%s\'. Server reply was:\n %s' % (dasQuery, dasData)
                    exit( 1 )
                mongo_query = jsondict[ 'mongo_query' ]
                filters     = mongo_query[ 'filters' ]
                data        = jsondict[ 'data' ]
                if debug:
                    print '%s DEBUG: Query in JSON dictionary:'%( self._label )
                    print '    \'%s\''%( mongo_query )
                    print '%s DEBUG: Filters in query:'%( self._label )
                    print '    \'%s\''%( filters )
                    print '%s DEBUG: Data in JSON dictionary:'%( self._label )
                    print '    \'%s\''%( data )
                for row in data:
                    filePath = [ r for r in das_client.get_value( row, filters ) ][ 0 ]
                    if debug:
                        print '%s DEBUG: Testing file entry \'%s\''%( self._label, filePath )
                    if len( filePath ) > 0:
                        if validVersion != version:
                            dasTest         = das_client.get_data( 'https://cmsweb.cern.ch', 'site dataset=%s | grep site.name'%( dataset ), 0, 999, False )
                            jsontestdict    = json.loads( dasTest )
                            mongo_testquery = jsontestdict[ 'mongo_query' ]
                            testfilters = mongo_testquery[ 'filters' ]
                            testdata    = jsontestdict[ 'data' ]
                            if debug:
                                print '%s DEBUG: Received DAS data (site test):'%( self._label )
                                print '    \'%s\''%( dasTest )
                                print '%s DEBUG: Determined JSON dictionary (site test):'%( self._label )
                                print '    \'%s\''%( jsontestdict )
                                print '%s DEBUG: Query in JSON dictionary (site test):'%( self._label )
                                print '    \'%s\''%( mongo_testquery )
                                print '%s DEBUG: Filters in query (site test):'%( self._label )
                                print '    \'%s\''%( testfilters )
                                print '%s DEBUG: Data in JSON dictionary (site test):'%( self._label )
                                print '    \'%s\''%( testdata )
                            foundSE = False
                            for testrow in testdata:
                                siteName = [ tr for tr in das_client.get_value( testrow, testfilters ) ][ 0 ]
                                if siteName == domainSE:
                                    foundSE = True
                                    break
                            if not foundSE:
                                if debug:
                                    print '%s DEBUG: Possible version \'v%s\' not available on SE \'%s\''%( self._label, version, domainSE )
                                break
                            validVersion = version
                            if debug:
                                print '%s DEBUG: Valid version set to \'v%i\''%( self._label, validVersion )
                        if numberOfFiles == 0:
                            break
                        # protect from double entries ( 'unique' flag in query does not work here)
                        if not filePath in filePathsTmp:
                            filePathsTmp.append( filePath )
                            if debug:
                                print '%s DEBUG: File \'%s\' found'%( self._label, filePath )
                            fileCount += 1
                            # needed, since and "limit" overrides "idx" in 'get_data' (==> "idx" set to '0' rather than "skipFiles")
                            if fileCount > skipFiles:
                                filePaths.append( filePath )
                        elif debug:
                            print '%s DEBUG: File \'%s\' found again'%( self._label, filePath )
                if validVersion > 0:
                    if numberOfFiles == 0 and debug:
                        print '%s DEBUG: No files requested'%( self._label )
                    break
        else:
            if debug:
                print '%s DEBUG: Using DBS query'%( self._label )
            for version in range( maxVersions, 0, -1 ):
                filePaths = []
                fileCount = 0
                dataset = '/%s/%s-%s-v%i/%s'%( relVal, cmsswVersion, globalTag, version, dataTier )
                dbsQuery = 'find file where dataset = %s'%( dataset )
                if debug:
                    print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
                    print '    \'%s\''%( dbsQuery )
                foundSE = False
                for line in os.popen( 'dbs search --query="%s"'%( dbsQuery ) ):
                    if line.find( '.root' ) != -1:
                        if validVersion != version:
                            if not foundSE:
                                dbsSiteQuery = 'find dataset where dataset = %s and site = %s'%( dataset, domainSE )
                                if debug:
                                    print '%s DEBUG: Querying site \'%s\' with'%( self._label, domainSE )
                                    print '    \'%s\''%( dbsSiteQuery )
                                for lineSite in os.popen( 'dbs search --query="%s"'%( dbsSiteQuery ) ):
                                    if lineSite.find( dataset ) != -1:
                                        foundSE = True
                                        break
                            if not foundSE:
                                if debug:
                                    print '%s DEBUG: Possible version \'v%s\' not available on SE \'%s\''%( self._label, version, domainSE )
                                break
                            validVersion = version
                            if debug:
                                print '%s DEBUG: Valid version set to \'v%i\''%( self._label, validVersion )
                        if numberOfFiles == 0:
                            break
                        filePath = line.replace( '\n', '' )
                        if debug:
                            print '%s DEBUG: File \'%s\' found'%( self._label, filePath )
                        fileCount += 1
                        if fileCount > skipFiles:
                            filePaths.append( filePath )
                        if not numberOfFiles < 0:
                            if numberOfFiles <= len( filePaths ):
                                break
                if validVersion > 0:
                    if numberOfFiles == 0 and debug:
                        print '%s DEBUG: No files requested'%( self._label )
                    break

        # Check output and return
        if validVersion == 0:
            print '%s INFO : No RelVal file(s) found at all in datasets \'%s*\' on SE \'%s\''%( self._label, datasetAll, domainSE )
            if debug:
                self.messageEmptyList()
        elif len( filePaths ) == 0:
            print '%s INFO : No RelVal file(s) picked up in dataset \'%s\''%( self._label, dataset )
            if debug:
                self.messageEmptyList()
        elif len( filePaths ) < numberOfFiles:
            print '%s INFO : Only %i RelVal file(s) instead of %i picked up in dataset \'%s\''%( self._label, len( filePaths ), numberOfFiles, dataset )

        if debug:
            print '%s DEBUG: returning %i file(s):\n%s'%( self._label, len( filePaths ), filePaths )
        return filePaths
Exemplo n.º 6
0
    def apply( self ):
        useDAS        = self._parameters[ 'useDAS'        ].value
        cmsswVersion  = self._parameters[ 'cmsswVersion'  ].value
        formerVersion = self._parameters[ 'formerVersion' ].value
        relVal        = self._parameters[ 'relVal'        ].value
        dataTier      = self._parameters[ 'dataTier'      ].value
        condition     = self._parameters[ 'condition'     ].value # only used for GT determination in initialization, if GT not explicitly given
        globalTag     = self._parameters[ 'globalTag'     ].value
        maxVersions   = self._parameters[ 'maxVersions'   ].value
        skipFiles     = self._parameters[ 'skipFiles'     ].value
        numberOfFiles = self._parameters[ 'numberOfFiles' ].value
        debug         = self._parameters[ 'debug'         ].value

        filePaths = []

        # Determine corresponding CMSSW version for RelVals
        preId      = '_pre'
        patchId    = '_patch'    # patch releases
        hltPatchId = '_hltpatch' # HLT patch releases
        dqmPatchId = '_dqmpatch' # DQM patch releases
        slhcId     = '_SLHC'     # SLHC releases
        rootId     = '_root'     # ROOT test releases
        ibId       = '_X_'       # IBs
        if patchId in cmsswVersion:
            cmsswVersion = cmsswVersion.split( patchId )[ 0 ]
        elif hltPatchId in cmsswVersion:
            cmsswVersion = cmsswVersion.split( hltPatchId )[ 0 ]
        elif dqmPatchId in cmsswVersion:
            cmsswVersion = cmsswVersion.split( dqmPatchId )[ 0 ]
        elif rootId in cmsswVersion:
            cmsswVersion = cmsswVersion.split( rootId )[ 0 ]
        elif slhcId in cmsswVersion:
            cmsswVersion = cmsswVersion.split( slhcId )[ 0 ]
        elif ibId in cmsswVersion or formerVersion:
            outputTuple = Popen( [ 'scram', 'l -c CMSSW' ], stdout = PIPE, stderr = PIPE ).communicate()
            if len( outputTuple[ 1 ] ) != 0:
                print '%s INFO : SCRAM error'%( self._label )
                if debug:
                    print '    from trying to determine last valid releases before \'%s\''%( cmsswVersion )
                    print
                    print outputTuple[ 1 ]
                    print
                    self.messageEmptyList()
                return filePaths
            versions = { 'last'      :''
                       , 'lastToLast':''
                       }
            for line in outputTuple[ 0 ].splitlines():
                version = line.split()[ 1 ]
                if cmsswVersion.split( ibId )[ 0 ] in version or cmsswVersion.rpartition( '_' )[ 0 ] in version:
                    if not ( patchId in version or hltPatchId in version or dqmPatchId in version or slhcId in version or ibId in version or rootId in version ):
                        versions[ 'lastToLast' ] = versions[ 'last' ]
                        versions[ 'last' ]       = version
                        if version == cmsswVersion:
                            break
            # FIXME: ordering of output problematic ('XYZ_pre10' before 'XYZ_pre2', no "formerVersion" for 'XYZ_pre1')
            if formerVersion:
                # Don't use pre-releases as "former version" for other releases than CMSSW_X_Y_0
                if preId in versions[ 'lastToLast' ] and not preId in versions[ 'last' ] and not versions[ 'last' ].endswith( '_0' ):
                    versions[ 'lastToLast' ] = versions[ 'lastToLast' ].split( preId )[ 0 ] # works only, if 'CMSSW_X_Y_0' esists ;-)
                # Use pre-release as "former version" for CMSSW_X_Y_0
                elif versions[ 'last' ].endswith( '_0' ) and not ( preId in versions[ 'lastToLast' ] and versions[ 'lastToLast' ].startswith( versions[ 'last' ] ) ):
                    versions[ 'lastToLast' ] = ''
                    for line in outputTuple[ 0 ].splitlines():
                        version      = line.split()[ 1 ]
                        versionParts = version.partition( preId )
                        if versionParts[ 0 ] == versions[ 'last' ] and versionParts[ 1 ] == preId:
                            versions[ 'lastToLast' ] = version
                        elif versions[ 'lastToLast' ] != '':
                            break
                # Don't use CMSSW_X_Y_0 as "former version" for pre-releases
                elif preId in versions[ 'last' ] and not preId in versions[ 'lastToLast' ] and versions[ 'lastToLast' ].endswith( '_0' ):
                    versions[ 'lastToLast' ] = '' # no alternative :-(
                cmsswVersion = versions[ 'lastToLast' ]
            else:
                cmsswVersion = versions[ 'last' ]

        # Debugging output
        if debug:
            print '%s DEBUG: Called with...'%( self._label )
            for key in self._parameters.keys():
               print '    %s:\t'%( key ),
               print self._parameters[ key ].value,
               if self._parameters[ key ].value is self.getDefaultParameters()[ key ].value:
                   print ' (default)'
               else:
                   print
               if key == 'cmsswVersion' and cmsswVersion != self._parameters[ key ].value:
                   if formerVersion:
                       print '    ==> modified to last to last valid release %s (s. \'formerVersion\' parameter)'%( cmsswVersion )
                   else:
                       print '    ==> modified to last valid release %s'%( cmsswVersion )

        # Check domain
        domain = socket.getfqdn().split( '.' )
        domainSE = ''
        if len( domain ) == 0:
            print '%s INFO : Cannot determine domain of this computer'%( self._label )
            if debug:
                self.messageEmptyList()
            return filePaths
        elif os.uname()[0] == "Darwin":
            print '%s INFO : Running on MacOSX without direct access to RelVal files.'%( self._label )
            if debug:
                self.messageEmptyList()
            return filePaths
        elif len( domain ) == 1:
            print '%s INFO : Running on local host \'%s\' without direct access to RelVal files'%( self._label, domain[ 0 ] )
            if debug:
                self.messageEmptyList()
            return filePaths
        if not ( ( domain[ -2 ] == 'cern' and domain[ -1 ] == 'ch' ) or ( domain[ -2 ] == 'fnal' and domain[ -1 ] == 'gov' ) ):
            print '%s INFO : Running on site \'%s.%s\' without direct access to RelVal files'%( self._label, domain[ -2 ], domain[ -1 ] )
            if debug:
                self.messageEmptyList()
            return filePaths
        if domain[ -2 ] == 'cern':
            domainSE = 'T2_CH_CERN'
        elif domain[ -2 ] == 'fnal':
            domainSE = 'T1_US_FNAL_MSS'
        if debug:
            print '%s DEBUG: Running at site \'%s.%s\''%( self._label, domain[ -2 ], domain[ -1 ] )
            print '%s DEBUG: Looking for SE \'%s\''%( self._label, domainSE )

        # Find files
        validVersion = 0
        dataset    = ''
        datasetAll = '/%s/%s-%s-v*/%s'%( relVal, cmsswVersion, globalTag, dataTier )
        if useDAS:
            if debug:
                print '%s DEBUG: Using DAS query'%( self._label )
            dasLimit = numberOfFiles
            if dasLimit <= 0:
                dasLimit = 1
            for version in range( maxVersions, 0, -1 ):
                filePaths    = []
                filePathsTmp = []
                fileCount    = 0
                dataset = '/%s/%s-%s-v%i/%s'%( relVal, cmsswVersion, globalTag, version, dataTier )
                dasQuery = 'file dataset=%s | grep file.name'%( dataset )
                if debug:
                    print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
                    print '    \'%s\''%( dasQuery )
                # partially stolen from das_client.py for option '--format=plain', needs filter ("grep") in the query
                jsondict = self.runDAS(dasQuery,dasLimit)
                if debug:
                    print '%s DEBUG: Received DAS JSON dictionary:'%( self._label )
                    print '    \'%s\''%( jsondict )
                if jsondict[ 'status' ] != 'ok':
                    print 'There was a problem while querying DAS with query \'%s\'. Server reply was:\n %s' % (dasQuery, jsondict)
                    exit( 1 )
                mongo_query = jsondict[ 'mongo_query' ]
                filters     = mongo_query[ 'filters' ]
                data        = jsondict[ 'data' ]
                if debug:
                    print '%s DEBUG: Query in JSON dictionary:'%( self._label )
                    print '    \'%s\''%( mongo_query )
                    print '%s DEBUG: Filters in query:'%( self._label )
                    print '    \'%s\''%( filters )
                    print '%s DEBUG: Data in JSON dictionary:'%( self._label )
                    print '    \'%s\''%( data )
                for row in data:
                    filePath = [ r for r in das_client.get_value( row, filters[ 'grep' ] ) ][ 0 ]
                    if debug:
                        print '%s DEBUG: Testing file entry \'%s\''%( self._label, filePath )
                    if len( filePath ) > 0:
                        if validVersion != version:
                            jsontestdict = self.runDAS('site dataset=%s | grep site.name' % ( dataset ),  999)
                            mongo_testquery = jsontestdict[ 'mongo_query' ]
                            testfilters = mongo_testquery[ 'filters' ]
                            testdata    = jsontestdict[ 'data' ]
                            if debug:
                                print '%s DEBUG: Received DAS JSON dictionary (site test):'%( self._label )
                                print '    \'%s\''%( jsontestdict )
                                print '%s DEBUG: Query in JSON dictionary (site test):'%( self._label )
                                print '    \'%s\''%( mongo_testquery )
                                print '%s DEBUG: Filters in query (site test):'%( self._label )
                                print '    \'%s\''%( testfilters )
                                print '%s DEBUG: Data in JSON dictionary (site test):'%( self._label )
                                print '    \'%s\''%( testdata )
                            foundSE = False
                            for testrow in testdata:
                                siteName = [ tr for tr in das_client.get_value( testrow, testfilters[ 'grep' ] ) ][ 0 ]
                                if siteName == domainSE:
                                    foundSE = True
                                    break
                            if not foundSE:
                                if debug:
                                    print '%s DEBUG: Possible version \'v%s\' not available on SE \'%s\''%( self._label, version, domainSE )
                                break
                            validVersion = version
                            if debug:
                                print '%s DEBUG: Valid version set to \'v%i\''%( self._label, validVersion )
                        if numberOfFiles == 0:
                            break
                        # protect from double entries ( 'unique' flag in query does not work here)
                        if not filePath in filePathsTmp:
                            filePathsTmp.append( filePath )
                            if debug:
                                print '%s DEBUG: File \'%s\' found'%( self._label, filePath )
                            fileCount += 1
                            # needed, since and "limit" overrides "idx" in 'get_data' (==> "idx" set to '0' rather than "skipFiles")
                            if fileCount > skipFiles:
                                filePaths.append( filePath )
                        elif debug:
                            print '%s DEBUG: File \'%s\' found again'%( self._label, filePath )
                if validVersion > 0:
                    if numberOfFiles == 0 and debug:
                        print '%s DEBUG: No files requested'%( self._label )
                    break
        else:
            if debug:
                print '%s DEBUG: Using DBS query'%( self._label )
            print '%s WARNING: DBS query disabled for DBS3 transition to new API'%( self._label )
            #for version in range( maxVersions, 0, -1 ):
                #filePaths = []
                #fileCount = 0
                #dataset = '/%s/%s-%s-v%i/%s'%( relVal, cmsswVersion, globalTag, version, dataTier )
                #dbsQuery = 'find file where dataset = %s'%( dataset )
                #if debug:
                    #print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
                    #print '    \'%s\''%( dbsQuery )
                #foundSE = False
                #for line in os.popen( 'dbs search --query="%s"'%( dbsQuery ) ).readlines():
                    #if line.find( '.root' ) != -1:
                        #if validVersion != version:
                            #if not foundSE:
                                #dbsSiteQuery = 'find dataset where dataset = %s and site = %s'%( dataset, domainSE )
                                #if debug:
                                    #print '%s DEBUG: Querying site \'%s\' with'%( self._label, domainSE )
                                    #print '    \'%s\''%( dbsSiteQuery )
                                #for lineSite in os.popen( 'dbs search --query="%s"'%( dbsSiteQuery ) ).readlines():
                                    #if lineSite.find( dataset ) != -1:
                                        #foundSE = True
                                        #break
                            #if not foundSE:
                                #if debug:
                                    #print '%s DEBUG: Possible version \'v%s\' not available on SE \'%s\''%( self._label, version, domainSE )
                                #break
                            #validVersion = version
                            #if debug:
                                #print '%s DEBUG: Valid version set to \'v%i\''%( self._label, validVersion )
                        #if numberOfFiles == 0:
                            #break
                        #filePath = line.replace( '\n', '' )
                        #if debug:
                            #print '%s DEBUG: File \'%s\' found'%( self._label, filePath )
                        #fileCount += 1
                        #if fileCount > skipFiles:
                            #filePaths.append( filePath )
                        #if not numberOfFiles < 0:
                            #if numberOfFiles <= len( filePaths ):
                                #break
                #if validVersion > 0:
                    #if numberOfFiles == 0 and debug:
                        #print '%s DEBUG: No files requested'%( self._label )
                    #break

        # Check output and return
        if validVersion == 0:
            print '%s WARNING : No RelVal file(s) found at all in datasets \'%s*\' on SE \'%s\''%( self._label, datasetAll, domainSE )
            if debug:
                self.messageEmptyList()
        elif len( filePaths ) == 0:
            print '%s WARNING : No RelVal file(s) picked up in dataset \'%s\''%( self._label, dataset )
            if debug:
                self.messageEmptyList()
        elif len( filePaths ) < numberOfFiles:
            print '%s INFO : Only %i RelVal file(s) instead of %i picked up in dataset \'%s\''%( self._label, len( filePaths ), numberOfFiles, dataset )

        if debug:
            print '%s DEBUG: returning %i file(s):\n%s'%( self._label, len( filePaths ), filePaths )
        return filePaths
Exemplo n.º 7
0
    def apply(self):
        useDAS = self._parameters["useDAS"].value
        cmsswVersion = self._parameters["cmsswVersion"].value
        formerVersion = self._parameters["formerVersion"].value
        relVal = self._parameters["relVal"].value
        dataTier = self._parameters["dataTier"].value
        condition = self._parameters[
            "condition"
        ].value  # only used for GT determination in initialization, if GT not explicitly given
        globalTag = self._parameters["globalTag"].value
        maxVersions = self._parameters["maxVersions"].value
        skipFiles = self._parameters["skipFiles"].value
        numberOfFiles = self._parameters["numberOfFiles"].value
        debug = self._parameters["debug"].value

        filePaths = []

        # Determine corresponding CMSSW version for RelVals
        preId = "_pre"
        patchId = "_patch"  # patch releases
        hltPatchId = "_hltpatch"  # HLT patch releases
        dqmPatchId = "_dqmpatch"  # DQM patch releases
        slhcId = "_SLHC"  # SLHC releases
        rootId = "_root"  # ROOT test releases
        ibId = "_X_"  # IBs
        if patchId in cmsswVersion:
            cmsswVersion = cmsswVersion.split(patchId)[0]
        elif hltPatchId in cmsswVersion:
            cmsswVersion = cmsswVersion.split(hltPatchId)[0]
        elif dqmPatchId in cmsswVersion:
            cmsswVersion = cmsswVersion.split(dqmPatchId)[0]
        elif rootId in cmsswVersion:
            cmsswVersion = cmsswVersion.split(rootId)[0]
        elif slhcId in cmsswVersion:
            cmsswVersion = cmsswVersion.split(slhcId)[0]
        elif ibId in cmsswVersion or formerVersion:
            outputTuple = Popen(["scram", "l -c CMSSW"], stdout=PIPE, stderr=PIPE).communicate()
            if len(outputTuple[1]) != 0:
                print "%s INFO : SCRAM error" % (self._label)
                if debug:
                    print "    from trying to determine last valid releases before '%s'" % (cmsswVersion)
                    print
                    print outputTuple[1]
                    print
                    self.messageEmptyList()
                return filePaths
            versions = {"last": "", "lastToLast": ""}
            for line in outputTuple[0].splitlines():
                version = line.split()[1]
                if cmsswVersion.split(ibId)[0] in version or cmsswVersion.rpartition("_")[0] in version:
                    if not (
                        patchId in version
                        or hltPatchId in version
                        or dqmPatchId in version
                        or slhcId in version
                        or ibId in version
                        or rootId in version
                    ):
                        versions["lastToLast"] = versions["last"]
                        versions["last"] = version
                        if version == cmsswVersion:
                            break
            # FIXME: ordering of output problematic ('XYZ_pre10' before 'XYZ_pre2', no "formerVersion" for 'XYZ_pre1')
            if formerVersion:
                # Don't use pre-releases as "former version" for other releases than CMSSW_X_Y_0
                if (
                    preId in versions["lastToLast"]
                    and not preId in versions["last"]
                    and not versions["last"].endswith("_0")
                ):
                    versions["lastToLast"] = versions["lastToLast"].split(preId)[
                        0
                    ]  # works only, if 'CMSSW_X_Y_0' esists ;-)
                # Use pre-release as "former version" for CMSSW_X_Y_0
                elif versions["last"].endswith("_0") and not (
                    preId in versions["lastToLast"] and versions["lastToLast"].startswith(versions["last"])
                ):
                    versions["lastToLast"] = ""
                    for line in outputTuple[0].splitlines():
                        version = line.split()[1]
                        versionParts = version.partition(preId)
                        if versionParts[0] == versions["last"] and versionParts[1] == preId:
                            versions["lastToLast"] = version
                        elif versions["lastToLast"] != "":
                            break
                # Don't use CMSSW_X_Y_0 as "former version" for pre-releases
                elif (
                    preId in versions["last"]
                    and not preId in versions["lastToLast"]
                    and versions["lastToLast"].endswith("_0")
                ):
                    versions["lastToLast"] = ""  # no alternative :-(
                cmsswVersion = versions["lastToLast"]
            else:
                cmsswVersion = versions["last"]

        # Debugging output
        if debug:
            print "%s DEBUG: Called with..." % (self._label)
            for key in self._parameters.keys():
                print "    %s:\t" % (key),
                print self._parameters[key].value,
                if self._parameters[key].value is self.getDefaultParameters()[key].value:
                    print " (default)"
                else:
                    print
                if key == "cmsswVersion" and cmsswVersion != self._parameters[key].value:
                    if formerVersion:
                        print "    ==> modified to last to last valid release %s (s. 'formerVersion' parameter)" % (
                            cmsswVersion
                        )
                    else:
                        print "    ==> modified to last valid release %s" % (cmsswVersion)

        # Check domain
        domain = socket.getfqdn().split(".")
        domainSE = ""
        if len(domain) == 0:
            print "%s INFO : Cannot determine domain of this computer" % (self._label)
            if debug:
                self.messageEmptyList()
            return filePaths
        elif os.uname()[0] == "Darwin":
            print "%s INFO : Running on MacOSX without direct access to RelVal files." % (self._label)
            if debug:
                self.messageEmptyList()
            return filePaths
        elif len(domain) == 1:
            print "%s INFO : Running on local host '%s' without direct access to RelVal files" % (
                self._label,
                domain[0],
            )
            if debug:
                self.messageEmptyList()
            return filePaths
        if not ((domain[-2] == "cern" and domain[-1] == "ch") or (domain[-2] == "fnal" and domain[-1] == "gov")):
            print "%s INFO : Running on site '%s.%s' without direct access to RelVal files" % (
                self._label,
                domain[-2],
                domain[-1],
            )
            if debug:
                self.messageEmptyList()
            return filePaths
        if domain[-2] == "cern":
            domainSE = "T2_CH_CERN"
        elif domain[-2] == "fnal":
            domainSE = "T1_US_FNAL_MSS"
        if debug:
            print "%s DEBUG: Running at site '%s.%s'" % (self._label, domain[-2], domain[-1])
            print "%s DEBUG: Looking for SE '%s'" % (self._label, domainSE)

        # Find files
        validVersion = 0
        dataset = ""
        datasetAll = "/%s/%s-%s-v*/%s" % (relVal, cmsswVersion, globalTag, dataTier)
        if useDAS:
            if debug:
                print "%s DEBUG: Using DAS query" % (self._label)
            dasLimit = numberOfFiles
            if dasLimit <= 0:
                dasLimit += 1
            for version in range(maxVersions, 0, -1):
                filePaths = []
                filePathsTmp = []
                fileCount = 0
                dataset = "/%s/%s-%s-v%i/%s" % (relVal, cmsswVersion, globalTag, version, dataTier)
                dasQuery = "file dataset=%s | grep file.name" % (dataset)
                if debug:
                    print "%s DEBUG: Querying dataset '%s' with" % (self._label, dataset)
                    print "    '%s'" % (dasQuery)
                # partially stolen from das_client.py for option '--format=plain', needs filter ("grep") in the query
                jsondict = das_client.get_data("https://cmsweb.cern.ch", dasQuery, 0, dasLimit, False)
                if debug:
                    print "%s DEBUG: Received DAS JSON dictionary:" % (self._label)
                    print "    '%s'" % (jsondict)
                if jsondict["status"] != "ok":
                    print "There was a problem while querying DAS with query '%s'. Server reply was:\n %s" % (
                        dasQuery,
                        jsondict,
                    )
                    exit(1)
                mongo_query = jsondict["mongo_query"]
                filters = mongo_query["filters"]
                data = jsondict["data"]
                if debug:
                    print "%s DEBUG: Query in JSON dictionary:" % (self._label)
                    print "    '%s'" % (mongo_query)
                    print "%s DEBUG: Filters in query:" % (self._label)
                    print "    '%s'" % (filters)
                    print "%s DEBUG: Data in JSON dictionary:" % (self._label)
                    print "    '%s'" % (data)
                for row in data:
                    filePath = [r for r in das_client.get_value(row, filters["grep"])][0]
                    if debug:
                        print "%s DEBUG: Testing file entry '%s'" % (self._label, filePath)
                    if len(filePath) > 0:
                        if validVersion != version:
                            jsontestdict = das_client.get_data(
                                "https://cmsweb.cern.ch", "site dataset=%s | grep site.name" % (dataset), 0, 999, False
                            )
                            mongo_testquery = jsontestdict["mongo_query"]
                            testfilters = mongo_testquery["filters"]
                            testdata = jsontestdict["data"]
                            if debug:
                                print "%s DEBUG: Received DAS JSON dictionary (site test):" % (self._label)
                                print "    '%s'" % (jsontestdict)
                                print "%s DEBUG: Query in JSON dictionary (site test):" % (self._label)
                                print "    '%s'" % (mongo_testquery)
                                print "%s DEBUG: Filters in query (site test):" % (self._label)
                                print "    '%s'" % (testfilters)
                                print "%s DEBUG: Data in JSON dictionary (site test):" % (self._label)
                                print "    '%s'" % (testdata)
                            foundSE = False
                            for testrow in testdata:
                                siteName = [tr for tr in das_client.get_value(testrow, testfilters["grep"])][0]
                                if siteName == domainSE:
                                    foundSE = True
                                    break
                            if not foundSE:
                                if debug:
                                    print "%s DEBUG: Possible version 'v%s' not available on SE '%s'" % (
                                        self._label,
                                        version,
                                        domainSE,
                                    )
                                break
                            validVersion = version
                            if debug:
                                print "%s DEBUG: Valid version set to 'v%i'" % (self._label, validVersion)
                        if numberOfFiles == 0:
                            break
                        # protect from double entries ( 'unique' flag in query does not work here)
                        if not filePath in filePathsTmp:
                            filePathsTmp.append(filePath)
                            if debug:
                                print "%s DEBUG: File '%s' found" % (self._label, filePath)
                            fileCount += 1
                            # needed, since and "limit" overrides "idx" in 'get_data' (==> "idx" set to '0' rather than "skipFiles")
                            if fileCount > skipFiles:
                                filePaths.append(filePath)
                        elif debug:
                            print "%s DEBUG: File '%s' found again" % (self._label, filePath)
                if validVersion > 0:
                    if numberOfFiles == 0 and debug:
                        print "%s DEBUG: No files requested" % (self._label)
                    break
        else:
            if debug:
                print "%s DEBUG: Using DBS query" % (self._label)
            print "%s WARNING: DBS query disabled for DBS3 transition to new API" % (self._label)
            # for version in range( maxVersions, 0, -1 ):
            # filePaths = []
            # fileCount = 0
            # dataset = '/%s/%s-%s-v%i/%s'%( relVal, cmsswVersion, globalTag, version, dataTier )
            # dbsQuery = 'find file where dataset = %s'%( dataset )
            # if debug:
            # print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
            # print '    \'%s\''%( dbsQuery )
            # foundSE = False
            # for line in os.popen( 'dbs search --query="%s"'%( dbsQuery ) ).readlines():
            # if line.find( '.root' ) != -1:
            # if validVersion != version:
            # if not foundSE:
            # dbsSiteQuery = 'find dataset where dataset = %s and site = %s'%( dataset, domainSE )
            # if debug:
            # print '%s DEBUG: Querying site \'%s\' with'%( self._label, domainSE )
            # print '    \'%s\''%( dbsSiteQuery )
            # for lineSite in os.popen( 'dbs search --query="%s"'%( dbsSiteQuery ) ).readlines():
            # if lineSite.find( dataset ) != -1:
            # foundSE = True
            # break
            # if not foundSE:
            # if debug:
            # print '%s DEBUG: Possible version \'v%s\' not available on SE \'%s\''%( self._label, version, domainSE )
            # break
            # validVersion = version
            # if debug:
            # print '%s DEBUG: Valid version set to \'v%i\''%( self._label, validVersion )
            # if numberOfFiles == 0:
            # break
            # filePath = line.replace( '\n', '' )
            # if debug:
            # print '%s DEBUG: File \'%s\' found'%( self._label, filePath )
            # fileCount += 1
            # if fileCount > skipFiles:
            # filePaths.append( filePath )
            # if not numberOfFiles < 0:
            # if numberOfFiles <= len( filePaths ):
            # break
            # if validVersion > 0:
            # if numberOfFiles == 0 and debug:
            # print '%s DEBUG: No files requested'%( self._label )
            # break

        # Check output and return
        if validVersion == 0:
            print "%s WARNING : No RelVal file(s) found at all in datasets '%s*' on SE '%s'" % (
                self._label,
                datasetAll,
                domainSE,
            )
            if debug:
                self.messageEmptyList()
        elif len(filePaths) == 0:
            print "%s WARNING : No RelVal file(s) picked up in dataset '%s'" % (self._label, dataset)
            if debug:
                self.messageEmptyList()
        elif len(filePaths) < numberOfFiles:
            print "%s INFO : Only %i RelVal file(s) instead of %i picked up in dataset '%s'" % (
                self._label,
                len(filePaths),
                numberOfFiles,
                dataset,
            )

        if debug:
            print "%s DEBUG: returning %i file(s):\n%s" % (self._label, len(filePaths), filePaths)
        return filePaths
Exemplo n.º 8
0
    files = []
    # Query DAS
    query = "file dataset=%(dataset)s run=%(run)i lumi=%(lumi)i | grep file.name" % event
    jsondict = das_client.get_data('https://cmsweb.cern.ch', query, 0, 0, False)
    status = jsondict['status']
    if status != 'ok':
        print "DAS query status: %s"%(status)
        return files
    
    mongo_query = jsondict['mongo_query']
    filters = mongo_query['filters']
    data = jsondict['data']

files = []
    for row in data:
        file = [r for r in das_client.get_value(row, filters['grep'])][0]
        if len(file) > 0 and not file in files:
            files.append(file)

return files


def fullCPMpath():
    base = os.environ.get ('CMSSW_BASE')
    if not base:
        raise RuntimeError, "CMSSW Environment not set"
    retval = "%s/src/PhysicsTools/Utilities/configuration/copyPickMerge_cfg.py" \
        % base
if os.path.exists (retval):
    return retval
    base = os.environ.get ('CMSSW_RELEASE_BASE')