def getDASData (query):
results = []
jsondict = das_client.get_data ('https://cmsweb.cern.ch', query, 0, 0, False)
status = jsondict['status']
if status != 'ok':
print "DAS query status: %s"%(status)
return results
mongo_query = jsondict['mongo_query']
filters = mongo_query['filters']
data = jsondict['data']
for row in data:
result = [r for r in das_client.get_value (row, filters['grep'])][0]
if len (result) > 0 and not result in results:
results.append (result)
return results
def getFileNames (event):
files = []
# Query DAS
query = "file dataset=%(dataset)s run=%(run)i lumi=%(lumi)i | grep file.name" % event
jsondict = das_client.get_data('https://cmsweb.cern.ch', query, 0, 0, False)
status = jsondict['status']
if status != 'ok':
print "DAS query status: %s"%(status)
return files
mongo_query = jsondict['mongo_query']
filters = mongo_query['filters']
data = jsondict['data']
files = []
for row in data:
file = [r for r in das_client.get_value(row, filters['grep'])][0]
if len(file) > 0 and not file in files:
files.append(file)
return files
def getFileNames (event):
files = []
# Query DAS
query = "file dataset=%(dataset)s run=%(run)i lumi=%(lumi)i | grep file.name" % event
jsondict = das_client.get_data('https://cmsweb.cern.ch', query, 0, 0, False)
status = jsondict['status']
if status != 'ok':
print "DAS query status: %s"%(status)
return files
mongo_query = jsondict['mongo_query']
filters = mongo_query['filters']
data = jsondict['data']
files = []
for row in data:
file = [r for r in das_client.get_value(row, filters['grep'])][0]
if len(file) > 0 and not file in files:
files.append(file)
return files
def getFileNames(event):
files = []
# Query DAS
query = "file dataset=%(dataset)s run=%(run)i lumi=%(lumi)i | grep file.name" % event
jsondict = das_client.get_data("https://cmsweb.cern.ch", query, 0, 0, False)
status = jsondict["status"]
if status != "ok":
print "DAS query status: %s" % (status)
return files
mongo_query = jsondict["mongo_query"]
filters = mongo_query["filters"]
data = jsondict["data"]
files = []
for row in data:
file = [r for r in das_client.get_value(row, filters["grep"])][0]
if len(file) > 0 and not file in files:
files.append(file)
return files
def apply( self ):
useDAS = self._parameters[ 'useDAS' ].value
cmsswVersion = self._parameters[ 'cmsswVersion' ].value
formerVersion = self._parameters[ 'formerVersion' ].value
relVal = self._parameters[ 'relVal' ].value
dataTier = self._parameters[ 'dataTier' ].value
condition = self._parameters[ 'condition' ].value # only used for GT determination in initialization, if GT not explicitly given
globalTag = self._parameters[ 'globalTag' ].value
maxVersions = self._parameters[ 'maxVersions' ].value
skipFiles = self._parameters[ 'skipFiles' ].value
numberOfFiles = self._parameters[ 'numberOfFiles' ].value
debug = self._parameters[ 'debug' ].value
filePaths = []
# Determine corresponding CMSSW version for RelVals
preId = '_pre'
patchId = '_patch' # patch releases
hltPatchId = '_hltpatch' # HLT patch releases
dqmPatchId = '_dqmpatch' # DQM patch releases
slhcId = '_SLHC' # SLHC releases
rootId = '_root' # ROOT test releases
ibId = '_X_' # IBs
if patchId in cmsswVersion:
cmsswVersion = cmsswVersion.split( patchId )[ 0 ]
elif hltPatchId in cmsswVersion:
cmsswVersion = cmsswVersion.split( hltPatchId )[ 0 ]
elif dqmPatchId in cmsswVersion:
cmsswVersion = cmsswVersion.split( dqmPatchId )[ 0 ]
elif rootId in cmsswVersion:
cmsswVersion = cmsswVersion.split( rootId )[ 0 ]
elif slhcId in cmsswVersion:
cmsswVersion = cmsswVersion.split( slhcId )[ 0 ]
elif ibId in cmsswVersion or formerVersion:
outputTuple = Popen( [ 'scram', 'l -c CMSSW' ], stdout = PIPE, stderr = PIPE ).communicate()
if len( outputTuple[ 1 ] ) != 0:
print '%s INFO : SCRAM error'%( self._label )
if debug:
print ' from trying to determine last valid releases before \'%s\''%( cmsswVersion )
print
print outputTuple[ 1 ]
print
self.messageEmptyList()
return filePaths
versions = { 'last' :''
, 'lastToLast':''
}
for line in outputTuple[ 0 ].splitlines():
version = line.split()[ 1 ]
if cmsswVersion.split( ibId )[ 0 ] in version or cmsswVersion.rpartition( '_' )[ 0 ] in version:
if not ( patchId in version or hltPatchId in version or dqmPatchId in version or slhcId in version or ibId in version or rootId in version ):
versions[ 'lastToLast' ] = versions[ 'last' ]
versions[ 'last' ] = version
if version == cmsswVersion:
break
# FIXME: ordering of output problematic ('XYZ_pre10' before 'XYZ_pre2', no "formerVersion" for 'XYZ_pre1')
if formerVersion:
# Don't use pre-releases as "former version" for other releases than CMSSW_X_Y_0
if preId in versions[ 'lastToLast' ] and not preId in versions[ 'last' ] and not versions[ 'last' ].endswith( '_0' ):
versions[ 'lastToLast' ] = versions[ 'lastToLast' ].split( preId )[ 0 ] # works only, if 'CMSSW_X_Y_0' esists ;-)
# Use pre-release as "former version" for CMSSW_X_Y_0
elif versions[ 'last' ].endswith( '_0' ) and not ( preId in versions[ 'lastToLast' ] and versions[ 'lastToLast' ].startswith( versions[ 'last' ] ) ):
versions[ 'lastToLast' ] = ''
for line in outputTuple[ 0 ].splitlines():
version = line.split()[ 1 ]
versionParts = version.partition( preId )
if versionParts[ 0 ] == versions[ 'last' ] and versionParts[ 1 ] == preId:
versions[ 'lastToLast' ] = version
elif versions[ 'lastToLast' ] != '':
break
# Don't use CMSSW_X_Y_0 as "former version" for pre-releases
elif preId in versions[ 'last' ] and not preId in versions[ 'lastToLast' ] and versions[ 'lastToLast' ].endswith( '_0' ):
versions[ 'lastToLast' ] = '' # no alternative :-(
cmsswVersion = versions[ 'lastToLast' ]
else:
cmsswVersion = versions[ 'last' ]
# Debugging output
if debug:
print '%s DEBUG: Called with...'%( self._label )
for key in self._parameters.keys():
print ' %s:\t'%( key ),
print self._parameters[ key ].value,
if self._parameters[ key ].value is self.getDefaultParameters()[ key ].value:
print ' (default)'
else:
print
if key == 'cmsswVersion' and cmsswVersion != self._parameters[ key ].value:
if formerVersion:
print ' ==> modified to last to last valid release %s (s. \'formerVersion\' parameter)'%( cmsswVersion )
else:
print ' ==> modified to last valid release %s'%( cmsswVersion )
# Check domain
domain = socket.getfqdn().split( '.' )
domainSE = ''
if len( domain ) == 0:
print '%s INFO : Cannot determine domain of this computer'%( self._label )
if debug:
self.messageEmptyList()
return filePaths
elif os.uname()[0] == "Darwin":
print '%s INFO : Running on MacOSX without direct access to RelVal files.'%( self._label )
if debug:
self.messageEmptyList()
return filePaths
elif len( domain ) == 1:
print '%s INFO : Running on local host \'%s\' without direct access to RelVal files'%( self._label, domain[ 0 ] )
if debug:
self.messageEmptyList()
return filePaths
if not ( ( domain[ -2 ] == 'cern' and domain[ -1 ] == 'ch' ) or ( domain[ -2 ] == 'fnal' and domain[ -1 ] == 'gov' ) ):
print '%s INFO : Running on site \'%s.%s\' without direct access to RelVal files'%( self._label, domain[ -2 ], domain[ -1 ] )
if debug:
self.messageEmptyList()
return filePaths
if domain[ -2 ] == 'cern':
domainSE = 'T2_CH_CERN'
elif domain[ -2 ] == 'fnal':
domainSE = 'T1_US_FNAL_MSS'
if debug:
print '%s DEBUG: Running at site \'%s.%s\''%( self._label, domain[ -2 ], domain[ -1 ] )
print '%s DEBUG: Looking for SE \'%s\''%( self._label, domainSE )
# Find files
validVersion = 0
dataset = ''
datasetAll = '/%s/%s-%s-v*/%s'%( relVal, cmsswVersion, globalTag, dataTier )
if useDAS:
if debug:
print '%s DEBUG: Using DAS query'%( self._label )
dasLimit = numberOfFiles
if dasLimit <= 0:
dasLimit += 1
for version in range( maxVersions, 0, -1 ):
filePaths = []
filePathsTmp = []
fileCount = 0
dataset = '/%s/%s-%s-v%i/%s'%( relVal, cmsswVersion, globalTag, version, dataTier )
dasQuery = 'file dataset=%s | grep file.name'%( dataset )
if debug:
print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
print ' \'%s\''%( dasQuery )
# partially stolen from das_client.py for option '--format=plain', needs filter ("grep") in the query
dasData = das_client.get_data( 'https://cmsweb.cern.ch', dasQuery, 0, dasLimit, False )
jsondict = json.loads( dasData )
if debug:
print '%s DEBUG: Received DAS data:'%( self._label )
print ' \'%s\''%( dasData )
print '%s DEBUG: Determined JSON dictionary:'%( self._label )
print ' \'%s\''%( jsondict )
if jsondict[ 'status' ] != 'ok':
print 'There was a problem while querying DAS with query \'%s\'. Server reply was:\n %s' % (dasQuery, dasData)
exit( 1 )
mongo_query = jsondict[ 'mongo_query' ]
filters = mongo_query[ 'filters' ]
data = jsondict[ 'data' ]
if debug:
print '%s DEBUG: Query in JSON dictionary:'%( self._label )
print ' \'%s\''%( mongo_query )
print '%s DEBUG: Filters in query:'%( self._label )
print ' \'%s\''%( filters )
print '%s DEBUG: Data in JSON dictionary:'%( self._label )
print ' \'%s\''%( data )
for row in data:
filePath = [ r for r in das_client.get_value( row, filters ) ][ 0 ]
if debug:
print '%s DEBUG: Testing file entry \'%s\''%( self._label, filePath )
if len( filePath ) > 0:
if validVersion != version:
dasTest = das_client.get_data( 'https://cmsweb.cern.ch', 'site dataset=%s | grep site.name'%( dataset ), 0, 999, False )
jsontestdict = json.loads( dasTest )
mongo_testquery = jsontestdict[ 'mongo_query' ]
testfilters = mongo_testquery[ 'filters' ]
testdata = jsontestdict[ 'data' ]
if debug:
print '%s DEBUG: Received DAS data (site test):'%( self._label )
print ' \'%s\''%( dasTest )
print '%s DEBUG: Determined JSON dictionary (site test):'%( self._label )
print ' \'%s\''%( jsontestdict )
print '%s DEBUG: Query in JSON dictionary (site test):'%( self._label )
print ' \'%s\''%( mongo_testquery )
print '%s DEBUG: Filters in query (site test):'%( self._label )
print ' \'%s\''%( testfilters )
print '%s DEBUG: Data in JSON dictionary (site test):'%( self._label )
print ' \'%s\''%( testdata )
foundSE = False
for testrow in testdata:
siteName = [ tr for tr in das_client.get_value( testrow, testfilters ) ][ 0 ]
if siteName == domainSE:
foundSE = True
break
if not foundSE:
if debug:
print '%s DEBUG: Possible version \'v%s\' not available on SE \'%s\''%( self._label, version, domainSE )
break
validVersion = version
if debug:
print '%s DEBUG: Valid version set to \'v%i\''%( self._label, validVersion )
if numberOfFiles == 0:
break
# protect from double entries ( 'unique' flag in query does not work here)
if not filePath in filePathsTmp:
filePathsTmp.append( filePath )
if debug:
print '%s DEBUG: File \'%s\' found'%( self._label, filePath )
fileCount += 1
# needed, since and "limit" overrides "idx" in 'get_data' (==> "idx" set to '0' rather than "skipFiles")
if fileCount > skipFiles:
filePaths.append( filePath )
elif debug:
print '%s DEBUG: File \'%s\' found again'%( self._label, filePath )
if validVersion > 0:
if numberOfFiles == 0 and debug:
print '%s DEBUG: No files requested'%( self._label )
break
else:
if debug:
print '%s DEBUG: Using DBS query'%( self._label )
for version in range( maxVersions, 0, -1 ):
filePaths = []
fileCount = 0
dataset = '/%s/%s-%s-v%i/%s'%( relVal, cmsswVersion, globalTag, version, dataTier )
dbsQuery = 'find file where dataset = %s'%( dataset )
if debug:
print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
print ' \'%s\''%( dbsQuery )
foundSE = False
for line in os.popen( 'dbs search --query="%s"'%( dbsQuery ) ):
if line.find( '.root' ) != -1:
if validVersion != version:
if not foundSE:
dbsSiteQuery = 'find dataset where dataset = %s and site = %s'%( dataset, domainSE )
if debug:
print '%s DEBUG: Querying site \'%s\' with'%( self._label, domainSE )
print ' \'%s\''%( dbsSiteQuery )
for lineSite in os.popen( 'dbs search --query="%s"'%( dbsSiteQuery ) ):
if lineSite.find( dataset ) != -1:
foundSE = True
break
if not foundSE:
if debug:
print '%s DEBUG: Possible version \'v%s\' not available on SE \'%s\''%( self._label, version, domainSE )
break
validVersion = version
if debug:
print '%s DEBUG: Valid version set to \'v%i\''%( self._label, validVersion )
if numberOfFiles == 0:
break
filePath = line.replace( '\n', '' )
if debug:
print '%s DEBUG: File \'%s\' found'%( self._label, filePath )
fileCount += 1
if fileCount > skipFiles:
filePaths.append( filePath )
if not numberOfFiles < 0:
if numberOfFiles <= len( filePaths ):
break
if validVersion > 0:
if numberOfFiles == 0 and debug:
print '%s DEBUG: No files requested'%( self._label )
break
# Check output and return
if validVersion == 0:
print '%s INFO : No RelVal file(s) found at all in datasets \'%s*\' on SE \'%s\''%( self._label, datasetAll, domainSE )
if debug:
self.messageEmptyList()
elif len( filePaths ) == 0:
print '%s INFO : No RelVal file(s) picked up in dataset \'%s\''%( self._label, dataset )
if debug:
self.messageEmptyList()
elif len( filePaths ) < numberOfFiles:
print '%s INFO : Only %i RelVal file(s) instead of %i picked up in dataset \'%s\''%( self._label, len( filePaths ), numberOfFiles, dataset )
if debug:
print '%s DEBUG: returning %i file(s):\n%s'%( self._label, len( filePaths ), filePaths )
return filePaths
def apply( self ):
useDAS = self._parameters[ 'useDAS' ].value
cmsswVersion = self._parameters[ 'cmsswVersion' ].value
formerVersion = self._parameters[ 'formerVersion' ].value
relVal = self._parameters[ 'relVal' ].value
dataTier = self._parameters[ 'dataTier' ].value
condition = self._parameters[ 'condition' ].value # only used for GT determination in initialization, if GT not explicitly given
globalTag = self._parameters[ 'globalTag' ].value
maxVersions = self._parameters[ 'maxVersions' ].value
skipFiles = self._parameters[ 'skipFiles' ].value
numberOfFiles = self._parameters[ 'numberOfFiles' ].value
debug = self._parameters[ 'debug' ].value
filePaths = []
# Determine corresponding CMSSW version for RelVals
preId = '_pre'
patchId = '_patch' # patch releases
hltPatchId = '_hltpatch' # HLT patch releases
dqmPatchId = '_dqmpatch' # DQM patch releases
slhcId = '_SLHC' # SLHC releases
rootId = '_root' # ROOT test releases
ibId = '_X_' # IBs
if patchId in cmsswVersion:
cmsswVersion = cmsswVersion.split( patchId )[ 0 ]
elif hltPatchId in cmsswVersion:
cmsswVersion = cmsswVersion.split( hltPatchId )[ 0 ]
elif dqmPatchId in cmsswVersion:
cmsswVersion = cmsswVersion.split( dqmPatchId )[ 0 ]
elif rootId in cmsswVersion:
cmsswVersion = cmsswVersion.split( rootId )[ 0 ]
elif slhcId in cmsswVersion:
cmsswVersion = cmsswVersion.split( slhcId )[ 0 ]
elif ibId in cmsswVersion or formerVersion:
outputTuple = Popen( [ 'scram', 'l -c CMSSW' ], stdout = PIPE, stderr = PIPE ).communicate()
if len( outputTuple[ 1 ] ) != 0:
print '%s INFO : SCRAM error'%( self._label )
if debug:
print ' from trying to determine last valid releases before \'%s\''%( cmsswVersion )
print
print outputTuple[ 1 ]
print
self.messageEmptyList()
return filePaths
versions = { 'last' :''
, 'lastToLast':''
}
for line in outputTuple[ 0 ].splitlines():
version = line.split()[ 1 ]
if cmsswVersion.split( ibId )[ 0 ] in version or cmsswVersion.rpartition( '_' )[ 0 ] in version:
if not ( patchId in version or hltPatchId in version or dqmPatchId in version or slhcId in version or ibId in version or rootId in version ):
versions[ 'lastToLast' ] = versions[ 'last' ]
versions[ 'last' ] = version
if version == cmsswVersion:
break
# FIXME: ordering of output problematic ('XYZ_pre10' before 'XYZ_pre2', no "formerVersion" for 'XYZ_pre1')
if formerVersion:
# Don't use pre-releases as "former version" for other releases than CMSSW_X_Y_0
if preId in versions[ 'lastToLast' ] and not preId in versions[ 'last' ] and not versions[ 'last' ].endswith( '_0' ):
versions[ 'lastToLast' ] = versions[ 'lastToLast' ].split( preId )[ 0 ] # works only, if 'CMSSW_X_Y_0' esists ;-)
# Use pre-release as "former version" for CMSSW_X_Y_0
elif versions[ 'last' ].endswith( '_0' ) and not ( preId in versions[ 'lastToLast' ] and versions[ 'lastToLast' ].startswith( versions[ 'last' ] ) ):
versions[ 'lastToLast' ] = ''
for line in outputTuple[ 0 ].splitlines():
version = line.split()[ 1 ]
versionParts = version.partition( preId )
if versionParts[ 0 ] == versions[ 'last' ] and versionParts[ 1 ] == preId:
versions[ 'lastToLast' ] = version
elif versions[ 'lastToLast' ] != '':
break
# Don't use CMSSW_X_Y_0 as "former version" for pre-releases
elif preId in versions[ 'last' ] and not preId in versions[ 'lastToLast' ] and versions[ 'lastToLast' ].endswith( '_0' ):
versions[ 'lastToLast' ] = '' # no alternative :-(
cmsswVersion = versions[ 'lastToLast' ]
else:
cmsswVersion = versions[ 'last' ]
# Debugging output
if debug:
print '%s DEBUG: Called with...'%( self._label )
for key in self._parameters.keys():
print ' %s:\t'%( key ),
print self._parameters[ key ].value,
if self._parameters[ key ].value is self.getDefaultParameters()[ key ].value:
print ' (default)'
else:
print
if key == 'cmsswVersion' and cmsswVersion != self._parameters[ key ].value:
if formerVersion:
print ' ==> modified to last to last valid release %s (s. \'formerVersion\' parameter)'%( cmsswVersion )
else:
print ' ==> modified to last valid release %s'%( cmsswVersion )
# Check domain
domain = socket.getfqdn().split( '.' )
domainSE = ''
if len( domain ) == 0:
print '%s INFO : Cannot determine domain of this computer'%( self._label )
if debug:
self.messageEmptyList()
return filePaths
elif os.uname()[0] == "Darwin":
print '%s INFO : Running on MacOSX without direct access to RelVal files.'%( self._label )
if debug:
self.messageEmptyList()
return filePaths
elif len( domain ) == 1:
print '%s INFO : Running on local host \'%s\' without direct access to RelVal files'%( self._label, domain[ 0 ] )
if debug:
self.messageEmptyList()
return filePaths
if not ( ( domain[ -2 ] == 'cern' and domain[ -1 ] == 'ch' ) or ( domain[ -2 ] == 'fnal' and domain[ -1 ] == 'gov' ) ):
print '%s INFO : Running on site \'%s.%s\' without direct access to RelVal files'%( self._label, domain[ -2 ], domain[ -1 ] )
if debug:
self.messageEmptyList()
return filePaths
if domain[ -2 ] == 'cern':
domainSE = 'T2_CH_CERN'
elif domain[ -2 ] == 'fnal':
domainSE = 'T1_US_FNAL_MSS'
if debug:
print '%s DEBUG: Running at site \'%s.%s\''%( self._label, domain[ -2 ], domain[ -1 ] )
print '%s DEBUG: Looking for SE \'%s\''%( self._label, domainSE )
# Find files
validVersion = 0
dataset = ''
datasetAll = '/%s/%s-%s-v*/%s'%( relVal, cmsswVersion, globalTag, dataTier )
if useDAS:
if debug:
print '%s DEBUG: Using DAS query'%( self._label )
dasLimit = numberOfFiles
if dasLimit <= 0:
dasLimit = 1
for version in range( maxVersions, 0, -1 ):
filePaths = []
filePathsTmp = []
fileCount = 0
dataset = '/%s/%s-%s-v%i/%s'%( relVal, cmsswVersion, globalTag, version, dataTier )
dasQuery = 'file dataset=%s | grep file.name'%( dataset )
if debug:
print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
print ' \'%s\''%( dasQuery )
# partially stolen from das_client.py for option '--format=plain', needs filter ("grep") in the query
jsondict = self.runDAS(dasQuery,dasLimit)
if debug:
print '%s DEBUG: Received DAS JSON dictionary:'%( self._label )
print ' \'%s\''%( jsondict )
if jsondict[ 'status' ] != 'ok':
print 'There was a problem while querying DAS with query \'%s\'. Server reply was:\n %s' % (dasQuery, jsondict)
exit( 1 )
mongo_query = jsondict[ 'mongo_query' ]
filters = mongo_query[ 'filters' ]
data = jsondict[ 'data' ]
if debug:
print '%s DEBUG: Query in JSON dictionary:'%( self._label )
print ' \'%s\''%( mongo_query )
print '%s DEBUG: Filters in query:'%( self._label )
print ' \'%s\''%( filters )
print '%s DEBUG: Data in JSON dictionary:'%( self._label )
print ' \'%s\''%( data )
for row in data:
filePath = [ r for r in das_client.get_value( row, filters[ 'grep' ] ) ][ 0 ]
if debug:
print '%s DEBUG: Testing file entry \'%s\''%( self._label, filePath )
if len( filePath ) > 0:
if validVersion != version:
jsontestdict = self.runDAS('site dataset=%s | grep site.name' % ( dataset ), 999)
mongo_testquery = jsontestdict[ 'mongo_query' ]
testfilters = mongo_testquery[ 'filters' ]
testdata = jsontestdict[ 'data' ]
if debug:
print '%s DEBUG: Received DAS JSON dictionary (site test):'%( self._label )
print ' \'%s\''%( jsontestdict )
print '%s DEBUG: Query in JSON dictionary (site test):'%( self._label )
print ' \'%s\''%( mongo_testquery )
print '%s DEBUG: Filters in query (site test):'%( self._label )
print ' \'%s\''%( testfilters )
print '%s DEBUG: Data in JSON dictionary (site test):'%( self._label )
print ' \'%s\''%( testdata )
foundSE = False
for testrow in testdata:
siteName = [ tr for tr in das_client.get_value( testrow, testfilters[ 'grep' ] ) ][ 0 ]
if siteName == domainSE:
foundSE = True
break
if not foundSE:
if debug:
print '%s DEBUG: Possible version \'v%s\' not available on SE \'%s\''%( self._label, version, domainSE )
break
validVersion = version
if debug:
print '%s DEBUG: Valid version set to \'v%i\''%( self._label, validVersion )
if numberOfFiles == 0:
break
# protect from double entries ( 'unique' flag in query does not work here)
if not filePath in filePathsTmp:
filePathsTmp.append( filePath )
if debug:
print '%s DEBUG: File \'%s\' found'%( self._label, filePath )
fileCount += 1
# needed, since and "limit" overrides "idx" in 'get_data' (==> "idx" set to '0' rather than "skipFiles")
if fileCount > skipFiles:
filePaths.append( filePath )
elif debug:
print '%s DEBUG: File \'%s\' found again'%( self._label, filePath )
if validVersion > 0:
if numberOfFiles == 0 and debug:
print '%s DEBUG: No files requested'%( self._label )
break
else:
if debug:
print '%s DEBUG: Using DBS query'%( self._label )
print '%s WARNING: DBS query disabled for DBS3 transition to new API'%( self._label )
#for version in range( maxVersions, 0, -1 ):
#filePaths = []
#fileCount = 0
#dataset = '/%s/%s-%s-v%i/%s'%( relVal, cmsswVersion, globalTag, version, dataTier )
#dbsQuery = 'find file where dataset = %s'%( dataset )
#if debug:
#print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
#print ' \'%s\''%( dbsQuery )
#foundSE = False
#for line in os.popen( 'dbs search --query="%s"'%( dbsQuery ) ).readlines():
#if line.find( '.root' ) != -1:
#if validVersion != version:
#if not foundSE:
#dbsSiteQuery = 'find dataset where dataset = %s and site = %s'%( dataset, domainSE )
#if debug:
#print '%s DEBUG: Querying site \'%s\' with'%( self._label, domainSE )
#print ' \'%s\''%( dbsSiteQuery )
#for lineSite in os.popen( 'dbs search --query="%s"'%( dbsSiteQuery ) ).readlines():
#if lineSite.find( dataset ) != -1:
#foundSE = True
#break
#if not foundSE:
#if debug:
#print '%s DEBUG: Possible version \'v%s\' not available on SE \'%s\''%( self._label, version, domainSE )
#break
#validVersion = version
#if debug:
#print '%s DEBUG: Valid version set to \'v%i\''%( self._label, validVersion )
#if numberOfFiles == 0:
#break
#filePath = line.replace( '\n', '' )
#if debug:
#print '%s DEBUG: File \'%s\' found'%( self._label, filePath )
#fileCount += 1
#if fileCount > skipFiles:
#filePaths.append( filePath )
#if not numberOfFiles < 0:
#if numberOfFiles <= len( filePaths ):
#break
#if validVersion > 0:
#if numberOfFiles == 0 and debug:
#print '%s DEBUG: No files requested'%( self._label )
#break
# Check output and return
if validVersion == 0:
print '%s WARNING : No RelVal file(s) found at all in datasets \'%s*\' on SE \'%s\''%( self._label, datasetAll, domainSE )
if debug:
self.messageEmptyList()
elif len( filePaths ) == 0:
print '%s WARNING : No RelVal file(s) picked up in dataset \'%s\''%( self._label, dataset )
if debug:
self.messageEmptyList()
elif len( filePaths ) < numberOfFiles:
print '%s INFO : Only %i RelVal file(s) instead of %i picked up in dataset \'%s\''%( self._label, len( filePaths ), numberOfFiles, dataset )
if debug:
print '%s DEBUG: returning %i file(s):\n%s'%( self._label, len( filePaths ), filePaths )
return filePaths
def apply(self):
useDAS = self._parameters["useDAS"].value
cmsswVersion = self._parameters["cmsswVersion"].value
formerVersion = self._parameters["formerVersion"].value
relVal = self._parameters["relVal"].value
dataTier = self._parameters["dataTier"].value
condition = self._parameters[
"condition"
].value # only used for GT determination in initialization, if GT not explicitly given
globalTag = self._parameters["globalTag"].value
maxVersions = self._parameters["maxVersions"].value
skipFiles = self._parameters["skipFiles"].value
numberOfFiles = self._parameters["numberOfFiles"].value
debug = self._parameters["debug"].value
filePaths = []
# Determine corresponding CMSSW version for RelVals
preId = "_pre"
patchId = "_patch" # patch releases
hltPatchId = "_hltpatch" # HLT patch releases
dqmPatchId = "_dqmpatch" # DQM patch releases
slhcId = "_SLHC" # SLHC releases
rootId = "_root" # ROOT test releases
ibId = "_X_" # IBs
if patchId in cmsswVersion:
cmsswVersion = cmsswVersion.split(patchId)[0]
elif hltPatchId in cmsswVersion:
cmsswVersion = cmsswVersion.split(hltPatchId)[0]
elif dqmPatchId in cmsswVersion:
cmsswVersion = cmsswVersion.split(dqmPatchId)[0]
elif rootId in cmsswVersion:
cmsswVersion = cmsswVersion.split(rootId)[0]
elif slhcId in cmsswVersion:
cmsswVersion = cmsswVersion.split(slhcId)[0]
elif ibId in cmsswVersion or formerVersion:
outputTuple = Popen(["scram", "l -c CMSSW"], stdout=PIPE, stderr=PIPE).communicate()
if len(outputTuple[1]) != 0:
print "%s INFO : SCRAM error" % (self._label)
if debug:
print " from trying to determine last valid releases before '%s'" % (cmsswVersion)
print
print outputTuple[1]
print
self.messageEmptyList()
return filePaths
versions = {"last": "", "lastToLast": ""}
for line in outputTuple[0].splitlines():
version = line.split()[1]
if cmsswVersion.split(ibId)[0] in version or cmsswVersion.rpartition("_")[0] in version:
if not (
patchId in version
or hltPatchId in version
or dqmPatchId in version
or slhcId in version
or ibId in version
or rootId in version
):
versions["lastToLast"] = versions["last"]
versions["last"] = version
if version == cmsswVersion:
break
# FIXME: ordering of output problematic ('XYZ_pre10' before 'XYZ_pre2', no "formerVersion" for 'XYZ_pre1')
if formerVersion:
# Don't use pre-releases as "former version" for other releases than CMSSW_X_Y_0
if (
preId in versions["lastToLast"]
and not preId in versions["last"]
and not versions["last"].endswith("_0")
):
versions["lastToLast"] = versions["lastToLast"].split(preId)[
0
] # works only, if 'CMSSW_X_Y_0' esists ;-)
# Use pre-release as "former version" for CMSSW_X_Y_0
elif versions["last"].endswith("_0") and not (
preId in versions["lastToLast"] and versions["lastToLast"].startswith(versions["last"])
):
versions["lastToLast"] = ""
for line in outputTuple[0].splitlines():
version = line.split()[1]
versionParts = version.partition(preId)
if versionParts[0] == versions["last"] and versionParts[1] == preId:
versions["lastToLast"] = version
elif versions["lastToLast"] != "":
break
# Don't use CMSSW_X_Y_0 as "former version" for pre-releases
elif (
preId in versions["last"]
and not preId in versions["lastToLast"]
and versions["lastToLast"].endswith("_0")
):
versions["lastToLast"] = "" # no alternative :-(
cmsswVersion = versions["lastToLast"]
else:
cmsswVersion = versions["last"]
# Debugging output
if debug:
print "%s DEBUG: Called with..." % (self._label)
for key in self._parameters.keys():
print " %s:\t" % (key),
print self._parameters[key].value,
if self._parameters[key].value is self.getDefaultParameters()[key].value:
print " (default)"
else:
print
if key == "cmsswVersion" and cmsswVersion != self._parameters[key].value:
if formerVersion:
print " ==> modified to last to last valid release %s (s. 'formerVersion' parameter)" % (
cmsswVersion
)
else:
print " ==> modified to last valid release %s" % (cmsswVersion)
# Check domain
domain = socket.getfqdn().split(".")
domainSE = ""
if len(domain) == 0:
print "%s INFO : Cannot determine domain of this computer" % (self._label)
if debug:
self.messageEmptyList()
return filePaths
elif os.uname()[0] == "Darwin":
print "%s INFO : Running on MacOSX without direct access to RelVal files." % (self._label)
if debug:
self.messageEmptyList()
return filePaths
elif len(domain) == 1:
print "%s INFO : Running on local host '%s' without direct access to RelVal files" % (
self._label,
domain[0],
)
if debug:
self.messageEmptyList()
return filePaths
if not ((domain[-2] == "cern" and domain[-1] == "ch") or (domain[-2] == "fnal" and domain[-1] == "gov")):
print "%s INFO : Running on site '%s.%s' without direct access to RelVal files" % (
self._label,
domain[-2],
domain[-1],
)
if debug:
self.messageEmptyList()
return filePaths
if domain[-2] == "cern":
domainSE = "T2_CH_CERN"
elif domain[-2] == "fnal":
domainSE = "T1_US_FNAL_MSS"
if debug:
print "%s DEBUG: Running at site '%s.%s'" % (self._label, domain[-2], domain[-1])
print "%s DEBUG: Looking for SE '%s'" % (self._label, domainSE)
# Find files
validVersion = 0
dataset = ""
datasetAll = "/%s/%s-%s-v*/%s" % (relVal, cmsswVersion, globalTag, dataTier)
if useDAS:
if debug:
print "%s DEBUG: Using DAS query" % (self._label)
dasLimit = numberOfFiles
if dasLimit <= 0:
dasLimit += 1
for version in range(maxVersions, 0, -1):
filePaths = []
filePathsTmp = []
fileCount = 0
dataset = "/%s/%s-%s-v%i/%s" % (relVal, cmsswVersion, globalTag, version, dataTier)
dasQuery = "file dataset=%s | grep file.name" % (dataset)
if debug:
print "%s DEBUG: Querying dataset '%s' with" % (self._label, dataset)
print " '%s'" % (dasQuery)
# partially stolen from das_client.py for option '--format=plain', needs filter ("grep") in the query
jsondict = das_client.get_data("https://cmsweb.cern.ch", dasQuery, 0, dasLimit, False)
if debug:
print "%s DEBUG: Received DAS JSON dictionary:" % (self._label)
print " '%s'" % (jsondict)
if jsondict["status"] != "ok":
print "There was a problem while querying DAS with query '%s'. Server reply was:\n %s" % (
dasQuery,
jsondict,
)
exit(1)
mongo_query = jsondict["mongo_query"]
filters = mongo_query["filters"]
data = jsondict["data"]
if debug:
print "%s DEBUG: Query in JSON dictionary:" % (self._label)
print " '%s'" % (mongo_query)
print "%s DEBUG: Filters in query:" % (self._label)
print " '%s'" % (filters)
print "%s DEBUG: Data in JSON dictionary:" % (self._label)
print " '%s'" % (data)
for row in data:
filePath = [r for r in das_client.get_value(row, filters["grep"])][0]
if debug:
print "%s DEBUG: Testing file entry '%s'" % (self._label, filePath)
if len(filePath) > 0:
if validVersion != version:
jsontestdict = das_client.get_data(
"https://cmsweb.cern.ch", "site dataset=%s | grep site.name" % (dataset), 0, 999, False
)
mongo_testquery = jsontestdict["mongo_query"]
testfilters = mongo_testquery["filters"]
testdata = jsontestdict["data"]
if debug:
print "%s DEBUG: Received DAS JSON dictionary (site test):" % (self._label)
print " '%s'" % (jsontestdict)
print "%s DEBUG: Query in JSON dictionary (site test):" % (self._label)
print " '%s'" % (mongo_testquery)
print "%s DEBUG: Filters in query (site test):" % (self._label)
print " '%s'" % (testfilters)
print "%s DEBUG: Data in JSON dictionary (site test):" % (self._label)
print " '%s'" % (testdata)
foundSE = False
for testrow in testdata:
siteName = [tr for tr in das_client.get_value(testrow, testfilters["grep"])][0]
if siteName == domainSE:
foundSE = True
break
if not foundSE:
if debug:
print "%s DEBUG: Possible version 'v%s' not available on SE '%s'" % (
self._label,
version,
domainSE,
)
break
validVersion = version
if debug:
print "%s DEBUG: Valid version set to 'v%i'" % (self._label, validVersion)
if numberOfFiles == 0:
break
# protect from double entries ( 'unique' flag in query does not work here)
if not filePath in filePathsTmp:
filePathsTmp.append(filePath)
if debug:
print "%s DEBUG: File '%s' found" % (self._label, filePath)
fileCount += 1
# needed, since and "limit" overrides "idx" in 'get_data' (==> "idx" set to '0' rather than "skipFiles")
if fileCount > skipFiles:
filePaths.append(filePath)
elif debug:
print "%s DEBUG: File '%s' found again" % (self._label, filePath)
if validVersion > 0:
if numberOfFiles == 0 and debug:
print "%s DEBUG: No files requested" % (self._label)
break
else:
if debug:
print "%s DEBUG: Using DBS query" % (self._label)
print "%s WARNING: DBS query disabled for DBS3 transition to new API" % (self._label)
# for version in range( maxVersions, 0, -1 ):
# filePaths = []
# fileCount = 0
# dataset = '/%s/%s-%s-v%i/%s'%( relVal, cmsswVersion, globalTag, version, dataTier )
# dbsQuery = 'find file where dataset = %s'%( dataset )
# if debug:
# print '%s DEBUG: Querying dataset \'%s\' with'%( self._label, dataset )
# print ' \'%s\''%( dbsQuery )
# foundSE = False
# for line in os.popen( 'dbs search --query="%s"'%( dbsQuery ) ).readlines():
# if line.find( '.root' ) != -1:
# if validVersion != version:
# if not foundSE:
# dbsSiteQuery = 'find dataset where dataset = %s and site = %s'%( dataset, domainSE )
# if debug:
# print '%s DEBUG: Querying site \'%s\' with'%( self._label, domainSE )
# print ' \'%s\''%( dbsSiteQuery )
# for lineSite in os.popen( 'dbs search --query="%s"'%( dbsSiteQuery ) ).readlines():
# if lineSite.find( dataset ) != -1:
# foundSE = True
# break
# if not foundSE:
# if debug:
# print '%s DEBUG: Possible version \'v%s\' not available on SE \'%s\''%( self._label, version, domainSE )
# break
# validVersion = version
# if debug:
# print '%s DEBUG: Valid version set to \'v%i\''%( self._label, validVersion )
# if numberOfFiles == 0:
# break
# filePath = line.replace( '\n', '' )
# if debug:
# print '%s DEBUG: File \'%s\' found'%( self._label, filePath )
# fileCount += 1
# if fileCount > skipFiles:
# filePaths.append( filePath )
# if not numberOfFiles < 0:
# if numberOfFiles <= len( filePaths ):
# break
# if validVersion > 0:
# if numberOfFiles == 0 and debug:
# print '%s DEBUG: No files requested'%( self._label )
# break
# Check output and return
if validVersion == 0:
print "%s WARNING : No RelVal file(s) found at all in datasets '%s*' on SE '%s'" % (
self._label,
datasetAll,
domainSE,
)
if debug:
self.messageEmptyList()
elif len(filePaths) == 0:
print "%s WARNING : No RelVal file(s) picked up in dataset '%s'" % (self._label, dataset)
if debug:
self.messageEmptyList()
elif len(filePaths) < numberOfFiles:
print "%s INFO : Only %i RelVal file(s) instead of %i picked up in dataset '%s'" % (
self._label,
len(filePaths),
numberOfFiles,
dataset,
)
if debug:
print "%s DEBUG: returning %i file(s):\n%s" % (self._label, len(filePaths), filePaths)
return filePaths
files = []
# Query DAS
query = "file dataset=%(dataset)s run=%(run)i lumi=%(lumi)i | grep file.name" % event
jsondict = das_client.get_data('https://cmsweb.cern.ch', query, 0, 0, False)
status = jsondict['status']
if status != 'ok':
print "DAS query status: %s"%(status)
return files
mongo_query = jsondict['mongo_query']
filters = mongo_query['filters']
data = jsondict['data']
files = []
for row in data:
file = [r for r in das_client.get_value(row, filters['grep'])][0]
if len(file) > 0 and not file in files:
files.append(file)
return files
def fullCPMpath():
base = os.environ.get ('CMSSW_BASE')
if not base:
raise RuntimeError, "CMSSW Environment not set"
retval = "%s/src/PhysicsTools/Utilities/configuration/copyPickMerge_cfg.py" \
% base
if os.path.exists (retval):
return retval
base = os.environ.get ('CMSSW_RELEASE_BASE')