def injectFilesFromDBS(inputFileset, datasetPath, runsWhiteList=[]):
"""
_injectFilesFromDBS_
"""
print("injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name))
args = {}
args["url"] = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
args["version"] = "DBS_2_1_1"
args["mode"] = "GET"
dbsApi = DbsApi(args)
dbsResults = dbsApi.listFileArray(path=datasetPath, retriveList=["retrive_lumi", "retrive_run"])
print(" found %d files, inserting into wmbs..." % (len(dbsResults)))
for dbsResult in dbsResults:
if runsWhiteList and str(dbsResult["LumiList"][0]["RunNumber"]) not in runsWhiteList:
continue
myFile = File(lfn=dbsResult["LogicalFileName"], size=dbsResult["FileSize"],
events=dbsResult["NumberOfEvents"], checksums={"cksum": dbsResult["Checksum"]},
locations="cmssrm.fnal.gov", merged=True)
myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"])
for lumi in dbsResult["LumiList"]:
myRun.appendLumi(lumi["LumiSectionNumber"])
myFile.addRun(myRun)
myFile.create()
inputFileset.addFile(myFile)
if len(inputFileset) < 1:
raise Exception("No files were selected!")
inputFileset.commit()
inputFileset.markOpen(False)
return
def __init__(self, url, **contact):
args = {"url": url, "level": 'ERROR', "version": ''}
args.update(contact)
#try:
self.dbs = DbsApi(args)
#except DbsException, ex:
# msg = "Error in DBSReader with DbsApi\n"
# msg += "%s\n" % formatEx(ex)
# raise DBSReaderError(msg)
# setup DLS api - with either dbs or phedex depending on dbs instance
if url.count('cmsdbsprod.cern.ch/cms_dbs_prod_global') or \
self.dbs.getServerInfo()['InstanceName'] == 'GLOBAL':
dlsType = 'DLS_TYPE_PHEDEX'
dlsUrl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod'
else:
dlsType = 'DLS_TYPE_DBS'
dlsUrl = url
try:
self.dls = dlsClient.getDlsApi(dls_type=dlsType,
dls_endpoint=dlsUrl,
version=args['version'])
except DlsApiError, ex:
msg = "Error in DBSReader with DlsApi\n"
msg += "%s\n" % str(ex)
raise DBSReaderError(msg)
def main():
from optparse import OptionParser
usage="usage: python setDatasetStatus.py --dataset=<DATASET_NAME> --status=<STATUS> --url=<DBSURL> {--files}"
parser = OptionParser(usage=usage)
parser.add_option('-d', '--dataset', dest='dataset', default=None, help='Dataset name')
parser.add_option('-s', '--status', dest='status', default=None, help='This will be the new status of the dataset/files')
parser.add_option('-D', '--url', dest='url', default='https://cmsdbsprod.cern.ch:8443/cms_dbs_prod_global_writer/servlet/DBSServlet', help='DBS URL')
parser.add_option('-f', '--files', action="store_true", default=False, dest='files', help='Validate or invalidate all files in dataset')
(opts, args) = parser.parse_args()
if opts.dataset == None:
print "--dataset option must be provided"
print usage;
sys.exit(1)
if opts.status == None:
print "--status option must be provided"
print usage;
sys.exit(1)
if opts.url == None:
print "--url option not provided."
print "Using %s"%opts.url
dbsargs = {'url' : opts.url}
dbsapi = DbsApi(dbsargs)
try:
dbsapi.updateProcDSStatus(opts.dataset, opts.status)
except Exception, ex:
print "Caught Exception %s " % str(ex)
sys.exit(1)
def injectFilesFromDBS(inputFileset, datasetPath):
"""
_injectFilesFromDBS_
"""
print "injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name)
args={}
args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
dbsApi = DbsApi(args)
dbsResults = dbsApi.listFiles(path = datasetPath, retriveList = ["retrive_block","retrive_lumi", "retrive_run"])
# NOTE : this is to limit the number of jobs to create ... simply using first 10 files get for the needed dataset
dbsResults =dbsResults[0:2]
print " found %d files, inserting into wmbs..." % (len(dbsResults))
for dbsResult in dbsResults:
myFile = File(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"],
events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]},
locations = set(['srm.ciemat.es','storm-se-01.ba.infn.it','storage01.lcg.cscs.ch']))
myRun = Run(runNumber = dbsResult["LumiList"][0]["RunNumber"])
for lumi in dbsResult["LumiList"]:
myRun.lumis.append(lumi["LumiSectionNumber"])
myFile.addRun(myRun)
myFile.create()
inputFileset.addFile(myFile)
inputFileset.commit()
inputFileset.markOpen(False)
return
def __init__(self, config):
"""
Use Configuration object from config
"""
# Config should have DBSInterface element
self.config = config.DBSInterface
args = {
'url': self.config.DBSUrl,
'level': 'ERROR',
"user": '******',
'version': self.config.DBSVersion
}
self.version = self.config.DBSVersion
self.globalDBSUrl = None
self.committedRuns = collections.deque(maxlen=1000)
self.maxBlockFiles = self.config.DBSBlockMaxFiles
self.maxBlockTime = self.config.DBSBlockMaxTime
self.maxBlockSize = self.config.DBSBlockMaxSize
self.maxFilesToCommit = self.config.MaxFilesToCommit
self.doGlobalMigration = getattr(self.config, 'doGlobalMigration',
True)
if getattr(self.config, 'globalDBSUrl', None) != None:
globalArgs = {
'url': self.config.globalDBSUrl,
'level': 'ERROR',
"user": '******',
'version': self.config.globalDBSVersion
}
self.globalDBSUrl = self.config.globalDBSUrl
else:
self.doGlobalMigration = False
try:
self.dbs = DbsApi(args)
if self.globalDBSUrl:
self.globalDBS = DbsApi(globalArgs)
except DbsException, ex:
msg = "Error in DBSWriterError with DbsApi\n"
msg += "%s\n" % formatEx(ex)
logging.error(msg)
raise DBSInterfaceError(msg)
def __init__(self, args):
# just make sure args value complies with dbs args
try:
from DBSAPI.dbsApi import DbsApi
DbsApi(args)
except ImportError:
# No dbsApi available, carry on
pass
self.args = args
def __init__(self, url, **contact):
args = {"url": url, "level": 'ERROR'}
args.update(contact)
try:
self.dbs = DbsApi(args)
except DbsException, ex:
msg = "Error in DBSReader with DbsApi\n"
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def __init__(self, url, **contact):
args = {"url": url, "level": 'ERROR'}
args.update(contact)
try:
self.dbs = DbsApi(args)
self.args = args
self.version = args.get('version', None)
self.globalDBSUrl = args.get('globalDBSUrl', None)
self.globalVersion = args.get('globalVersion', None)
if self.globalDBSUrl:
globalArgs = {'url': url, 'level': 'ERROR'}
globalArgs.update(contact)
self.globalDBS = DbsApi(globalArgs)
except DbsException, ex:
msg = "Error in DBSWriterError with DbsApi\n"
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
def setStatusDBS3(url3, dataset3, newStatus, files):
from dbs.apis.dbsClient import DbsApi
dbsapi = DbsApi(url=url3)
try:
dbsapi.updateDatasetType(dataset=dataset3, dataset_access_type=newStatus)
except Exception, ex:
print "Caught Exception %s " % str(ex)
sys.exit(1)
def importParentDataset(self, globalDBS, datasetpath):
"""
WARNING: it works only with DBS_2_0_9_patch_6
"""
args = {'url': globalDBS}
try:
api_reader = DbsApi(args)
except DbsApiException, ex:
msg = "%s\n" % formatEx(ex)
raise CrabException(msg)
def getFileNames(event, dbsOptions={}):
# Query DBS
try:
api = DbsApi(dbsOptions)
query = "find file where dataset=%(dataset)s and run=%(run)i and lumi=%(lumi)i" % event
xmldata = api.executeQuery(query)
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(),
ex.getErrorMessage())
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
def setStatusDBS2(url2, dataset, newStatus, files):
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsOptions import DbsOptionParser
dbsargs = {'url': url2}
dbsapi = DbsApi(dbsargs)
try:
dbsapi.updateProcDSStatus(dataset, newStatus)
except Exception, ex:
print "Caught DBS2 Exception %s " % str(ex)
sys.exit(1)
def __init__(self):
#, srcURL, dstURL):
try:
srcURL = "http://cmsdbsdev1.cern.ch:8880/DBSANZ/servlet/DBSServlet"
dstURL = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
args = {}
args['url'] = srcURL
args['mode'] = 'POST'
args['version'] = 'DBS_2_0_5'
self.srcApi = DbsApi(args)
args['url'] = dstURL
self.dstApi = DbsApi(args)
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(),
ex.getErrorMessage())
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
def injectFilesFromDBS(inputFileset, datasetPath):
"""
_injectFilesFromDBS_
"""
print "injecting files from %s into %s, please wait..." % (
datasetPath, inputFileset.name)
args = {}
args[
"url"] = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
dbsApi = DbsApi(args)
dbsResults = dbsApi.listFiles(path=datasetPath,
retriveList=["retrive_lumi", "retrive_run"])
dbsResults = dbsResults[0:10]
print " found %d files, inserting into wmbs..." % (len(dbsResults))
for dbsResult in dbsResults:
myFile = File(lfn=dbsResult["LogicalFileName"],
size=dbsResult["FileSize"],
events=dbsResult["NumberOfEvents"],
checksums={"cksum": dbsResult["Checksum"]},
locations="cmssrm.fnal.gov",
merged=True)
myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"])
for lumi in dbsResult["LumiList"]:
myRun.lumis.append(lumi["LumiSectionNumber"])
myFile.addRun(myRun)
myFile.create()
inputFileset.addFile(myFile)
dbsFile = DBSBufferFile(lfn=dbsResult["LogicalFileName"],
size=dbsResult["FileSize"],
events=dbsResult["NumberOfEvents"],
checksums={"cksum": dbsResult["Checksum"]},
locations="cmssrm.fnal.gov",
status="LOCAL")
dbsFile.setDatasetPath(datasetPath)
dbsFile.setAlgorithm(appName="cmsRun",
appVer="Unknown",
appFam="Unknown",
psetHash="Unknown",
configContent="Unknown")
dbsFile.create()
inputFileset.commit()
inputFileset.markOpen(False)
return
def __init__(self, dbsUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"):
"""
Configure the feeder
"""
FeederImpl.__init__(self)
# DBS parameter
self.args = { "url" : dbsUrl, "level" : 'ERROR'}
self.dbs = DbsApi(self.args)
self.purgeTime = 96
self.reopenTime = 120
self.dbsReader = DBSReader(dbsUrl)
self.connectionAttempts = 5
def injectFilesFromDBS(inputFileset, datasetPath):
"""
_injectFilesFromDBS_
"""
print("injecting files from %s into %s, please wait..." %
(datasetPath, inputFileset.name))
args = {}
args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
args["mode"] = "GET"
dbsApi = DbsApi(args)
dbsResults = dbsApi.listFileArray(
path=datasetPath, retriveList=["retrive_lumi", "retrive_run"])
# Limiter on number of files
dbsResults = dbsResults[0:20]
print(" found %d files, inserting into wmbs..." % (len(dbsResults)))
for dbsResult in dbsResults:
myFile = File(lfn=dbsResult["LogicalFileName"],
size=dbsResult["FileSize"],
events=dbsResult["NumberOfEvents"],
checksums={"cksum": dbsResult["Checksum"]},
locations="cmssrm.fnal.gov",
merged=True)
myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"])
for lumi in dbsResult["LumiList"]:
myRun.appendLumi(lumi["LumiSectionNumber"])
myFile.addRun(myRun)
myFile.create()
inputFileset.addFile(myFile)
dbsFile = DBSBufferFile(lfn=dbsResult["LogicalFileName"],
size=dbsResult["FileSize"],
events=dbsResult["NumberOfEvents"],
checksums={"cksum": dbsResult["Checksum"]},
locations="cmssrm.fnal.gov",
status="NOTUPLOADED")
dbsFile.setDatasetPath(datasetPath)
dbsFile.setAlgorithm(appName="cmsRun",
appVer="Unknown",
appFam="Unknown",
psetHash="Unknown",
configContent="Unknown")
dbsFile.create()
inputFileset.commit()
inputFileset.markOpen(False)
return
def __init__(self, dbsUrl = \
"http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"):
"""
Configure the feeder
"""
FeederImpl.__init__(self)
# DBS parameter
self.args = {"url": dbsUrl, "level": 'ERROR'}
self.dbs = DbsApi(self.args)
self.purgeTime = 96
self.reopenTime = 120
self.dbsReader = DBSReader(dbsUrl)
self.connectionAttempts = 5
def getDataset(filename):
"Interface with the DBS API to get the dataset used"
from xml.dom.minidom import parseString
from DBSAPI.dbsApi import DbsApi
args = {}
args['url']='http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet'
args['version']='DBS_2_0_9'
args['mode']='POST'
api = DbsApi(args)
data = api.executeQuery("find dataset where file="+filename)
domresults = parseString(data)
dbs = domresults.getElementsByTagName('dbs')
result = dbs[0].getElementsByTagName('results')
rows=result[0].getElementsByTagName('row')
dataset=(rows[0].getElementsByTagName('dataset'))[0] #rows should have only one element and produce a one element array
node=(dataset.childNodes)[0] #childNodes should be a one element array
return str(node.data) #The output is in unicode, so it has to be translated through str
def get_dbs_info(toFind, requirements):
"Interface with the DBS API to get the whatever you want of a requirements. ALWAYS RETURN A LIST OF STRINGS"
args = {}
args['url']='http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet'
args['version']='DBS_2_0_9'
args['mode']='POST'
api = DbsApi(args)
data = api.executeQuery("find %s where %s" % (toFind, requirements))
domresults = parseString(data)
dbs = domresults.getElementsByTagName('dbs')
result = dbs[0].getElementsByTagName('results')
rows=result[0].getElementsByTagName('row')
retList = []
for row in rows:
resultXML = row.getElementsByTagName(toFind)[0]
node=(resultXML.childNodes)[0] #childNodes should be a one element array
retList.append(str(node.nodeValue))
return retList
def __init__(self, nodes, phedexURL = "https://cmsweb.cern.ch/phedex/datasvc/json/prod/fileReplicas", dbsURL = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"):
# Add the node specification the URL
nodeList = self.makelist( nodes )
for node in nodeList:
# for the first node need to be a bit different
if node == nodeList[0]:
self.nodeURL = phedexURL + "?node=%s" % node
else:
self.nodeURL += "&node=%s" % node
self.dbsURL = dbsURL
try:
#optManager = DbsOptionParser()
#(opts,args) = optManager.getOpt()
#opts[ 'url' ] = dbsURL
self.dbsapi = DbsApi( {'url': dbsURL } )#opts.__dict__)
except DbsApiException as ex:
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() )
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser
import pdb
try:
if (len(sys.argv) < 3) :
print "You need to provide Local instance and Global instance URLs"
sys.exit(1)
localArgs={}
globalArgs={}
localArgs['url']=sys.argv[1]
apiLocal = DbsApi(localArgs)
globalArgs['url']=sys.argv[1]
globalApi = DbsApi(globalArgs)
print "List ALL Processed Datasets from : %s " % localArgs['url']
localDatasets = apiLocal.listProcessedDatasets("*")
# Let us see if this dataset is in Global already
for localProc in localDatasets:
partialName = "/%s/%s" % (localProc['PrimaryDataset']['Name'], localProc['Name'])
print "*** Testing for Dataset %s/* " %partialName
globalCounterPart = globalApi.listProcessedDatasets(patternPrim=localProc['PrimaryDataset']['Name'],
patternProc=localProc['Name'])
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
# API Unit tests for the DBS JavaServer.
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser
try:
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
#print opts.__dict__
api = DbsApi(opts.__dict__)
#for block in api.listBlocks("/TestPrimary_001_20070315_03h12m26s/TestProcessed_20070315_03h12m26s/GEN-SIM"):
#for block in api.listBlocks("/test_primary_001/TestProcessedDS001/GEN-SIM"):
#for block in api.listBlocks("/chi1/CMSSW_1_6_7-CSA07-3268/GEN-SIM-DIGI-RAW"):
#for block in api.listBlocks("/dataset_PD_110/CRUZET3-v1-unmerged/RAW"):
for block in api.listBlocks(
"/RelValSingleMuPt10/CMSSW_2_1_2_IDEAL_V6_v3/GEN-SIM-RECO"):
#for block in api.listBlocks(block_name="/test_primary_001*"):
#for block in api.listBlocks("", "/TestPrimary_001_20070315_02h26m11s/TestProcessed_20070315_02h26m11s/GEN-SIM#016712"):
#for block in api.listBlocks("/test_primary_001/TestProcessedDS001/GEN-SIM", "/test_primary_001/TestProcessedDS001/GEN*"):
#for block in api.listBlocks("/TestPrimary_001_20070315_02h53m32s/TestPrimary_001_20070315_02h53m32s/GEN-SIM"):
#print "%s %s" % (block['Name'], block['StorageElementList'])
#print " %s" % block['Name']
print " %s" % block
results = []
for result in myThread.dbi.processData(sql):
results.extend(result.fetchall())
blocks = {}
for row in results:
if row[0] not in blocks:
blocks[row[0]] = []
blocks[row[0]].append(row[1])
args = {}
args["url"] = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
dbsApi = DbsApi(args)
badBlocks = []
badFiles = []
for blockName in blocks.keys():
print("%s:" % blockName)
blockFiles = []
try:
dbsFiles = dbsApi.listFileArray(blockName=blockName)
except Exception as ex:
dbsFiles = []
for dbsFile in dbsFiles:
blockFiles.append(dbsFile["LogicalFileName"])
if len(blockFiles) != len(blocks[blockName]):
except getopt.GetoptError:
print "Please specify --dataset"
sys.exit(2)
for myopt, myarg in myopts :
if myopt == "--dataset" :
datasetpath = myarg
if datasetpath == None:
print "Please specify --dataset"
sys.exit(2)
cern_se = 'srm.cern.ch'
try:
api = DbsApi(dbsargs)
try:
blocks = api.listBlocks(dataset=datasetpath,block_name="*",storage_element_name="*");
except:
blocks = []
for block in blocks:
for se in block['StorageElementList'] :
if se['Name'] != cern_se :
api.deleteReplicaFromBlock(block['Name'],se['Name'])
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(),
ex.getErrorMessage() )
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
"\nThis keywrod can be used for both source or target DBS" + \
"\npython dbsMigrateBlocks.py LOCALDBS TARGET_DBS_URL /CSA06-081-os-minbias/DIGI/CMSSW_0_8_1-GEN-SIM-DIGI-1154005302-merged" + \
"\n****************************************************************"
if (len(sys.argv) < 4) :
print usage
sys.exit(1)
op = "both"
if (len(sys.argv) == 5) :
op = sys.argv[4]
argsSrc['url'] = sys.argv[1]
argsTar['url'] = sys.argv[2]
if(argsSrc['url'] == "LOCALDBS" ):
apiSrc = DbsApi(opts.__dict__)
else :
apiSrc = DbsApi(argsSrc)
if(argsTar['url'] == "LOCALDBS" ):
apiTar = DbsApi(opts.__dict__)
else:
apiTar = DbsApi(argsTar)
path = sys.argv[3]
print path
#name = argsSrc['url'].replace('/','_') + "_" + argsTar['url'].replace('/', '_') + path.replace('/', '_')
#name = argsSrc['url'].replace('/','_').replace(':', '_') + "_" + argsTar['url'].replace('/', '_').replace(':', '_')
name = "Transfer_"
def __init__(self,
dls_endpoint=None,
verbosity=dlsApi.DLS_VERB_WARN,
**kwd):
"""
Constructor of the class. It creates a DBS interface object using the
specified dls_endpoint and optionally additional parameters in **kwd.
It also sets the verbosity of the DLS API.
Any additional arguments for the DBS interface may be specified as part
of **kwd. Otherwise they may be specified in a DBS configuration
file (see below). For every case, the arguments in the configuration
file take less precedence.
At the time of writing, the minimum argument required to build the DBS
interface is the DBS server endpoint. For others, please check DBS
client documentation.
The server endpoint is got from a string in the URL form, usually:
"http[s]://hname[:port]/path/to/DLS". This API tries to retrieve that
value from several sources (in this order):
- specified dls_endpoint
- DLS_ENDPOINT environmental variable
- specified URL argument (in **kwd)
- specified URL in the configuration file (see below)
- DLS catalog advertised in the Information System (if implemented)
Any other DBS-specific parameter can be specified in the following ways
(in order of precedence):
- variable within **kwd (version, for the DBS client version)
- variable within the DBS configuration file
The DBS configuration file (see DBS documentation) will be passed to the
DBS interface. It can specify server endpoint, logging level, and other
variables, but it is overwritten by values specified by other means
(as explained above). The config file will be searched in (in this order):
- specified dbs_client_config argument (in **kwd)
- DBS_CLIENT_CONFIG environmental variable
If the necessary arguments cannot be obtained in any of these ways, the
instantiation is denied and a DlsConfigError is raised.
The verbosity level affects invocations of all methods in this object.
See the dlsApi.DlsApi.setVerbosity method for information on accepted values.
The underlying DBS interface verbosity can be controlled via the corresponding
DBS configuration parameter (in **kwd or config file).
If the checkEndpoint (**kwd) is set to True, the provided endpoint is
checked. This makes sense where more than one query are to be made next.
For simple queries, any error in the endpoint will be noticed in the query
itself, so the check would be redundant and not efficient.
@exception DlsConfigError: if the DBS interface object cannot be set up correctly
@param dls_endpoint: the DLS server to be used, as a string "hname[:port]/path/to/DLS"
@param verbosity: value for the verbosity level
@param kwd: Flags:
- checkEndpoint: Boolean (default False) for testing of the DLS endpoint
- URL: DLS server (DBS) endpoint
- version: DBS client version
- dbs_client_config: config file for DBS interface to use
- any other DBS-specific
"""
# Keywords
checkEndpoint = False
if (kwd.has_key("checkEndpoint")):
checkEndpoint = kwd.get("checkEndpoint")
# Let the parent set the server endpoint (if possible) and verbosity
dlsApi.DlsApi.__init__(self, dls_endpoint, verbosity)
# If the server is there, put it in the dict to pass
kwd["url"] = self.server
# Now go for the configuration file
if kwd.has_key("dbs_client_config"):
self.dbscfg = kwd["dbs_client_config"]
if (self.dbscfg):
environ["DBS_CLIENT_CONFIG"] = self.dbscfg
# Don't do this. Leave DBS default (dealing with level in conf file is also missing)
# # Adjust DBS verbosity to WARNING (unless overriden)
# keys = map(str.upper, kwd.keys())
# if(not kwd.has_key("LEVEL")):
# kwd["level"] = 'WARNING'
# Try to create the DBS interface object (if there are missing params, it'll shout)
try:
self.dbsapi = DbsApi(kwd)
except (DbsApiException, DbsException), inst:
msg = "Could not set the interface to the DLS server. "
msg += "Error trying to instantiate DBS interface object"
self._mapException(inst, msg, msg, errorTolerant=False)
def _queryAndCompareWithDBS(self, pileupDict, defaultArguments, dbsUrl):
"""
pileupDict is a Python dictionary containing particular pileup
configuration information. Query DBS on given dataset contained
now in both input defaultArguments as well as in the pileupDict
and compare values.
"""
args = {}
# this should have been set in CMSSWStepHelper along with
# the pileup configuration
args["url"] = dbsUrl
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
dbsApi = DbsApi(args)
inputArgs = defaultArguments["PileupConfig"]
self.assertEqual(len(inputArgs), len(pileupDict),
"Number of pileup types different.")
for pileupType in inputArgs:
m = ("pileup type '%s' not in PileupFetcher-produced pileup "
"configuration: '%s'" % (pileupType, pileupDict))
self.failUnless(pileupType in pileupDict, m)
# now query DBS for compare actual results on files lists for each
# pileup type and dataset and location (storage element names)
# pileupDict is saved in the file and now comparing items of this
# configuration with actual DBS results, the structure of pileupDict:
# {"pileupTypeA": {"BlockA": {"FileList": [], "StorageElementNames": []},
# "BlockB": {"FileList": [], "StorageElementName": []}, ....}
for pileupType, datasets in inputArgs.items():
# this is from the pileup configuration produced by PileupFetcher
blockDict = pileupDict[pileupType]
m = "Number of datasets for pileup type '%s' is not equal." % pileupType
self.assertEqual(len(blockDict), len(datasets), m)
for dataset in datasets:
dbsFileBlocks = dbsApi.listBlocks(dataset=dataset)
fileList = [
] # list of files in the block (dbsFile["LogicalFileName"])
storageElemNames = [] # list of StorageElementName
for dbsFileBlock in dbsFileBlocks:
blockName = dbsFileBlock["Name"]
# each DBS block has a list under 'StorageElementList', iterate over
for storElem in dbsFileBlock["StorageElementList"]:
storageElemNames.append(storElem["Name"])
# now get list of files in the block
dbsFiles = dbsApi.listFiles(blockName=blockName)
for dbsFile in dbsFiles:
fileList.append(dbsFile["LogicalFileName"])
# now compare the sets:
m = ("StorageElementNames don't agree for pileup type '%s', "
"dataset '%s' in configuration: '%s'" %
(pileupType, dataset, pileupDict))
self.assertEqual(blockDict[blockName]["StorageElementNames"],
storageElemNames, m)
m = ("FileList don't agree for pileup type '%s', dataset '%s' "
" in configuration: '%s'" %
(pileupType, dataset, pileupDict))
self.assertEqual(blockDict[blockName]["FileList"], fileList)
sys.exit(0)
if opts['flag'] in ('', None):
print "You must specify a valid QIM, use --flag= or --help"
sys.exit(0)
if opts['value'] in ('', None):
print "You must specify a valid value: GOOD, BAD, UNKNOWN or a INTEGER value, use --value= or --help"
sys.exit(0)
flag = DbsDQFlag(
Name=opts['flag'],
Value=opts['value'],
)
run_dq = DbsRunLumiDQ(
RunNumber=opts['run'],
#LumiSectionNumber=123,
DQFlagList=[flag])
api = DbsApi(opts)
api.updateRunLumiDQ(opts['dataset'], [run_dq])
print "%s is set %s for run: %s" % (opts['flag'], opts['value'],
opts['run'])
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(),
ex.getErrorMessage())
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
srcURL = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
#dstURL = "http://cmssrv48.fnal.gov:8383/DBSlocal/servlet/DBSServlet"
#dstURL = "http://vocmsvm05.cern.ch:8880/INT2RG_admin/servlet/DBSServlet"
dstURL = "http://cmssrv17.fnal.gov:8989/DBSTEST/servlet/DBSServlet"
#path = "/DY_mumu_10/CMSSW_1_3_1-Spring07-1349/GEN-SIM-DIGI-RECO"
#path = "/Cosmics/Commissioning08-MW32_v1/RAW"
#path = "/RelValSinglePiPt100/CMSSW_3_0_0_pre7_IDEAL_30X_v1/GEN-SIM-RECO"
path = "/RelValSinglePiPt100/CMSSW_3_0_0_pre7_IDEAL_30X_v1/GEN-SIM-RECO"
try:
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
args = {}
args['url'] = dstURL
args['mode'] = 'POST'
api = DbsApi(args)
#api = DbsApi(opts.__dict__)
block = ""
if len(sys.argv) > 4:
block = sys.argv[4]
api.migrateDatasetContents(srcURL, dstURL, path, block, False, True)
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(),
ex.getErrorMessage())
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
print "Done"
SRC_URL = 'http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet' DST_URL = 'https://cmsdbsprod.cern.ch:8443/cms_dbs_ph_analysis_02_writer/servlet/DBSServlet' DATASET = '/MinimumBias/Commissioning10-PromptReco-v9/RECO' from DBSAPI.dbsApi import DbsApi DbsApi().migrateDatasetContents(SRC_URL, DST_URL, DATASET)
