def injectFilesFromDBS(inputFileset, datasetPath):
"""
_injectFilesFromDBS_
"""
print "injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name)
args={}
args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
dbsApi = DbsApi(args)
dbsResults = dbsApi.listFiles(path = datasetPath, retriveList = ["retrive_block","retrive_lumi", "retrive_run"])
# NOTE : this is to limit the number of jobs to create ... simply using first 10 files get for the needed dataset
dbsResults =dbsResults[0:2]
print " found %d files, inserting into wmbs..." % (len(dbsResults))
for dbsResult in dbsResults:
myFile = File(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"],
events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]},
locations = set(['srm.ciemat.es','storm-se-01.ba.infn.it','storage01.lcg.cscs.ch']))
myRun = Run(runNumber = dbsResult["LumiList"][0]["RunNumber"])
for lumi in dbsResult["LumiList"]:
myRun.lumis.append(lumi["LumiSectionNumber"])
myFile.addRun(myRun)
myFile.create()
inputFileset.addFile(myFile)
inputFileset.commit()
inputFileset.markOpen(False)
return
def dbsGetDatasetType( dataset ):
d = {}
d['url']='http://cmsdbsprod.cern.ch/cms_dbs_ph_analysis_02/servlet/DBSServlet'
d['quiet']=True
api = DbsApi( d )
print api.listDatasetSummary(dataset.name)
res = api.executeQuery('find datatype where dataset=%s' % dataset.name)
def __init__(self, url, **contact):
args = {"url": url, "level": 'ERROR', "version": ''}
args.update(contact)
#try:
self.dbs = DbsApi(args)
#except DbsException, ex:
# msg = "Error in DBSReader with DbsApi\n"
# msg += "%s\n" % formatEx(ex)
# raise DBSReaderError(msg)
# setup DLS api - with either dbs or phedex depending on dbs instance
if url.count('cmsdbsprod.cern.ch/cms_dbs_prod_global') or \
self.dbs.getServerInfo()['InstanceName'] == 'GLOBAL':
dlsType = 'DLS_TYPE_PHEDEX'
dlsUrl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod'
else:
dlsType = 'DLS_TYPE_DBS'
dlsUrl = url
try:
self.dls = dlsClient.getDlsApi(dls_type=dlsType,
dls_endpoint=dlsUrl,
version=args['version'])
except DlsApiError, ex:
msg = "Error in DBSReader with DlsApi\n"
msg += "%s\n" % str(ex)
raise DBSReaderError(msg)
def injectFilesFromDBS(inputFileset, datasetPath, runsWhiteList=[]):
"""
_injectFilesFromDBS_
"""
print("injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name))
args = {}
args["url"] = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
args["version"] = "DBS_2_1_1"
args["mode"] = "GET"
dbsApi = DbsApi(args)
dbsResults = dbsApi.listFileArray(path=datasetPath, retriveList=["retrive_lumi", "retrive_run"])
print(" found %d files, inserting into wmbs..." % (len(dbsResults)))
for dbsResult in dbsResults:
if runsWhiteList and str(dbsResult["LumiList"][0]["RunNumber"]) not in runsWhiteList:
continue
myFile = File(lfn=dbsResult["LogicalFileName"], size=dbsResult["FileSize"],
events=dbsResult["NumberOfEvents"], checksums={"cksum": dbsResult["Checksum"]},
locations="cmssrm.fnal.gov", merged=True)
myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"])
for lumi in dbsResult["LumiList"]:
myRun.appendLumi(lumi["LumiSectionNumber"])
myFile.addRun(myRun)
myFile.create()
inputFileset.addFile(myFile)
if len(inputFileset) < 1:
raise Exception("No files were selected!")
inputFileset.commit()
inputFileset.markOpen(False)
return
def injectFilesFromDBS(inputFileset, datasetPath, runsWhiteList=[]):
"""
_injectFilesFromDBS_
"""
print "injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name)
args={}
args["url"] = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
args["version"] = "DBS_2_1_1"
args["mode"] = "GET"
dbsApi = DbsApi(args)
dbsResults = dbsApi.listFiles(path = datasetPath, retriveList = ["retrive_lumi", "retrive_run"])
print " found %d files, inserting into wmbs..." % (len(dbsResults))
for dbsResult in dbsResults:
if runsWhiteList and str(dbsResult["LumiList"][0]["RunNumber"]) not in runsWhiteList:
continue
myFile = File(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"],
events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]},
locations = "cmssrm.fnal.gov", merged = True)
myRun = Run(runNumber = dbsResult["LumiList"][0]["RunNumber"])
for lumi in dbsResult["LumiList"]:
myRun.lumis.append(lumi["LumiSectionNumber"])
myFile.addRun(myRun)
myFile.create()
inputFileset.addFile(myFile)
if len(inputFileset) < 1:
raise Exception, "No files were selected!"
inputFileset.commit()
inputFileset.markOpen(False)
return
def main():
from optparse import OptionParser
usage="usage: python setDatasetStatus.py --dataset=<DATASET_NAME> --status=<STATUS> --url=<DBSURL> {--files}"
parser = OptionParser(usage=usage)
parser.add_option('-d', '--dataset', dest='dataset', default=None, help='Dataset name')
parser.add_option('-s', '--status', dest='status', default=None, help='This will be the new status of the dataset/files')
parser.add_option('-D', '--url', dest='url', default='https://cmsdbsprod.cern.ch:8443/cms_dbs_prod_global_writer/servlet/DBSServlet', help='DBS URL')
parser.add_option('-f', '--files', action="store_true", default=False, dest='files', help='Validate or invalidate all files in dataset')
(opts, args) = parser.parse_args()
if opts.dataset == None:
print "--dataset option must be provided"
print usage;
sys.exit(1)
if opts.status == None:
print "--status option must be provided"
print usage;
sys.exit(1)
if opts.url == None:
print "--url option not provided."
print "Using %s"%opts.url
dbsargs = {'url' : opts.url}
dbsapi = DbsApi(dbsargs)
try:
dbsapi.updateProcDSStatus(opts.dataset, opts.status)
except Exception, ex:
print "Caught Exception %s " % str(ex)
sys.exit(1)
def _queryAndCompareWithDBS(self, pileupDict, defaultArguments, dbsUrl):
"""
pileupDict is a Python dictionary containing particular pileup
configuration information. Query DBS on given dataset contained
now in both input defaultArguments as well as in the pileupDict
and compare values.
"""
args = {}
# this should have been set in CMSSWStepHelper along with
# the pileup configuration
args["url"] = dbsUrl
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
dbsApi = DbsApi(args)
inputArgs = defaultArguments["PileupConfig"]
self.assertEqual(len(inputArgs), len(pileupDict),
"Number of pileup types different.")
for pileupType in inputArgs:
m = ("pileup type '%s' not in PileupFetcher-produced pileup "
"configuration: '%s'" % (pileupType, pileupDict))
self.failUnless(pileupType in pileupDict, m)
# now query DBS for compare actual results on files lists for each
# pileup type and dataset and location (storage element names)
# pileupDict is saved in the file and now comparing items of this
# configuration with actual DBS results, the structure of pileupDict:
# {"pileupTypeA": {"BlockA": {"FileList": [], "StorageElementNames": []},
# "BlockB": {"FileList": [], "StorageElementName": []}, ....}
for pileupType, datasets in inputArgs.items():
# this is from the pileup configuration produced by PileupFetcher
blockDict = pileupDict[pileupType]
m = "Number of datasets for pileup type '%s' is not equal." % pileupType
self.assertEqual(len(blockDict), len(datasets), m)
for dataset in datasets:
dbsFileBlocks = dbsApi.listBlocks(dataset = dataset)
fileList = [] # list of files in the block (dbsFile["LogicalFileName"])
storageElemNames = [] # list of StorageElementName
for dbsFileBlock in dbsFileBlocks:
blockName = dbsFileBlock["Name"]
# each DBS block has a list under 'StorageElementList', iterate over
for storElem in dbsFileBlock["StorageElementList"]:
storageElemNames.append(storElem["Name"])
# now get list of files in the block
dbsFiles = dbsApi.listFiles(blockName = blockName)
for dbsFile in dbsFiles:
fileList.append(dbsFile["LogicalFileName"])
# now compare the sets:
m = ("StorageElementNames don't agree for pileup type '%s', "
"dataset '%s' in configuration: '%s'" % (pileupType, dataset, pileupDict))
self.assertEqual(blockDict[blockName]["StorageElementNames"], storageElemNames, m)
m = ("FileList don't agree for pileup type '%s', dataset '%s' "
" in configuration: '%s'" % (pileupType, dataset, pileupDict))
self.assertEqual(blockDict[blockName]["FileList"], fileList)
def setStatusDBS3(url3, dataset3, newStatus, files):
from dbs.apis.dbsClient import DbsApi
dbsapi = DbsApi(url=url3)
try:
dbsapi.updateDatasetType(dataset=dataset3, dataset_access_type=newStatus)
except Exception, ex:
print "Caught Exception %s " % str(ex)
sys.exit(1)
def setStatusDBS3(url3, dataset3, newStatus, files):
from dbs.apis.dbsClient import DbsApi
dbsapi = DbsApi(url=url3)
try:
dbsapi.updateDatasetType(dataset=dataset3, dataset_access_type=newStatus)
except Exception, ex:
print "Caught Exception %s " % str(ex)
sys.exit(1)
def getFileNames (event, dbsOptions = {}):
# Query DBS
try:
api = DbsApi (dbsOptions)
query = "find file where dataset=%(dataset)s and run=%(run)i and lumi=%(lumi)i" % event
xmldata = api.executeQuery(query)
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() )
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
def getFileNames(event, dbsOptions={}):
# Query DBS
try:
api = DbsApi(dbsOptions)
query = "find file where dataset=%(dataset)s and run=%(run)i and lumi=%(lumi)i" % event
xmldata = api.executeQuery(query)
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(),
ex.getErrorMessage())
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
def setStatusDBS2(url2, dataset, newStatus, files):
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsOptions import DbsOptionParser
dbsargs = {'url': url2}
dbsapi = DbsApi(dbsargs)
try:
dbsapi.updateProcDSStatus(dataset, newStatus)
except Exception, ex:
print "Caught DBS2 Exception %s " % str(ex)
sys.exit(1)
def setStatusDBS2(url2, dataset, newStatus, files):
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsOptions import DbsOptionParser
dbsargs = {"url": url2}
dbsapi = DbsApi(dbsargs)
try:
dbsapi.updateProcDSStatus(dataset, newStatus)
except Exception, ex:
print "Caught DBS2 Exception %s " % str(ex)
sys.exit(1)
def injectFilesFromDBS(inputFileset, datasetPath):
"""
_injectFilesFromDBS_
"""
print "injecting files from %s into %s, please wait..." % (
datasetPath, inputFileset.name)
args = {}
args[
"url"] = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
dbsApi = DbsApi(args)
dbsResults = dbsApi.listFiles(path=datasetPath,
retriveList=["retrive_lumi", "retrive_run"])
dbsResults = dbsResults[0:10]
print " found %d files, inserting into wmbs..." % (len(dbsResults))
for dbsResult in dbsResults:
myFile = File(lfn=dbsResult["LogicalFileName"],
size=dbsResult["FileSize"],
events=dbsResult["NumberOfEvents"],
checksums={"cksum": dbsResult["Checksum"]},
locations="cmssrm.fnal.gov",
merged=True)
myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"])
for lumi in dbsResult["LumiList"]:
myRun.lumis.append(lumi["LumiSectionNumber"])
myFile.addRun(myRun)
myFile.create()
inputFileset.addFile(myFile)
dbsFile = DBSBufferFile(lfn=dbsResult["LogicalFileName"],
size=dbsResult["FileSize"],
events=dbsResult["NumberOfEvents"],
checksums={"cksum": dbsResult["Checksum"]},
locations="cmssrm.fnal.gov",
status="LOCAL")
dbsFile.setDatasetPath(datasetPath)
dbsFile.setAlgorithm(appName="cmsRun",
appVer="Unknown",
appFam="Unknown",
psetHash="Unknown",
configContent="Unknown")
dbsFile.create()
inputFileset.commit()
inputFileset.markOpen(False)
return
def injectFilesFromDBS(inputFileset, datasetPath):
"""
_injectFilesFromDBS_
"""
print("injecting files from %s into %s, please wait..." %
(datasetPath, inputFileset.name))
args = {}
args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
args["mode"] = "GET"
dbsApi = DbsApi(args)
dbsResults = dbsApi.listFileArray(
path=datasetPath, retriveList=["retrive_lumi", "retrive_run"])
# Limiter on number of files
dbsResults = dbsResults[0:20]
print(" found %d files, inserting into wmbs..." % (len(dbsResults)))
for dbsResult in dbsResults:
myFile = File(lfn=dbsResult["LogicalFileName"],
size=dbsResult["FileSize"],
events=dbsResult["NumberOfEvents"],
checksums={"cksum": dbsResult["Checksum"]},
locations="cmssrm.fnal.gov",
merged=True)
myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"])
for lumi in dbsResult["LumiList"]:
myRun.appendLumi(lumi["LumiSectionNumber"])
myFile.addRun(myRun)
myFile.create()
inputFileset.addFile(myFile)
dbsFile = DBSBufferFile(lfn=dbsResult["LogicalFileName"],
size=dbsResult["FileSize"],
events=dbsResult["NumberOfEvents"],
checksums={"cksum": dbsResult["Checksum"]},
locations="cmssrm.fnal.gov",
status="NOTUPLOADED")
dbsFile.setDatasetPath(datasetPath)
dbsFile.setAlgorithm(appName="cmsRun",
appVer="Unknown",
appFam="Unknown",
psetHash="Unknown",
configContent="Unknown")
dbsFile.create()
inputFileset.commit()
inputFileset.markOpen(False)
return
def __init__(
self,
nodes,
phedexURL="https://cmsweb.cern.ch/phedex/datasvc/json/prod/fileReplicas",
dbsURL="https://cmsweb.cern.ch/dbs/prod/global/DBSReader",
):
# Add the node specification the URL
nodeList = self.makelist(nodes)
for node in nodeList:
# for the first node need to be a bit different
if node == nodeList[0]:
self.nodeURL = phedexURL + "?node=%s" % node
else:
self.nodeURL += "&node=%s" % node
self.dbsURL = dbsURL
try:
# optManager = DbsOptionParser()
# (opts,args) = optManager.getOpt()
# opts[ 'url' ] = dbsURL
self.dbsapi = DbsApi({"url": dbsURL}) # opts.__dict__)
except DbsApiException as ex:
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage())
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
def __init__(self):
paConfig = loadProdAgentConfiguration()
t0astDBConfig = paConfig.getConfig("Tier0DB")
self.t0astDBConn = Tier0DB.Tier0DB(t0astDBConfig, manageGlobal = True)
self.t0astDBConn.connect()
localDBSConfig = paConfig.getConfig("LocalDBS")
globalDBSConfig = paConfig.getConfig("GlobalDBSDLS")
# phedexConfig = paConfig.getConfig("PhEDExConfig")
self.localDBSUrl = localDBSConfig["DBSURL"]
self.globalDBSUrl = globalDBSConfig["DBSURL"]
# self.phedexDSUrl = phedexConfig["DataServiceURL"]
self.localDbsApi = DbsApi({'url':localDBSConfig["DBSURL"]})
self.globalDbsApi = DbsApi({'url':globalDBSConfig["DBSURL"]})
def getDataset(filename):
"Interface with the DBS API to get the dataset used"
from xml.dom.minidom import parseString
from DBSAPI.dbsApi import DbsApi
args = {}
args['url']='http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet'
args['version']='DBS_2_0_9'
args['mode']='POST'
api = DbsApi(args)
data = api.executeQuery("find dataset where file="+filename)
domresults = parseString(data)
dbs = domresults.getElementsByTagName('dbs')
result = dbs[0].getElementsByTagName('results')
rows=result[0].getElementsByTagName('row')
dataset=(rows[0].getElementsByTagName('dataset'))[0] #rows should have only one element and produce a one element array
node=(dataset.childNodes)[0] #childNodes should be a one element array
return str(node.data) #The output is in unicode, so it has to be translated through str
def get_good_dq_for(dataset):
try:
opts = {}
opts['url'] = str(DbsConfig(opts).url())
run_dq_search_criteria = []
full_dq_info = []
api = DbsApi(opts)
full_dq_info = {dataset: api.listRunLumiDQ(dataset=dataset, runLumiDQList=[run_dq_search_criteria], dqVersion="")}
good_dq_info, dq_keys = process_raw_dbs(full_dq_info)
return good_dq_info, dq_keys, full_dq_info
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() )
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
def __init__(self, config):
"""
Use Configuration object from config
"""
# Config should have DBSInterface element
self.config = config.DBSInterface
args = {
'url': self.config.DBSUrl,
'level': 'ERROR',
"user": '******',
'version': self.config.DBSVersion
}
self.version = self.config.DBSVersion
self.globalDBSUrl = None
self.committedRuns = collections.deque(maxlen=1000)
self.maxBlockFiles = self.config.DBSBlockMaxFiles
self.maxBlockTime = self.config.DBSBlockMaxTime
self.maxBlockSize = self.config.DBSBlockMaxSize
self.maxFilesToCommit = self.config.MaxFilesToCommit
self.doGlobalMigration = getattr(self.config, 'doGlobalMigration',
True)
if getattr(self.config, 'globalDBSUrl', None) != None:
globalArgs = {
'url': self.config.globalDBSUrl,
'level': 'ERROR',
"user": '******',
'version': self.config.globalDBSVersion
}
self.globalDBSUrl = self.config.globalDBSUrl
else:
self.doGlobalMigration = False
try:
self.dbs = DbsApi(args)
if self.globalDBSUrl:
self.globalDBS = DbsApi(globalArgs)
except DbsException, ex:
msg = "Error in DBSWriterError with DbsApi\n"
msg += "%s\n" % formatEx(ex)
logging.error(msg)
raise DBSInterfaceError(msg)
def get_dbs_info(toFind, requirements):
"Interface with the DBS API to get the whatever you want of a requirements. ALWAYS RETURN A LIST OF STRINGS"
args = {}
args['url']='http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet'
args['version']='DBS_2_0_9'
args['mode']='POST'
api = DbsApi(args)
data = api.executeQuery("find %s where %s" % (toFind, requirements))
domresults = parseString(data)
dbs = domresults.getElementsByTagName('dbs')
result = dbs[0].getElementsByTagName('results')
rows=result[0].getElementsByTagName('row')
retList = []
for row in rows:
resultXML = row.getElementsByTagName(toFind)[0]
node=(resultXML.childNodes)[0] #childNodes should be a one element array
retList.append(str(node.nodeValue))
return retList
def __init__(self, url, **contact):
args = {"url": url, "level": 'ERROR'}
args.update(contact)
try:
self.dbs = DbsApi(args)
except DbsException, ex:
msg = "Error in DBSReader with DbsApi\n"
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def __init__(self, url, **contact):
args = {"url": url, "level": 'ERROR'}
args.update(contact)
try:
self.dbs = DbsApi(args)
self.args = args
self.version = args.get('version', None)
self.globalDBSUrl = args.get('globalDBSUrl', None)
self.globalVersion = args.get('globalVersion', None)
if self.globalDBSUrl:
globalArgs = {'url': url, 'level': 'ERROR'}
globalArgs.update(contact)
self.globalDBS = DbsApi(globalArgs)
except DbsException, ex:
msg = "Error in DBSWriterError with DbsApi\n"
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
def __init__(self, args):
# just make sure args value complies with dbs args
try:
from DBSAPI.dbsApi import DbsApi
DbsApi(args)
except ImportError:
# No dbsApi available, carry on
pass
self.args = args
def importParentDataset(self, globalDBS, datasetpath):
"""
WARNING: it works only with DBS_2_0_9_patch_6
"""
args = {'url': globalDBS}
try:
api_reader = DbsApi(args)
except DbsApiException, ex:
msg = "%s\n" % formatEx(ex)
raise CrabException(msg)
def __init__(self):
#, srcURL, dstURL):
try:
srcURL = "http://cmsdbsdev1.cern.ch:8880/DBSANZ/servlet/DBSServlet"
dstURL = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
args = {}
args['url'] = srcURL
args['mode'] = 'POST'
args['version'] = 'DBS_2_0_5'
self.srcApi = DbsApi(args)
args['url'] = dstURL
self.dstApi = DbsApi(args)
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(),
ex.getErrorMessage())
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
def __init__(self, nodes, phedexURL = "https://cmsweb.cern.ch/phedex/datasvc/json/prod/fileReplicas", dbsURL = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"):
# Add the node specification the URL
nodeList = self.makelist( nodes )
for node in nodeList:
# for the first node need to be a bit different
if node == nodeList[0]:
self.nodeURL = phedexURL + "?node=%s" % node
else:
self.nodeURL += "&node=%s" % node
self.dbsURL = dbsURL
try:
#optManager = DbsOptionParser()
#(opts,args) = optManager.getOpt()
#opts[ 'url' ] = dbsURL
self.dbsapi = DbsApi( {'url': dbsURL } )#opts.__dict__)
except DbsApiException as ex:
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() )
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
def injectFilesFromDBS(inputFileset, datasetPath):
"""
_injectFilesFromDBS_
"""
print "injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name)
args={}
args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
dbsApi = DbsApi(args)
dbsResults = dbsApi.listFileArray(path = datasetPath, retriveList = ["retrive_lumi", "retrive_run"])
dbsResults = dbsResults[0:10]
print " found %d files, inserting into wmbs..." % (len(dbsResults))
for dbsResult in dbsResults:
myFile = File(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"],
events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]},
locations = "cmssrm.fnal.gov", merged = True)
myRun = Run(runNumber = dbsResult["LumiList"][0]["RunNumber"])
for lumi in dbsResult["LumiList"]:
myRun.lumis.append(lumi["LumiSectionNumber"])
myFile.addRun(myRun)
myFile.create()
inputFileset.addFile(myFile)
dbsFile = DBSBufferFile(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"],
events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]},
locations = "cmssrm.fnal.gov", status = "LOCAL")
dbsFile.setDatasetPath(datasetPath)
dbsFile.setAlgorithm(appName = "cmsRun", appVer = "Unknown", appFam = "Unknown",
psetHash = "Unknown", configContent = "Unknown")
dbsFile.create()
inputFileset.commit()
inputFileset.markOpen(False)
return
def __init__(self, dbsUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"):
"""
Configure the feeder
"""
FeederImpl.__init__(self)
# DBS parameter
self.args = { "url" : dbsUrl, "level" : 'ERROR'}
self.dbs = DbsApi(self.args)
self.purgeTime = 96
self.reopenTime = 120
self.dbsReader = DBSReader(dbsUrl)
self.connectionAttempts = 5
def __init__(self, dbsUrl = \
"http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"):
"""
Configure the feeder
"""
FeederImpl.__init__(self)
# DBS parameter
self.args = {"url": dbsUrl, "level": 'ERROR'}
self.dbs = DbsApi(self.args)
self.purgeTime = 96
self.reopenTime = 120
self.dbsReader = DBSReader(dbsUrl)
self.connectionAttempts = 5
def __init__(self, url, **contact):
args = { "url" : url, "level" : 'ERROR'}
args.update(contact)
try:
self.dbs = DbsApi(args)
self.args = args
self.version = args.get('version', None)
self.globalDBSUrl = args.get('globalDBSUrl', None)
self.globalVersion = args.get('globalVersion', None)
if self.globalDBSUrl:
globalArgs = {'url': url, 'level': 'ERROR'}
globalArgs.update(contact)
self.globalDBS = DbsApi(globalArgs)
except DbsException as ex:
msg = "Error in DBSWriterError with DbsApi\n"
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
self.reader = DBSReader(**args)
def __init__(self, config):
"""
Use Configuration object from config
"""
# Config should have DBSInterface element
self.config = config.DBSInterface
args = {'url': self.config.DBSUrl,
'level': 'ERROR',
"user" :'NORMAL',
'version': self.config.DBSVersion}
self.version = self.config.DBSVersion
self.globalDBSUrl = None
self.committedRuns = collections.deque(maxlen = 1000)
self.maxFilesToCommit = self.config.MaxFilesToCommit
self.doGlobalMigration = getattr(self.config, 'doGlobalMigration', True)
if getattr(self.config, 'globalDBSUrl', None) != None:
globalArgs = {'url': self.config.globalDBSUrl,
'level': 'ERROR',
"user" :'NORMAL',
'version': self.config.globalDBSVersion}
self.globalDBSUrl = self.config.globalDBSUrl
else:
self.doGlobalMigration = False
try:
self.dbs = DbsApi(args)
if self.globalDBSUrl:
self.globalDBS = DbsApi(globalArgs)
except DbsException as ex:
msg = "Error in DBSWriterError with DbsApi\n"
msg += "%s\n" % formatEx(ex)
logging.error(msg)
raise DBSInterfaceError(msg)
return
def __init__(self, url, **contact):
args = {"url": url, "level": "ERROR", "version": ""}
args.update(contact)
# try:
self.dbs = DbsApi(args)
# except DbsException, ex:
# msg = "Error in DBSReader with DbsApi\n"
# msg += "%s\n" % formatEx(ex)
# raise DBSReaderError(msg)
# setup DLS api - with either dbs or phedex depending on dbs instance
if url.count("cmsdbsprod.cern.ch/cms_dbs_prod_global") or self.dbs.getServerInfo()["InstanceName"] == "GLOBAL":
dlsType = "DLS_TYPE_PHEDEX"
dlsUrl = "https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
else:
dlsType = "DLS_TYPE_DBS"
dlsUrl = url
try:
self.dls = dlsClient.getDlsApi(dls_type=dlsType, dls_endpoint=dlsUrl, version=args["version"])
except DlsApiError, ex:
msg = "Error in DBSReader with DlsApi\n"
msg += "%s\n" % str(ex)
raise DBSReaderError(msg)
class DBSReader:
"""
_DBSReader_
General API for reading data from DBS
"""
def __init__(self, url, **contact):
args = { "url" : url, "level" : 'ERROR', "version" : ''}
args.update(contact)
#try:
self.dbs = DbsApi(args)
#except DbsException, ex:
# msg = "Error in DBSReader with DbsApi\n"
# msg += "%s\n" % formatEx(ex)
# raise DBSReaderError(msg)
# setup DLS api - with either dbs or phedex depending on dbs instance
if url.count('cmsdbsprod.cern.ch/cms_dbs_prod_global') or \
self.dbs.getServerInfo()['InstanceName'] == 'GLOBAL':
dlsType = 'DLS_TYPE_PHEDEX'
dlsUrl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod'
else:
dlsType = 'DLS_TYPE_DBS'
dlsUrl = url
try:
self.dls = dlsClient.getDlsApi(dls_type = dlsType, dls_endpoint = dlsUrl, version = args['version'])
except DlsApiError, ex:
msg = "Error in DBSReader with DlsApi\n"
msg += "%s\n" % str(ex)
raise DBSReaderError(msg)
except DbsException, ex:
msg = "Error in DBSReader with DbsApi\n"
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
#!/usr/bin/env python
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser
optManager = DbsOptionParser()
(opts,args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
try:
print api.remap('/RelValSingleMuPt10/CMSSW_2_1_0_pre10_IDEAL_V5_v4/GEN-SIM-DIGI-RAW-HLTDEBUG', '/RelValSingleMuPt10/CMSSW_2_1_0_pre10_IDEAL_V5_v4/GEN-SIM-DIGI-RAW-HLTDEBUG-RECO', '/RelValSingleMuPt10/CMSSW_2_1_0_pre10_IDEAL_V5_v4/GEN-SIM-DIGI-RAW-HLTDEBUG#870b391e-5590-40c5-a063-8109b1a23a8f')
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() )
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
print "Done"
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
# API Unit tests for the DBS JavaServer.
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser
try:
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
#print opts.__dict__
api = DbsApi(opts.__dict__)
#for block in api.listBlocks("/TestPrimary_001_20070315_03h12m26s/TestProcessed_20070315_03h12m26s/GEN-SIM"):
#for block in api.listBlocks("/test_primary_001/TestProcessedDS001/GEN-SIM"):
#for block in api.listBlocks("/chi1/CMSSW_1_6_7-CSA07-3268/GEN-SIM-DIGI-RAW"):
#for block in api.listBlocks("/dataset_PD_110/CRUZET3-v1-unmerged/RAW"):
for block in api.listBlocks(
"/RelValSingleMuPt10/CMSSW_2_1_2_IDEAL_V6_v3/GEN-SIM-RECO"):
#for block in api.listBlocks(block_name="/test_primary_001*"):
#for block in api.listBlocks("", "/TestPrimary_001_20070315_02h26m11s/TestProcessed_20070315_02h26m11s/GEN-SIM#016712"):
#for block in api.listBlocks("/test_primary_001/TestProcessedDS001/GEN-SIM", "/test_primary_001/TestProcessedDS001/GEN*"):
#for block in api.listBlocks("/TestPrimary_001_20070315_02h53m32s/TestPrimary_001_20070315_02h53m32s/GEN-SIM"):
#print "%s %s" % (block['Name'], block['StorageElementList'])
#print " %s" % block['Name']
print " %s" % block
results = []
for result in myThread.dbi.processData(sql):
results.extend(result.fetchall())
blocks = {}
for row in results:
if row[0] not in blocks:
blocks[row[0]] = []
blocks[row[0]].append(row[1])
args = {}
args["url"] = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
dbsApi = DbsApi(args)
badBlocks = []
badFiles = []
for blockName in blocks.keys():
print("%s:" % blockName)
blockFiles = []
try:
dbsFiles = dbsApi.listFileArray(blockName=blockName)
except Exception as ex:
dbsFiles = []
for dbsFile in dbsFiles:
blockFiles.append(dbsFile["LogicalFileName"])
if len(blockFiles) != len(blocks[blockName]):
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
#
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset
from DBSAPI.dbsOptions import DbsOptionParser
from DBSAPI.dbsDQFlag import DbsDQFlag
from DBSAPI.dbsRunLumiDQ import DbsRunLumiDQ
optManager = DbsOptionParser()
(opts,args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
def print_flags_nice(dqHierarchyList):
if len(dqHierarchyList) <= 0:
print "No DQ information for this run found"
for aDQ in dqHierarchyList:
print "\nRunNumber: ", aDQ['RunNumber']
print "LumiSectionNumber: ", aDQ['LumiSectionNumber']
for aSubDQ in aDQ['DQFlagList']:
print " ", aSubDQ['Name'], aSubDQ['Value']
for aSubSubDQ in aSubDQ['SubSysFlagList']:
print " ", aSubSubDQ['Name'], aSubSubDQ['Value']
for abSubSubDQ in aSubSubDQ['SubSysFlagList'] :
print " ", abSubSubDQ['Name'], abSubSubDQ['Value']
#!/usr/bin/env python
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
# API Unit tests for the DBS JavaServer.
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser
try:
optManager = DbsOptionParser()
(opts,args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
for dataset in api.listDatasetPaths():
print "\n %s" %str(dataset)
print "Dataset parent: %s" %str(api.listPathParents)
for block in api.listBlocks(dataset):
print "block: %s" %str(block['Name'])
for parent in api.listBlockParents(block_name=block['Name']):
print "Parent: %s" % str(parent['Name'])
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() )
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
#!/usr/bin/env python
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
# API Unit tests for the DBS JavaServer.
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser
try:
optManager = DbsOptionParser()
(opts,args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
xml = api.listDatasetContents("/test_primary_001/TestProcessedDS001/SIM", "/test_primary_001/TestProcessedDS001/SIM#12345");
print xml
print api.insertDatasetContents(xml)
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() )
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
import time
import sys
import random
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser
dstURL = sys.argv[1]
optManager = DbsOptionParser()
(opts,args) = optManager.getOpt()
args = {}
args['url'] = dstURL
args['mode']='POST'
api = DbsApi(args)
try:
print "\n\nListing Datasets "
paths = api.listDatasetPaths()
ranIndex = random.randint(1,len(paths))
#myDataset = paths[ranIndex]
myDataset = "/Wjets-sherpa/Summer08_IDEAL_V12_v1/GEN-SIM-RAW"
print "Selected dataset is %s", myDataset
print "\nListing Blocks "
blocks = api.listBlocks(myDataset)
#print blocks
print "\n Listing Files "
#!/usr/bin/env python
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
try:
print api.insertFileParent("NEW_TEST0005xxxxxxxxxx6", "NEW_TEST0002")
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(),
ex.getErrorMessage())
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
print "Done"
import sys
import os
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser
from DBSAPI.dbsApi import DbsApi
try:
optManager = DbsOptionParser()
(opts,args) = optManager.getOpt()
args = {}
args['url']='http://cmssrv17.fnal.gov:8989/DBS_1_0_5_STABLE/servlet/DBSServlet'
#args['version']='DBS_1_0_7'
args['mode']='POST'
api = DbsApi(args)
#api = DbsApi(opts.__dict__)
srcURL = sys.argv[1]
dstURL = sys.argv[2]
path = sys.argv[3]
block = ""
if len(sys.argv) > 4 :
block = sys.argv[4]
api.migrateDatasetContents(srcURL, dstURL, path, block , False, True)
#print api.listPrimaryDatasets();
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() )
if ex.getErrorCode() not in (None, ""):
#!/usr/bin/env python
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
#
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser
try:
optManager = DbsOptionParser()
(opts,args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
print "Analysis Daatset...."
#for analysis in api.listAnalysisDataset("*t005", "/test_primary_001/TestProcessedDS001/SIM"):
#for analysis in api.listAnalysisDataset(version=1):
#for analysis in api.listAnalysisDataset(path="/RelVal131QCD_pt600_800/CMSSW_1_3_1-1176201507/GEN-SIM-DIGI-RECO", version=0):
for analysis in api.listAnalysisDataset():
#print " %s" % analysis
print " %s, %s" % (analysis['Name'], analysis['Version'])
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() )
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
print "Done"
#!/usr/bin/env python
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsRun import DbsRun
from DBSAPI.dbsOptions import DbsOptionParser
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
try:
#select Dataset, Complete from ProcDSRuns where Run=361;
#run=62243
run = ""
path = '/Cosmics/BeamCommissioning08_CRUZET4_V4P_CSCSkim_trial_v3/RECO'
for runStatus in api.listProcDSRunStatus(path, run):
print "RunNumber: " + str(runStatus['RunNumber']) + " Status:" + str(
runStatus['Done'])
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(),
ex.getErrorMessage())
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsAlgorithm import DbsAlgorithm
from DBSAPI.dbsFileBlock import DbsFileBlock
from DBSAPI.dbsFile import DbsFile
from DBSAPI.dbsLumiSection import DbsLumiSection
from DBSAPI.dbsQueryableParameterSet import DbsQueryableParameterSet
from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset
from DBSAPI.dbsProcessedDataset import DbsProcessedDataset
from DBSAPI.dbsOptions import DbsOptionParser
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
try:
merge_algo = DbsAlgorithm(
ExecutableName="EdmFastMerge",
ApplicationVersion="v101",
ApplicationFamily="Merge",
)
path = "/test_primary_001/TestProcessedDS001/SIM"
merge_proc = api.insertMergedDataset(path, "ThisISMergedDataset001",
merge_algo)
# File will go into THIS Block
block = DbsFileBlock(StorageElement=['test1', 'test3'],
Name="/test_primary_001/TestProcessedDS001/SIM#12345")
import sys
import random
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset
from DBSAPI.dbsFileBlock import DbsFileBlock
from DBSAPI.dbsProcessedDataset import DbsProcessedDataset
from DBSAPI.dbsOptions import DbsOptionParser
optManager = DbsOptionParser()
(opts,args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
#primary = DbsPrimaryDataset (Name = "test_primary_anzar_03")
#primary = DbsPrimaryDataset (Name = "TestPrimary1164750596.79")
primary = DbsPrimaryDataset (Name = "test_primary_anzar_001")
proc = DbsProcessedDataset (
PrimaryDataset=primary,
Name="TestProcessedDS001",
TierList=['SIM', 'RECO'],
)
#ran = str(int(random.random()*10000000))
#block = DbsFileBlock (
# Name= "/this/isaqqqqqqqqqxstbddddlock#016712"+ ran,
# )
#!/usr/bin/env python
#
# API Unit tests for the DBS JavaServer.
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser
try:
optManager = DbsOptionParser()
(opts,args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
for block in api.listBlocks("/test_primary_anzar_001/SIM/TestProcessedDS001"):
#for block in api.listBlocks("/TestPrimary1167862926.47/SIM1167862926.47/TestProcessed1167862926.47", "/*hahah#12345"):
#for block in api.listBlocks("/TestPrimary1167862926.47/SIM1167862926.47/TestProcessed1167862926.47", "/this/*"):
#for block in api.listBlocks("/TestPrimary1167862926.47/SIM1167862926.47/TestProcessed1167862926.47", "/this/ff*"):
#for block in api.listBlocks("/TestPrimary1167862926.47/SIM1167862926.47/TestProcessed1167862926.47", "/this/hahah#12345"):
#for block in api.listBlocks("/TestPrimary1167862926.47/SIM1167862926.47/TestProcessed1167862926.47", "/this/hahah#12345"):
print " %s" % block
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() )
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
#!/usr/bin/env python
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
#
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser
try:
optManager = DbsOptionParser()
(opts,args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
print "Analysis Daatset Defs...."
#for analysisdef in api.listAnalysisDatasetDefinition("*"):
for analysisdef in api.listAnalysisDatasetDefinition():
print " %s" % analysisdef
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() )
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
print "Done"
if (block != None) and (datasetPath != None):
print "\n options --block or --datasetPath are mutually exclusive"
print usage
sys.exit(1)
print ">>>>> DBS URL : %s"%(url)
import logging
logging.disable(logging.INFO)
# //
# // Get API to DBS
#//
args = {'url' : url , 'level' : 'ERROR'}
dbsapi = DbsApi(args)
# //
# // Delete dataset
#//
if (datasetPath):
print "Deleting datasetPath=%s"%datasetPath
dbsapi.deleteProcDS(datasetPath)
if (block):
dbsreader = DBSReader(url)
getdatasetPath = dbsreader.blockToDatasetPath(block)
print "Deleting block=%s from datasetPath=%s"%(block,getdatasetPath)
dbsapi.deleteBlock(getdatasetPath,block)
results = []
for result in myThread.dbi.processData(sql):
results.extend(result.fetchall())
blocks = {}
for row in results:
if not blocks.has_key(row[0]):
blocks[row[0]] = []
blocks[row[0]].append(row[1])
args = {}
args["url"] = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
dbsApi = DbsApi(args)
badBlocks = []
badFiles = []
for blockName in blocks.keys():
print "%s:" % blockName
blockFiles = []
try:
dbsFiles = dbsApi.listFiles(blockName = blockName)
except Exception, ex:
dbsFiles = []
for dbsFile in dbsFiles:
blockFiles.append(dbsFile["LogicalFileName"])
if len(blockFiles) != len(blocks[blockName]):
#!/usr/bin/env python
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
#
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser
try:
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
try:
path = "/TestPrimary_002_20070205_11h52m29s/SIM_20070205_11h52m29s/TestProcessed_20070205_11h52m29s"
token = path.split("/")
proc = api.listProcessedDatasets(token[1], token[2], token[3])[0]
print "proc fetched from DBS %s" % proc
proc['Name'] = "TestProcessed_20070205_11h52m29s_MERGED"
proc['ParentList'] = [path]
print "proc modified to be inserted in DBS %s" % proc
api.insertProcessedDataset(proc)
print "Result: %s" % proc
except DbsDatabaseError, e:
print e