def __init__(self): """ Construct api object. """ ## cgi service API DEFAULT_URL = "http://cmsdoc.cern.ch/cms/aprom/DBS/CGIServer/prodquery" args = {} args['instance']="Dev/fanfani" self.api = DbsCgiApi(DEFAULT_URL, args)
class DBSInfo_EDM: def __init__(self): """ Construct api object. """ ## cgi service API DEFAULT_URL = "http://cmsdoc.cern.ch/cms/aprom/DBS/CGIServer/prodquery" args = {} args['instance']="Dev/fanfani" self.api = DbsCgiApi(DEFAULT_URL, args) ## set log level # self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_INFO_) #self.api.setLogLevel(dbsApi.DBS_LOG_LEVEL_QUIET_) def getMatchingDatasets (self, datasetPath): """ Query DBS to get provenance """ try: list = self.api.listProcessedDatasets("%s" %datasetPath) except dbsApi.InvalidDataTier, ex: raise DBSInvalidDataTierError(ex.getClassName(),ex.getErrorMessage()) except dbsApi.DbsApiException, ex: raise DBSError(ex.getClassName(),ex.getErrorMessage())
from dbsFile import DbsFile from dbsFileBlock import DbsFileBlock from dbsEventCollection import DbsEventCollection from dbsPrimaryDataset import DbsPrimaryDataset from dbsProcessedDataset import DbsProcessedDataset from dbsProcessing import DbsProcessing from dbsApi import DbsApi, DbsApiException, InvalidDataTier # DEFAULT_URL = "http://cmsdoc.cern.ch/cms/aprom/DBS/CGIServer/prodquery" DEFAULT_URL = "exec:../../Servers/CGIServer/prodquery" try: args = {} if len(sys.argv) == 2: args["instance"] = sys.argv[1] api = DbsCgiApi(DEFAULT_URL, args) primary = DbsPrimaryDataset(datasetName="test_primary_anzar") print "" processing = DbsProcessing( primaryDataset=primary, processingName="test_process_anzar", applicationConfig={ "application": {"executable": "testexe", "version": "test", "family": "test"}, "parameterSet": {"hash": "test", "content": "test"}, }, ) print "Creating processing %s" % processing try: api.createProcessing(processing)
# Unit tests for the DBS CGI implementation. import sys from dbsCgiApi import DbsCgiApi from dbsException import DbsException from dbsFileBlock import DbsFileBlock from dbsApi import DbsApi, DbsApiException, InvalidDataTier DEFAULT_URL = "http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquerytest2" #DEFAULT_URL = "http://cmsdoc.cern.ch/cms/aprom/DBS/CGIServer/dbsxml" # DEFAULT_URL = "exec:../../Servers/CGIServer/dbsxml" try: args = {} if len(sys.argv) == 2: args['instance'] = sys.argv[1] api = DbsCgiApi(DEFAULT_URL, args) #api.setLogLevel(DBS_LOG_LEVEL_ALL_) # api.setDebug(1) # List all datasets and count of files in all their blocks """ print "Listing datasets blocks/files" blocks = files = 0 for dataset in api.listDatasets(): # "/*/Hit/*" print "Files for: %s" % dataset for block in api.getDatasetFileBlocks (dataset): blocks += 1 files += len(block.getFileList()) print " %s: %d files" % (block.getBlockName(), len(block.getFileList())) print >> sys.stderr, "%d blocks, %d files" % (blocks, files)
from dbsPrimaryDataset import DbsPrimaryDataset from dbsProcessedDataset import DbsProcessedDataset from dbsProcessing import DbsProcessing from dbsApi import DbsApi, DbsApiException, InvalidDataTier #DEFAULT_URL = "http://cmsdoc.cern.ch/cms/aprom/DBS/CGIServer/prodquery" DEFAULT_URL = "http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquerytest2" #DEFAULT_URL = "http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquery" #DEFAULT_URL = "exec:/home/sekhri/cgi/java/test/in.sh" #DEFAULT_URL = "exec:../CGIServer/prodquery" try: args = {} if len(sys.argv) == 2: args['instance'] = sys.argv[1] api = DbsCgiApi(DEFAULT_URL, args) #api.setLogLevel(DBS_LOG_LEVEL_ALL_) # api.setDebug(1) # Attempt to create a primary dataset print "" primary = DbsPrimaryDataset (datasetName = "test_primary_anzar") print "Creating primary dataset %s" % primary try: api.createPrimaryDataset (primary) print "Result: %s" % primary except DbsCgiObjectExists, ex: print "Object existed already, passing"
from dbsPrimaryDataset import DbsPrimaryDataset from dbsProcessedDataset import DbsProcessedDataset from dbsProcessing import DbsProcessing from dbsApi import DbsApi, DbsApiException, InvalidDataTier, DBS_LOG_LEVEL_ALL_ DEFAULT_URL = "http://cmsdoc.cern.ch/cms/aprom/DBS/CGIServer/prodquery" # DEFAULT_URL = "exec:../../Servers/CGIServer/prodquery" try: nop = 1 if (len(sys.argv) > 2): nop = int(sys.argv[2]) nevc = 1000 if (len(sys.argv) > 3): nevc = int(sys.argv[3]) api = DbsCgiApi(DEFAULT_URL, { 'instance' : sys.argv[1] }) api.setLogLevel(DBS_LOG_LEVEL_ALL_) # api.setDebug(1) # Make a new test dataset, insert group of event collections basename = "test_big_%f" % time.time() primary = DbsPrimaryDataset (datasetName = basename) processing = DbsProcessing (primaryDataset = primary, processingName = "test_process", applicationConfig = { 'application' : { 'executable' : 'testexe', 'version' : 'testversion', 'family' : 'testfamily' }, 'parameterSet' : { 'hash' : basename, 'content' : basename }}) block = DbsFileBlock (processing = processing)
print usage sys.exit(1) if dbsinstance == None: print "--DBSAddress option not provided. For example : --DBSAddress=MCLocal_4/Writer \n" print usage sys.exit(1) if datasetPath == None: print "--datasetPath option not provided. For example : --datasetPath=/primarydataset/datatier/processeddataset \n" print usage sys.exit(1) if logfile == None: print "--logfile option not provided. For example : --logfile=logs_TOB/proddatasetlist.txt \n" print usage sys.exit(1) f = open(logfile, "w") inargs = {'instance': dbsinstance} api = DbsCgiApi(DEFAULT_URL, inargs) datasets = api.listProcessedDatasets(datasetPath) for dataset in datasets: f.write("DatasetName: %s \n" % (dataset)) # f.close()
for opt, arg in opts: if opt == "--DBSAddress": dbsinstance = arg if opt == "--datasetName": datasetName = arg if opt == "--help": print usage sys.exit(1) if dbsinstance == None: print "--DBSAddress option not provided. For example : --DBSAddress=MCLocal_4/Writer \n" print usage sys.exit(1) if datasetName == None: print "--datasetPath option not provided. For example : --datasetPath=/primarydataset/datatier/processeddataset \n" print usage sys.exit(1) # inargs = {'instance' : dbsinstance} api = DbsCgiApi(DEFAULT_URL, inargs) nevttot=0 for block in api.getDatasetContents(datasetName): for evc in block.get('eventCollectionList'): nevttot = nevttot + evc.get('numberOfEvents') print nevttot
# Unit tests for the DBS CGI implementation. import sys from dbsCgiApi import DbsCgiApi, DbsCgiObjectExists from dbsException import DbsException from dbsApi import DbsApi, DbsApiException, InvalidDataTier DEFAULT_URL = "http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquery" #DEFAULT_URL = "exec:/home/sekhri/cgi/java/test/in.sh" #DEFAULT_URL = "exec:../CGIServer/prodquery" try: args = {} if len(sys.argv) == 2: args['instance'] = sys.argv[1] api = DbsCgiApi(DEFAULT_URL, args) print "Setting file status as invalid" api.setFileStatus ("tmpLFN4", "invalid") print "Setting file status as valid" #api.setFileStatus ("tmpLFN6", "valid") #api.setFileStatus ("tmpLFN6", None) #api.setFileAvailable ("tmpLFN6") #api.setFileUnavailable ("6B9EC-5026-DB11-9898-003048713B63.root") #api.setFileAvailable ("/store/unmerged/PreProd/2006/8/2/ProdAgentDevTest1/GEN-SIM-DIGI/0000/9CE6B9EC-5026-DB11-9898-003048713B63.root") except InvalidDataTier, ex: print "Caught InvalidDataTier API exception: %s" % (ex.getErrorMessage()) except DbsApiException, ex: print "Caught API exception %s: %s" % (ex.getClassName(), ex.getErrorMessage())
DEFAULT_URL = "http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquerytest2" #DEFAULT_URL = "http://cmsdoc.cern.ch/cms/aprom/DBS/CGIServer/dbsxml" #DEFAULT_URL = "exec:../../Servers/CGIServer/prodquerytest2" #DEFAULT_URL = "http://lxgate40.cern.ch/cms/prod/comp/DBS/CGIServer/prodquery" #DEFAULT_URL = "exec:../CGIServer/prodquery" #DEFAULT_URL = "exec:/home/sekhri/cgi/java/test/catout.sh" #DEFAULT_URL = "exec:/home/sekhri/cgi/java/test/abc.sh" #DEFAULT_URL = "exec:/home/sekhri/cgi/java/test/run.sh" #DEFAULT_URL = "exec:/home/sekhri/cgi/java/test/in.sh" #DEFAULT_URL = "exec:/home/sekhri/cgi/java/test/err.sh" try: args = {} if len(sys.argv) == 2: args['instance'] = sys.argv[1] print args api = DbsCgiApi(DEFAULT_URL, args) #api.setLogLevel(DBS_LOG_LEVEL_ALL_) # api.setDebug(1) try: # List all parameter sets print "" print "Listing parameter sets" for pset in api.listParameterSets("**"): print " %s" % pset except DbsCgiDatabaseError,e: print e try: # List all applications print ""
print "--DBSAddress option not provided. For example : --DBSAddress=MCLocal_4/Writer \n" print usage sys.exit(1) if datasetPath == None: print "--datasetPath option not provided. For example : --datasetPath=/primarydataset/datatier/processeddataset \n" print usage sys.exit(1) if logfile == None: print "--logfile option not provided. For example : --logfile=logs_TOB/proddatasetlist.txt \n" print usage sys.exit(1) # ############################### # f = open("logs/proddatasetlist.txt", "w"); f = open(logfile, "w"); inargs = {'instance' : dbsinstance} api = DbsCgiApi(DEFAULT_URL, inargs) datasets = api.listProcessedDatasets(datasetPath) for dataset in datasets: # print "Injection of dataset: %s"%dataset.get('datasetPathName') f.write(dataset.get('datasetPathName') + "\n") f.close()
import sys from dbsCgiApi import DbsCgiApi, DbsCgiDatabaseError from dbsException import DbsException from dbsApi import DbsApi, DbsApiException, InvalidDataTier #DEFAULT_URL = "http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquerytest2" #DEFAULT_URL = "http://cmsdoc.cern.ch/cms/aprom/DBS/CGIServer/dbsxml" #DEFAULT_URL = "exec:../../Servers/CGIServer/prodquery" DEFAULT_URL = "exec:../../Servers/CGIServer/prodquerytest2" try: args = {} args1 = {} if (len(sys.argv) == 3) | (len(sys.argv) == 2) : args['instance'] = sys.argv[1] api = DbsCgiApi(DEFAULT_URL, args) if len(sys.argv) == 3: args1['instance'] = sys.argv[2] else : args1['instance'] = sys.argv[1] api1 = DbsCgiApi(DEFAULT_URL, args1) #api.setLogLevel(DBS_LOG_LEVEL_ALL_) # api.setDebug(1) try: # Get dataset contents, returning a list of blocks with event collections #print "" otherDatasetPath = "/test_primary_anzar/Hit/test_process_anzar" #otherDatasetPath = "/CSA06-081-os-minbias/SIM/CMSSW_0_8_1-GEN-SIM-DIGI-1154005302" #for MCLocal_2/Writer #otherDatasetPath = "/RelVal080pre5SingleMuPlusPt100/SIM/GEN-SIM-DIGI-RECO-merged" #for Very Large Datasets use Block granulity to publish data
#!/usr/bin/env python # # Unit tests for the DBS CGI implementation. import sys from dbsCgiApi import DbsCgiApi from dbsException import DbsException from dbsApi import DbsApi, DbsApiException, InvalidDataTier DEFAULT_URL = "http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquerytest3" #DEFAULT_URL = "exec:../../Servers/CGIServer/prodquerytest3" try: args = {} if len(sys.argv) == 2: args['instance'] = sys.argv[1] api = DbsCgiApi(DEFAULT_URL, args) #dataset = "/CSA06-081-os-minbias/DIGI/CMSSW_0_8_1-GEN-SIM-DIGI-1154005302" #dataset = "/CSA06-083-os-SoftMuon/DIGI/CMSSW_0_8_3-GEN-SIM-DIGI-HLT-1156877643-merged" datasetsb = [ "/CSA06-083-os-EWKSoup/GEN/CMSSW_0_8_3-GEN-SIM-DIGI-HLT-1156877645", "/CSA06-083-os-EWKSoup/DIGI/CMSSW_0_8_3-GEN-SIM-DIGI-HLT-1156877645", "/CSA06-083-os-EWKSoup/SIM/CMSSW_0_8_3-GEN-SIM-DIGI-HLT-1156877645", "/CSA06-083-os-EWKSoup/HLT/CMSSW_0_8_3-GEN-SIM-DIGI-HLT-1156877645", ] datasetsa = [ "/CSA06-082-os-TTbar/DIGI/CMSSW_0_8_2-GEN-SIM-DIGI-1155826011-merged", "/CSA06-082-os-TTbar/SIM/CMSSW_0_8_2-GEN-SIM-DIGI-1155826011-merged", "/CSA06-082-os-TTbar/GEN/CMSSW_0_8_2-GEN-SIM-DIGI-1155826011-merged"
#!/usr/bin/env python # # Unit tests for the DBS CGI implementation. import sys from dbsCgiApi import DbsCgiApi from dbsException import DbsException from dbsApi import DbsApi, DbsApiException, InvalidDataTier DEFAULT_URL = "http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquerytest2" try: args = {} if len(sys.argv) == 2: args['instance'] = sys.argv[1] api = DbsCgiApi(DEFAULT_URL, args) # api.setDebug(1) # List all datasets and count of files in all their blocks print "Listing datasets blocks/files" # For MCLocal_1/Writer dataset = "/test_primary_anzar/Hit/test_process_anzar" print api.getLFNs ( "/test_primary_anzar/test_process_anzar#2eaa3188-9f81-490c-9f20-e10ac49ef785", dataset) print api.getLFNs ( "/test_primary_anzar/test_process_anzar#2eaa3188-9f81-490c-9f20-e10ac49ef785", "") print api.getLFNs ( "/test_primary_anzar/test_process_anzar#2eaa3188-9f81-490c-9f20-e10ac49ef785") print api.getLFNs ( "/test_primary_anzar/test_process_anzar#2eaa3188-9f81-490c-9f20-e10ac49ef785"," ") # For DevMC/Writer """ dataset = "/CSA06-083-os-EWKSoup/DIGI/CMSSW_0_8_3-GEN-SIM-DIGI-HLT-1156877645-merged" print api.getLFNs ( "/CSA06-083-os-EWKSoup/CMSSW_0_8_3-GEN-SIM-DIGI-HLT-1156877645-merged#502ac997-9a64-4bb3-8279-3c6328f35f54", dataset) print api.getLFNs ( "/CSA06-083-os-EWKSoup/CMSSW_0_8_3-GEN-SIM-DIGI-HLT-1156877645-merged#502ac997-9a64-4bb3-8279-3c6328f35f54")
#!/usr/bin/env python # # Unit tests for the DBS CGI implementation. import os, sys from dbsCgiApi import DbsCgiApi from dbsException import DbsException from dbsApi import DbsApi, DbsApiException, InvalidDataTier, DBS_LOG_LEVEL_ALL_ DEFAULT_URL = "http://cmsdoc.cern.ch/cms/aprom/DBS/CGIServer/dbsxml" # DEFAULT_URL = "exec:../../Servers/CGIServer/dbsxml" try: args = {} if len(sys.argv) == 2: args['instance'] = sys.argv[1] api = DbsCgiApi(DEFAULT_URL, args) api.setLogLevel(DBS_LOG_LEVEL_ALL_) # api.setDebug(1) # Datasets we play with datasetPattern = "/*/*/eg_2x1033PU761_TkMu_2_g133_OSC" datasetPath = "/eg03_jets_1e_pt2550/Digi/eg_2x1033PU761_TkMu_2_g133_OSC" otherDatasetPath = "/bt03_B0sJPsiX/Hit/bt_Hit245_2_g133" # List some datasets print "" print "Listing datasets %s" % datasetPattern datasets = api.listDatasets (datasetPattern) for dataset in datasets: print " %s" % dataset
for opt, arg in opts: if opt == "--DBSAddress": dbsinstance = arg if opt == "--datasetName": datasetName = arg if opt == "--help": print usage sys.exit(1) if dbsinstance == None: print "--DBSAddress option not provided. For example : --DBSAddress=MCLocal_4/Writer \n" print usage sys.exit(1) if datasetName == None: print "--datasetPath option not provided. For example : --datasetPath=/primarydataset/datatier/processeddataset \n" print usage sys.exit(1) # inargs = {'instance': dbsinstance} api = DbsCgiApi(DEFAULT_URL, inargs) nevttot = 0 for block in api.getDatasetContents(datasetName): for evc in block.get('eventCollectionList'): nevttot = nevttot + evc.get('numberOfEvents') print nevttot
#DEFAULT_URL = "http://venom.fnal.gov:8080/servlets-examples/servlet/RequestHeaderExample" #DEFAULT_URL = "http://venom.fnal.gov:8080/servlets-examples/servlet/HeaderExample" DEFAULT_URL = "http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquery" #DEFAULT_URL = "http://cmsdoc.cern.ch/cms/aprom/DBS/CGIServer/dbsxml" #DEFAULT_URL = "exec:../CGIServer/prodquery" #DEFAULT_URL = "exec:/home/sekhri/cgi/java/test/catout.sh" #DEFAULT_URL = "exec:/home/sekhri/cgi/java/test/abc.sh" #DEFAULT_URL = "exec:/home/sekhri/cgi/java/test/run.sh" #DEFAULT_URL = "exec:/home/sekhri/cgi/java/test/in.sh" #DEFAULT_URL = "exec:/home/sekhri/cgi/java/test/err.sh" try: args = {} if len(sys.argv) == 2: args['instance'] = sys.argv[1] print args api = DbsCgiApi(DEFAULT_URL, args) #api.setLogLevel(DBS_LOG_LEVEL_ALL_) # api.setDebug(1) # Datasets we play with datasetPattern = "/CMSSW_0_6_0_pre7/*/*" try: # List some datasets print "" print "Listing datasets %s" % datasetPattern datasets = api.listDatasetsFromApp (datasetPattern) for dataset in datasets: print " %s" % dataset except DbsCgiDatabaseError,e: print e
"\n\tget: This will just fetch the contents of the dataset and save it in a xml file. It will not write the contents to any other DBS instance." + \ "\n\tset: This will just write the contents of xml file to another DBS instance. It will not fetch the dataset contents first." + \ "\nIf you do not supply this op parameter then the dafault is assumed which is both." + \ "\nExample :" + \ "\npython dbsCgiMigrate.py MCLocal_1/Writer DevMC/Writer /CSA06-081-os-minbias/DIGI/CMSSW_0_8_1-GEN-SIM-DIGI-1154005302-merged" + \ "\n****************************************************************" if (len(sys.argv) < 4) : print usage sys.exit(1) op = "both" if (len(sys.argv) == 5) : op = sys.argv[4] args['instance'] = sys.argv[1] args1['instance'] = sys.argv[2] api = DbsCgiApi(DEFAULT_URL, args) api1 = DbsCgiApi(DEFAULT_URL, args1) path = sys.argv[3] print path name = args['instance'].replace('/','_') + "_" + args1['instance'].replace('/', '_') + path.replace('/', '_') if ((op == "both") | (op == "get")) : #Fetch the dataset contents and save them in a file xmlinput = api.getDatasetInfo(path) f = open(name + ".xml", "w"); f.write(xmlinput) f.close() print "Dataset information fetched from " + args['instance'] + " in XML format is saved in " + name + ".xml" if ((op == "both") | (op == "set")) :
#DEFAULT_URL = "http://cmsdoc.cern.ch/cms/test/aprom/DBS/CGIServer/prodquery" #DEFAULT_URL = "http://cmsdoc.cern.ch/cms/aprom/DBS/CGIServer/dbsxml" #DEFAULT_URL = "exec:../../Servers/CGIServer/prodquery" DEFAULT_URL = "exec:../../Servers/CGIServer/prodquerytest2" try: args = {} args1 = {} if (len(sys.argv) == 3) | (len(sys.argv) == 2) : args['instance'] = sys.argv[1] api = DbsCgiApi(DEFAULT_URL, args) if len(sys.argv) == 3: args1['instance'] = sys.argv[2] else : args1['instance'] = sys.argv[1] api1 = DbsCgiApi(DEFAULT_URL, args1) #api.setLogLevel(DBS_LOG_LEVEL_ALL_) # api.setDebug(1) try: pathList = [ "/CSA06-081-os-minbias/GEN/CMSSW_0_8_1-GEN-SIM-DIGI-1154005302-merged" , "/CSA06-081-os-minbias/SIM/CMSSW_0_8_1-GEN-SIM-DIGI-1154005302-merged" , "/CSA06-081-os-minbias/DIGI/CMSSW_0_8_1-GEN-SIM-DIGI-1154005302-merged" , ] # for MCLocal_3/Writer pathList = [ "/CSA06-082-os-ZMuMu/GEN/CMSSW_0_8_2-GEN-SIM-DIGI-1155826058-merged", "/CSA06-082-os-ZMuMu/SIM/CMSSW_0_8_2-GEN-SIM-DIGI-1155826058-merged", "/CSA06-082-os-ZMuMu/DIGI/CMSSW_0_8_2-GEN-SIM-DIGI-1155826058-merged" ] pathList = [ "/CSA06-083-os-SoftMuon/GEN/CMSSW_0_8_3-GEN-SIM-DIGI-HLT-1156877643-merged",