Exemplo n.º 1
0
def injectFilesFromDBS(inputFileset, datasetPath):
    """
    _injectFilesFromDBS_

    """
    print "injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name)
    args={}
    args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
    args["version"] = "DBS_2_0_9"
    args["mode"] = "GET"
    dbsApi = DbsApi(args)
    dbsResults = dbsApi.listFiles(path = datasetPath, retriveList = ["retrive_block","retrive_lumi", "retrive_run"])

    # NOTE : this is to limit the number of jobs to create ... simply using first 10 files get for the needed dataset
    dbsResults =dbsResults[0:2]


    print "  found %d files, inserting into wmbs..." % (len(dbsResults))


    for dbsResult in dbsResults:
        myFile = File(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"],
                      events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]},
                      locations = set(['srm.ciemat.es','storm-se-01.ba.infn.it','storage01.lcg.cscs.ch']))

        myRun = Run(runNumber = dbsResult["LumiList"][0]["RunNumber"])
        for lumi in dbsResult["LumiList"]:
            myRun.lumis.append(lumi["LumiSectionNumber"])
        myFile.addRun(myRun)
        myFile.create()
        inputFileset.addFile(myFile)

    inputFileset.commit()
    inputFileset.markOpen(False)
    return
Exemplo n.º 2
0
def dbsGetDatasetType( dataset ):
    d = {}
    d['url']='http://cmsdbsprod.cern.ch/cms_dbs_ph_analysis_02/servlet/DBSServlet'
    d['quiet']=True
    api = DbsApi( d )
    print api.listDatasetSummary(dataset.name)
    res = api.executeQuery('find datatype where dataset=%s' % dataset.name)
Exemplo n.º 3
0
    def __init__(self, url, **contact):
        args = {"url": url, "level": 'ERROR', "version": ''}
        args.update(contact)
        #try:
        self.dbs = DbsApi(args)
        #except DbsException, ex:
        #    msg = "Error in DBSReader with DbsApi\n"
        #    msg += "%s\n" % formatEx(ex)
        #    raise DBSReaderError(msg)

        # setup DLS api - with either dbs or phedex depending on dbs instance
        if url.count('cmsdbsprod.cern.ch/cms_dbs_prod_global') or \
                        self.dbs.getServerInfo()['InstanceName'] == 'GLOBAL':
            dlsType = 'DLS_TYPE_PHEDEX'
            dlsUrl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod'
        else:
            dlsType = 'DLS_TYPE_DBS'
            dlsUrl = url
        try:
            self.dls = dlsClient.getDlsApi(dls_type=dlsType,
                                           dls_endpoint=dlsUrl,
                                           version=args['version'])
        except DlsApiError, ex:
            msg = "Error in DBSReader with DlsApi\n"
            msg += "%s\n" % str(ex)
            raise DBSReaderError(msg)
Exemplo n.º 4
0
def injectFilesFromDBS(inputFileset, datasetPath, runsWhiteList=[]):
    """
    _injectFilesFromDBS_

    """
    print("injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name))
    args = {}
    args["url"] = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
    args["version"] = "DBS_2_1_1"
    args["mode"] = "GET"
    dbsApi = DbsApi(args)
    dbsResults = dbsApi.listFileArray(path=datasetPath, retriveList=["retrive_lumi", "retrive_run"])
    print("  found %d files, inserting into wmbs..." % (len(dbsResults)))

    for dbsResult in dbsResults:
        if runsWhiteList and str(dbsResult["LumiList"][0]["RunNumber"]) not in runsWhiteList:
            continue
        myFile = File(lfn=dbsResult["LogicalFileName"], size=dbsResult["FileSize"],
                      events=dbsResult["NumberOfEvents"], checksums={"cksum": dbsResult["Checksum"]},
                      locations="cmssrm.fnal.gov", merged=True)
        myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"])
        for lumi in dbsResult["LumiList"]:
            myRun.appendLumi(lumi["LumiSectionNumber"])
        myFile.addRun(myRun)
        myFile.create()
        inputFileset.addFile(myFile)

    if len(inputFileset) < 1:
        raise Exception("No files were selected!")

    inputFileset.commit()
    inputFileset.markOpen(False)
    return
def injectFilesFromDBS(inputFileset, datasetPath, runsWhiteList=[]):
    """
    _injectFilesFromDBS_

    """
    print "injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name)
    args={}
    args["url"] = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
    args["version"] = "DBS_2_1_1"
    args["mode"] = "GET"
    dbsApi = DbsApi(args)
    dbsResults = dbsApi.listFiles(path = datasetPath, retriveList = ["retrive_lumi", "retrive_run"])
    print "  found %d files, inserting into wmbs..." % (len(dbsResults))

    for dbsResult in dbsResults:
        if runsWhiteList and str(dbsResult["LumiList"][0]["RunNumber"]) not in runsWhiteList:
            continue
        myFile = File(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"],
                      events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]},
                      locations = "cmssrm.fnal.gov", merged = True)
        myRun = Run(runNumber = dbsResult["LumiList"][0]["RunNumber"])
        for lumi in dbsResult["LumiList"]:
            myRun.lumis.append(lumi["LumiSectionNumber"])
        myFile.addRun(myRun)
        myFile.create()
        inputFileset.addFile(myFile)

    if len(inputFileset) < 1:
        raise Exception, "No files were selected!"

    inputFileset.commit()
    inputFileset.markOpen(False)
    return
Exemplo n.º 6
0
def main():
    from optparse import OptionParser

    usage="usage: python setDatasetStatus.py --dataset=<DATASET_NAME> --status=<STATUS> --url=<DBSURL> {--files}"
    parser = OptionParser(usage=usage)

    parser.add_option('-d', '--dataset', dest='dataset', default=None, help='Dataset name')
    parser.add_option('-s', '--status', dest='status', default=None, help='This will be the new status of the dataset/files')
    parser.add_option('-D', '--url', dest='url', default='https://cmsdbsprod.cern.ch:8443/cms_dbs_prod_global_writer/servlet/DBSServlet', help='DBS URL')
    parser.add_option('-f', '--files', action="store_true", default=False, dest='files', help='Validate or invalidate all files in dataset')

    (opts, args) = parser.parse_args()

    if opts.dataset == None:
        print "--dataset option must be provided"
        print usage;
        sys.exit(1)
    if opts.status == None:
        print "--status option must be provided"
        print usage;
        sys.exit(1)
    if opts.url == None:
        print "--url option not provided."
        print "Using %s"%opts.url

    dbsargs = {'url' : opts.url}
    dbsapi = DbsApi(dbsargs)

    try:
        dbsapi.updateProcDSStatus(opts.dataset, opts.status)
    except Exception, ex:
        print "Caught Exception %s "  % str(ex)
        sys.exit(1)
Exemplo n.º 7
0
    def _queryAndCompareWithDBS(self, pileupDict, defaultArguments, dbsUrl):
        """
        pileupDict is a Python dictionary containing particular pileup
        configuration information. Query DBS on given dataset contained
        now in both input defaultArguments as well as in the pileupDict
        and compare values.
        
        """
        args = {}
        # this should have been set in CMSSWStepHelper along with
        # the pileup configuration
        args["url"] = dbsUrl
        args["version"] = "DBS_2_0_9"
        args["mode"] = "GET"
        dbsApi = DbsApi(args)
        
        inputArgs = defaultArguments["PileupConfig"]
        
        self.assertEqual(len(inputArgs), len(pileupDict),
                         "Number of pileup types different.")
        for pileupType in inputArgs:
            m = ("pileup type '%s' not in PileupFetcher-produced pileup "
                 "configuration: '%s'" % (pileupType, pileupDict))
            self.failUnless(pileupType in pileupDict, m)

        # now query DBS for compare actual results on files lists for each
        # pileup type and dataset and location (storage element names)
        # pileupDict is saved in the file and now comparing items of this
        # configuration with actual DBS results, the structure of pileupDict:            
        #    {"pileupTypeA": {"BlockA": {"FileList": [], "StorageElementNames": []}, 
        #                     "BlockB": {"FileList": [], "StorageElementName": []}, ....}                    
        for pileupType, datasets  in inputArgs.items():
            # this is from the pileup configuration produced by PileupFetcher
            blockDict = pileupDict[pileupType]
            m = "Number of datasets for pileup type '%s' is not equal." % pileupType 
            self.assertEqual(len(blockDict), len(datasets), m)
            
            for dataset in datasets:
                dbsFileBlocks = dbsApi.listBlocks(dataset = dataset)
                fileList = [] # list of files in the block (dbsFile["LogicalFileName"])
                storageElemNames = [] # list of StorageElementName
                for dbsFileBlock in dbsFileBlocks:
                    blockName = dbsFileBlock["Name"]
                    # each DBS block has a list under 'StorageElementList', iterate over
                    for storElem in dbsFileBlock["StorageElementList"]:
                        storageElemNames.append(storElem["Name"])
                    # now get list of files in the block
                    dbsFiles = dbsApi.listFiles(blockName = blockName)
                    for dbsFile in dbsFiles:
                        fileList.append(dbsFile["LogicalFileName"])
                # now compare the sets:
                m = ("StorageElementNames don't agree for pileup type '%s', "
                     "dataset '%s' in configuration: '%s'" % (pileupType, dataset, pileupDict))
                self.assertEqual(blockDict[blockName]["StorageElementNames"], storageElemNames, m)
                m = ("FileList don't agree for pileup type '%s', dataset '%s' "
                     " in configuration: '%s'" % (pileupType, dataset, pileupDict))
                self.assertEqual(blockDict[blockName]["FileList"], fileList)
def setStatusDBS3(url3, dataset3, newStatus, files):
    from dbs.apis.dbsClient import DbsApi

    dbsapi = DbsApi(url=url3)

    try:
        dbsapi.updateDatasetType(dataset=dataset3, dataset_access_type=newStatus)
    except Exception, ex:
        print "Caught Exception %s " % str(ex)
        sys.exit(1)
Exemplo n.º 9
0
def setStatusDBS3(url3, dataset3, newStatus, files):
    from dbs.apis.dbsClient import DbsApi

    dbsapi = DbsApi(url=url3)

    try:
        dbsapi.updateDatasetType(dataset=dataset3, dataset_access_type=newStatus)
    except Exception, ex:
        print "Caught Exception %s " % str(ex)
        sys.exit(1)
Exemplo n.º 10
0
def getFileNames (event, dbsOptions = {}):
    # Query DBS
    try:
        api = DbsApi (dbsOptions)
        query = "find file where dataset=%(dataset)s and run=%(run)i and lumi=%(lumi)i" % event

        xmldata = api.executeQuery(query)
    except DbsApiException, ex:
        print "Caught API Exception %s: %s "  % (ex.getClassName(), ex.getErrorMessage() )
        if ex.getErrorCode() not in (None, ""):
            print "DBS Exception Error Code: ", ex.getErrorCode()
Exemplo n.º 11
0
def getFileNames(event, dbsOptions={}):
    # Query DBS
    try:
        api = DbsApi(dbsOptions)
        query = "find file where dataset=%(dataset)s and run=%(run)i and lumi=%(lumi)i" % event

        xmldata = api.executeQuery(query)
    except DbsApiException, ex:
        print "Caught API Exception %s: %s " % (ex.getClassName(),
                                                ex.getErrorMessage())
        if ex.getErrorCode() not in (None, ""):
            print "DBS Exception Error Code: ", ex.getErrorCode()
Exemplo n.º 12
0
def setStatusDBS2(url2, dataset, newStatus, files):
    from DBSAPI.dbsApi import DbsApi
    from DBSAPI.dbsOptions import DbsOptionParser

    dbsargs = {'url': url2}
    dbsapi = DbsApi(dbsargs)

    try:
        dbsapi.updateProcDSStatus(dataset, newStatus)
    except Exception, ex:
        print "Caught DBS2 Exception %s " % str(ex)
        sys.exit(1)
def setStatusDBS2(url2, dataset, newStatus, files):
    from DBSAPI.dbsApi import DbsApi
    from DBSAPI.dbsOptions import DbsOptionParser

    dbsargs = {"url": url2}
    dbsapi = DbsApi(dbsargs)

    try:
        dbsapi.updateProcDSStatus(dataset, newStatus)
    except Exception, ex:
        print "Caught DBS2 Exception %s " % str(ex)
        sys.exit(1)
Exemplo n.º 14
0
def injectFilesFromDBS(inputFileset, datasetPath):
    """
    _injectFilesFromDBS_

    """
    print "injecting files from %s into %s, please wait..." % (
        datasetPath, inputFileset.name)
    args = {}
    args[
        "url"] = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
    args["version"] = "DBS_2_0_9"
    args["mode"] = "GET"
    dbsApi = DbsApi(args)
    dbsResults = dbsApi.listFiles(path=datasetPath,
                                  retriveList=["retrive_lumi", "retrive_run"])
    dbsResults = dbsResults[0:10]
    print "  found %d files, inserting into wmbs..." % (len(dbsResults))

    for dbsResult in dbsResults:
        myFile = File(lfn=dbsResult["LogicalFileName"],
                      size=dbsResult["FileSize"],
                      events=dbsResult["NumberOfEvents"],
                      checksums={"cksum": dbsResult["Checksum"]},
                      locations="cmssrm.fnal.gov",
                      merged=True)
        myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"])
        for lumi in dbsResult["LumiList"]:
            myRun.lumis.append(lumi["LumiSectionNumber"])
        myFile.addRun(myRun)
        myFile.create()
        inputFileset.addFile(myFile)

        dbsFile = DBSBufferFile(lfn=dbsResult["LogicalFileName"],
                                size=dbsResult["FileSize"],
                                events=dbsResult["NumberOfEvents"],
                                checksums={"cksum": dbsResult["Checksum"]},
                                locations="cmssrm.fnal.gov",
                                status="LOCAL")
        dbsFile.setDatasetPath(datasetPath)
        dbsFile.setAlgorithm(appName="cmsRun",
                             appVer="Unknown",
                             appFam="Unknown",
                             psetHash="Unknown",
                             configContent="Unknown")
        dbsFile.create()

    inputFileset.commit()
    inputFileset.markOpen(False)
    return
Exemplo n.º 15
0
def injectFilesFromDBS(inputFileset, datasetPath):
    """
    _injectFilesFromDBS_

    """
    print("injecting files from %s into %s, please wait..." %
          (datasetPath, inputFileset.name))
    args = {}
    args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
    args["mode"] = "GET"
    dbsApi = DbsApi(args)
    dbsResults = dbsApi.listFileArray(
        path=datasetPath, retriveList=["retrive_lumi", "retrive_run"])
    # Limiter on number of files
    dbsResults = dbsResults[0:20]
    print("  found %d files, inserting into wmbs..." % (len(dbsResults)))

    for dbsResult in dbsResults:
        myFile = File(lfn=dbsResult["LogicalFileName"],
                      size=dbsResult["FileSize"],
                      events=dbsResult["NumberOfEvents"],
                      checksums={"cksum": dbsResult["Checksum"]},
                      locations="cmssrm.fnal.gov",
                      merged=True)
        myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"])
        for lumi in dbsResult["LumiList"]:
            myRun.appendLumi(lumi["LumiSectionNumber"])
        myFile.addRun(myRun)
        myFile.create()
        inputFileset.addFile(myFile)

        dbsFile = DBSBufferFile(lfn=dbsResult["LogicalFileName"],
                                size=dbsResult["FileSize"],
                                events=dbsResult["NumberOfEvents"],
                                checksums={"cksum": dbsResult["Checksum"]},
                                locations="cmssrm.fnal.gov",
                                status="NOTUPLOADED")
        dbsFile.setDatasetPath(datasetPath)
        dbsFile.setAlgorithm(appName="cmsRun",
                             appVer="Unknown",
                             appFam="Unknown",
                             psetHash="Unknown",
                             configContent="Unknown")
        dbsFile.create()

    inputFileset.commit()
    inputFileset.markOpen(False)
    return
    def __init__(
        self,
        nodes,
        phedexURL="https://cmsweb.cern.ch/phedex/datasvc/json/prod/fileReplicas",
        dbsURL="https://cmsweb.cern.ch/dbs/prod/global/DBSReader",
    ):

        # Add the node specification the URL
        nodeList = self.makelist(nodes)
        for node in nodeList:
            # for the first node need to be a bit different
            if node == nodeList[0]:
                self.nodeURL = phedexURL + "?node=%s" % node
            else:
                self.nodeURL += "&node=%s" % node
        self.dbsURL = dbsURL

        try:
            # optManager  = DbsOptionParser()
            # (opts,args) = optManager.getOpt()
            # opts[ 'url' ] = dbsURL
            self.dbsapi = DbsApi({"url": dbsURL})  # opts.__dict__)
        except DbsApiException as ex:
            print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage())
            if ex.getErrorCode() not in (None, ""):
                print "DBS Exception Error Code: ", ex.getErrorCode()
 def __init__(self):
     paConfig = loadProdAgentConfiguration()    
     t0astDBConfig = paConfig.getConfig("Tier0DB")
 
     self.t0astDBConn = Tier0DB.Tier0DB(t0astDBConfig, manageGlobal = True)
     self.t0astDBConn.connect()
     
     localDBSConfig = paConfig.getConfig("LocalDBS")
     globalDBSConfig = paConfig.getConfig("GlobalDBSDLS")
     # phedexConfig = paConfig.getConfig("PhEDExConfig")
     
     self.localDBSUrl = localDBSConfig["DBSURL"]
     self.globalDBSUrl = globalDBSConfig["DBSURL"]
     
     # self.phedexDSUrl = phedexConfig["DataServiceURL"]
     self.localDbsApi = DbsApi({'url':localDBSConfig["DBSURL"]}) 
     self.globalDbsApi = DbsApi({'url':globalDBSConfig["DBSURL"]})
Exemplo n.º 18
0
def getDataset(filename):
    "Interface with the DBS API to get the dataset used"
    from xml.dom.minidom import parseString
    from DBSAPI.dbsApi import DbsApi
    args = {}
    args['url']='http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet'
    args['version']='DBS_2_0_9'
    args['mode']='POST'
    api = DbsApi(args)
    data = api.executeQuery("find dataset where file="+filename)
    domresults = parseString(data)
    dbs = domresults.getElementsByTagName('dbs')
    result = dbs[0].getElementsByTagName('results')
    rows=result[0].getElementsByTagName('row')
    dataset=(rows[0].getElementsByTagName('dataset'))[0] #rows should have only one element and produce a one element array
    node=(dataset.childNodes)[0] #childNodes should be a one element array
    return str(node.data) #The output is in unicode, so it has to be translated through str
Exemplo n.º 19
0
def get_good_dq_for(dataset):
	try:
		opts = {}
		opts['url'] = str(DbsConfig(opts).url())
		run_dq_search_criteria = []
		full_dq_info = []

		api = DbsApi(opts)
		full_dq_info =  {dataset: api.listRunLumiDQ(dataset=dataset, runLumiDQList=[run_dq_search_criteria], dqVersion="")}
		
		good_dq_info, dq_keys = process_raw_dbs(full_dq_info)
		
		return good_dq_info, dq_keys, full_dq_info
	except DbsApiException, ex:
  		print "Caught API Exception %s: %s "  % (ex.getClassName(), ex.getErrorMessage() )
  		if ex.getErrorCode() not in (None, ""):
    			print "DBS Exception Error Code: ", ex.getErrorCode()
Exemplo n.º 20
0
    def __init__(self, config):
        """
        Use Configuration object from config

        """
        # Config should have DBSInterface element
        self.config = config.DBSInterface

        args = {
            'url': self.config.DBSUrl,
            'level': 'ERROR',
            "user": '******',
            'version': self.config.DBSVersion
        }

        self.version = self.config.DBSVersion
        self.globalDBSUrl = None
        self.committedRuns = collections.deque(maxlen=1000)
        self.maxBlockFiles = self.config.DBSBlockMaxFiles
        self.maxBlockTime = self.config.DBSBlockMaxTime
        self.maxBlockSize = self.config.DBSBlockMaxSize
        self.maxFilesToCommit = self.config.MaxFilesToCommit
        self.doGlobalMigration = getattr(self.config, 'doGlobalMigration',
                                         True)

        if getattr(self.config, 'globalDBSUrl', None) != None:
            globalArgs = {
                'url': self.config.globalDBSUrl,
                'level': 'ERROR',
                "user": '******',
                'version': self.config.globalDBSVersion
            }
            self.globalDBSUrl = self.config.globalDBSUrl
        else:
            self.doGlobalMigration = False

        try:
            self.dbs = DbsApi(args)
            if self.globalDBSUrl:
                self.globalDBS = DbsApi(globalArgs)
        except DbsException, ex:
            msg = "Error in DBSWriterError with DbsApi\n"
            msg += "%s\n" % formatEx(ex)
            logging.error(msg)
            raise DBSInterfaceError(msg)
Exemplo n.º 21
0
def get_dbs_info(toFind, requirements):
    "Interface with the DBS API to get the whatever you want of a requirements. ALWAYS RETURN A LIST OF STRINGS"
    args = {}
    args['url']='http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet'
    args['version']='DBS_2_0_9'
    args['mode']='POST'
    api = DbsApi(args)
    data = api.executeQuery("find %s where %s" % (toFind, requirements))
    domresults = parseString(data)
    dbs = domresults.getElementsByTagName('dbs')
    result = dbs[0].getElementsByTagName('results')
    rows=result[0].getElementsByTagName('row')
    retList = []
    for row in rows:
        resultXML = row.getElementsByTagName(toFind)[0]
        node=(resultXML.childNodes)[0] #childNodes should be a one element array
        retList.append(str(node.nodeValue))
    return retList
Exemplo n.º 22
0
 def __init__(self, url, **contact):
     args = {"url": url, "level": 'ERROR'}
     args.update(contact)
     try:
         self.dbs = DbsApi(args)
     except DbsException, ex:
         msg = "Error in DBSReader with DbsApi\n"
         msg += "%s\n" % formatEx(ex)
         raise DBSReaderError(msg)
Exemplo n.º 23
0
    def __init__(self, url, **contact):
        args = {"url": url, "level": 'ERROR'}
        args.update(contact)
        try:
            self.dbs = DbsApi(args)
            self.args = args
            self.version = args.get('version', None)
            self.globalDBSUrl = args.get('globalDBSUrl', None)
            self.globalVersion = args.get('globalVersion', None)
            if self.globalDBSUrl:
                globalArgs = {'url': url, 'level': 'ERROR'}
                globalArgs.update(contact)
                self.globalDBS = DbsApi(globalArgs)

        except DbsException, ex:
            msg = "Error in DBSWriterError with DbsApi\n"
            msg += "%s\n" % formatEx(ex)
            raise DBSWriterError(msg)
Exemplo n.º 24
0
 def __init__(self, args):
     # just make sure args value complies with dbs args
     try:
         from DBSAPI.dbsApi import DbsApi
         DbsApi(args)
     except ImportError:
         # No dbsApi available, carry on
         pass
     self.args = args
Exemplo n.º 25
0
    def importParentDataset(self, globalDBS, datasetpath):
        """
           WARNING: it works only with DBS_2_0_9_patch_6
        """

        args = {'url': globalDBS}
        try:
            api_reader = DbsApi(args)
        except DbsApiException, ex:
            msg = "%s\n" % formatEx(ex)
            raise CrabException(msg)
Exemplo n.º 26
0
    def __init__(self):
        #, srcURL, dstURL):
        try:
            srcURL = "http://cmsdbsdev1.cern.ch:8880/DBSANZ/servlet/DBSServlet"
            dstURL = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"

            optManager = DbsOptionParser()
            (opts, args) = optManager.getOpt()
            args = {}
            args['url'] = srcURL
            args['mode'] = 'POST'
            args['version'] = 'DBS_2_0_5'
            self.srcApi = DbsApi(args)
            args['url'] = dstURL
            self.dstApi = DbsApi(args)

        except DbsApiException, ex:
            print "Caught API Exception %s: %s " % (ex.getClassName(),
                                                    ex.getErrorMessage())
            if ex.getErrorCode() not in (None, ""):
                print "DBS Exception Error Code: ", ex.getErrorCode()
Exemplo n.º 27
0
    def __init__(self, nodes, phedexURL = "https://cmsweb.cern.ch/phedex/datasvc/json/prod/fileReplicas", dbsURL = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"):

        # Add the node specification the URL
        nodeList = self.makelist( nodes )
        for node in nodeList:
            # for the first node need to be a bit different
            if node == nodeList[0]:
                self.nodeURL = phedexURL + "?node=%s" % node
            else:
                self.nodeURL += "&node=%s" % node
        self.dbsURL = dbsURL

        try:
            #optManager  = DbsOptionParser()
            #(opts,args) = optManager.getOpt()
            #opts[ 'url' ] = dbsURL
            self.dbsapi = DbsApi( {'url': dbsURL } )#opts.__dict__)
        except DbsApiException as ex:
            print "Caught API Exception %s: %s "  % (ex.getClassName(), ex.getErrorMessage() )
            if ex.getErrorCode() not in (None, ""):
                print "DBS Exception Error Code: ", ex.getErrorCode()
Exemplo n.º 28
0
def injectFilesFromDBS(inputFileset, datasetPath):
    """
    _injectFilesFromDBS_

    """
    print "injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name)
    args={}
    args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
    args["version"] = "DBS_2_0_9"
    args["mode"] = "GET"
    dbsApi = DbsApi(args)
    dbsResults = dbsApi.listFileArray(path = datasetPath, retriveList = ["retrive_lumi", "retrive_run"])
    dbsResults = dbsResults[0:10]
    print "  found %d files, inserting into wmbs..." % (len(dbsResults))

    for dbsResult in dbsResults:
        myFile = File(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"],
                      events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]},
                      locations = "cmssrm.fnal.gov", merged = True)
        myRun = Run(runNumber = dbsResult["LumiList"][0]["RunNumber"])
        for lumi in dbsResult["LumiList"]:
            myRun.lumis.append(lumi["LumiSectionNumber"])
        myFile.addRun(myRun)
        myFile.create()
        inputFileset.addFile(myFile)

        dbsFile = DBSBufferFile(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"],
                                events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]},
                                locations = "cmssrm.fnal.gov", status = "LOCAL")
        dbsFile.setDatasetPath(datasetPath)
        dbsFile.setAlgorithm(appName = "cmsRun", appVer = "Unknown", appFam = "Unknown",
                             psetHash = "Unknown", configContent = "Unknown")
        dbsFile.create()
        
    inputFileset.commit()
    inputFileset.markOpen(False)
    return
Exemplo n.º 29
0
    def __init__(self, dbsUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"):
        """
        Configure the feeder
        """

        FeederImpl.__init__(self)

        # DBS parameter
        self.args = { "url" : dbsUrl, "level" : 'ERROR'}
        self.dbs = DbsApi(self.args)
        self.purgeTime = 96
        self.reopenTime = 120

        self.dbsReader = DBSReader(dbsUrl)
        self.connectionAttempts = 5
Exemplo n.º 30
0
    def __init__(self, dbsUrl = \
         "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"):
        """
        Configure the feeder
        """

        FeederImpl.__init__(self)

        # DBS parameter
        self.args = {"url": dbsUrl, "level": 'ERROR'}
        self.dbs = DbsApi(self.args)
        self.purgeTime = 96
        self.reopenTime = 120

        self.dbsReader = DBSReader(dbsUrl)
        self.connectionAttempts = 5
Exemplo n.º 31
0
    def __init__(self, url,  **contact):
        args = { "url" : url, "level" : 'ERROR'}
        args.update(contact)
        try:
            self.dbs           = DbsApi(args)
            self.args          = args
            self.version       = args.get('version', None)
            self.globalDBSUrl  = args.get('globalDBSUrl', None)
            self.globalVersion = args.get('globalVersion', None)
            if self.globalDBSUrl:
                globalArgs = {'url': url, 'level': 'ERROR'}
                globalArgs.update(contact)
                self.globalDBS = DbsApi(globalArgs)

        except DbsException as ex:
            msg = "Error in DBSWriterError with DbsApi\n"
            msg += "%s\n" % formatEx(ex)
            raise DBSWriterError(msg)
        self.reader = DBSReader(**args)
Exemplo n.º 32
0
    def __init__(self, config):
        """
        Use Configuration object from config

        """
        # Config should have DBSInterface element
        self.config = config.DBSInterface

        args = {'url': self.config.DBSUrl,
                'level': 'ERROR',
                "user" :'NORMAL',
                'version': self.config.DBSVersion}

        self.version           = self.config.DBSVersion
        self.globalDBSUrl      = None
        self.committedRuns     = collections.deque(maxlen = 1000)
        self.maxFilesToCommit  = self.config.MaxFilesToCommit
        self.doGlobalMigration = getattr(self.config, 'doGlobalMigration', True)

        if getattr(self.config, 'globalDBSUrl', None) != None:
            globalArgs = {'url': self.config.globalDBSUrl,
                          'level': 'ERROR',
                          "user" :'NORMAL',
                          'version': self.config.globalDBSVersion}
            self.globalDBSUrl = self.config.globalDBSUrl
        else:
            self.doGlobalMigration = False

        try:
            self.dbs       = DbsApi(args)
            if self.globalDBSUrl:
                self.globalDBS = DbsApi(globalArgs)
        except DbsException as ex:
            msg = "Error in DBSWriterError with DbsApi\n"
            msg += "%s\n" % formatEx(ex)
            logging.error(msg)
            raise DBSInterfaceError(msg)

        return
Exemplo n.º 33
0
    def __init__(self, url, **contact):
        args = {"url": url, "level": "ERROR", "version": ""}
        args.update(contact)
        # try:
        self.dbs = DbsApi(args)
        # except DbsException, ex:
        #    msg = "Error in DBSReader with DbsApi\n"
        #    msg += "%s\n" % formatEx(ex)
        #    raise DBSReaderError(msg)

        # setup DLS api - with either dbs or phedex depending on dbs instance
        if url.count("cmsdbsprod.cern.ch/cms_dbs_prod_global") or self.dbs.getServerInfo()["InstanceName"] == "GLOBAL":
            dlsType = "DLS_TYPE_PHEDEX"
            dlsUrl = "https://cmsweb.cern.ch/phedex/datasvc/xml/prod"
        else:
            dlsType = "DLS_TYPE_DBS"
            dlsUrl = url
        try:
            self.dls = dlsClient.getDlsApi(dls_type=dlsType, dls_endpoint=dlsUrl, version=args["version"])
        except DlsApiError, ex:
            msg = "Error in DBSReader with DlsApi\n"
            msg += "%s\n" % str(ex)
            raise DBSReaderError(msg)
Exemplo n.º 34
0
class DBSReader:
    """
    _DBSReader_

    General API for reading data from DBS


    """
    def __init__(self, url, **contact):
        args = { "url" : url, "level" : 'ERROR', "version" : ''}
        args.update(contact)
        #try:
        self.dbs = DbsApi(args)
        #except DbsException, ex:
        #    msg = "Error in DBSReader with DbsApi\n"
        #    msg += "%s\n" % formatEx(ex)
        #    raise DBSReaderError(msg)

        # setup DLS api - with either dbs or phedex depending on dbs instance
        if url.count('cmsdbsprod.cern.ch/cms_dbs_prod_global') or \
                        self.dbs.getServerInfo()['InstanceName'] == 'GLOBAL':
            dlsType = 'DLS_TYPE_PHEDEX'
            dlsUrl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod'
        else:
            dlsType = 'DLS_TYPE_DBS'
            dlsUrl = url
        try:
            self.dls = dlsClient.getDlsApi(dls_type = dlsType, dls_endpoint = dlsUrl, version = args['version'])
        except DlsApiError, ex:
            msg = "Error in DBSReader with DlsApi\n"
            msg += "%s\n" % str(ex)
            raise DBSReaderError(msg)
        except DbsException, ex:
            msg = "Error in DBSReader with DbsApi\n"
            msg += "%s\n" % formatEx(ex)
            raise DBSReaderError(msg)
Exemplo n.º 35
0
#!/usr/bin/env python
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser

optManager  = DbsOptionParser()
(opts,args) = optManager.getOpt()
api = DbsApi(opts.__dict__)

try:
    print api.remap('/RelValSingleMuPt10/CMSSW_2_1_0_pre10_IDEAL_V5_v4/GEN-SIM-DIGI-RAW-HLTDEBUG', '/RelValSingleMuPt10/CMSSW_2_1_0_pre10_IDEAL_V5_v4/GEN-SIM-DIGI-RAW-HLTDEBUG-RECO', '/RelValSingleMuPt10/CMSSW_2_1_0_pre10_IDEAL_V5_v4/GEN-SIM-DIGI-RAW-HLTDEBUG#870b391e-5590-40c5-a063-8109b1a23a8f')


except DbsApiException, ex:
  print "Caught API Exception %s: %s "  % (ex.getClassName(), ex.getErrorMessage() )
  if ex.getErrorCode() not in (None, ""):
    print "DBS Exception Error Code: ", ex.getErrorCode()


print "Done"

Exemplo n.º 36
0
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
# API Unit tests for the DBS JavaServer.
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser

try:
    optManager = DbsOptionParser()
    (opts, args) = optManager.getOpt()
    #print opts.__dict__
    api = DbsApi(opts.__dict__)
    #for block in api.listBlocks("/TestPrimary_001_20070315_03h12m26s/TestProcessed_20070315_03h12m26s/GEN-SIM"):
    #for block in api.listBlocks("/test_primary_001/TestProcessedDS001/GEN-SIM"):
    #for block in api.listBlocks("/chi1/CMSSW_1_6_7-CSA07-3268/GEN-SIM-DIGI-RAW"):
    #for block in api.listBlocks("/dataset_PD_110/CRUZET3-v1-unmerged/RAW"):

    for block in api.listBlocks(
            "/RelValSingleMuPt10/CMSSW_2_1_2_IDEAL_V6_v3/GEN-SIM-RECO"):
        #for block in api.listBlocks(block_name="/test_primary_001*"):
        #for block in api.listBlocks("", "/TestPrimary_001_20070315_02h26m11s/TestProcessed_20070315_02h26m11s/GEN-SIM#016712"):
        #for block in api.listBlocks("/test_primary_001/TestProcessedDS001/GEN-SIM", "/test_primary_001/TestProcessedDS001/GEN*"):
        #for block in api.listBlocks("/TestPrimary_001_20070315_02h53m32s/TestPrimary_001_20070315_02h53m32s/GEN-SIM"):
        #print "%s  %s" % (block['Name'], block['StorageElementList'])
        #print "  %s" % block['Name']
        print "  %s" % block
Exemplo n.º 37
0
results = []
for result in myThread.dbi.processData(sql):
    results.extend(result.fetchall())

blocks = {}
for row in results:
    if row[0] not in blocks:
        blocks[row[0]] = []

    blocks[row[0]].append(row[1])

args = {}
args["url"] = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
dbsApi = DbsApi(args)

badBlocks = []
badFiles = []
for blockName in blocks.keys():
    print("%s:" % blockName)
    blockFiles = []
    try:
        dbsFiles = dbsApi.listFileArray(blockName=blockName)
    except Exception as ex:
        dbsFiles = []

    for dbsFile in dbsFiles:
        blockFiles.append(dbsFile["LogicalFileName"])

    if len(blockFiles) != len(blocks[blockName]):
Exemplo n.º 38
0
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
#
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset
from DBSAPI.dbsOptions import DbsOptionParser
from DBSAPI.dbsDQFlag import DbsDQFlag
from DBSAPI.dbsRunLumiDQ import DbsRunLumiDQ

optManager  = DbsOptionParser()
(opts,args) = optManager.getOpt()
api = DbsApi(opts.__dict__)


def print_flags_nice(dqHierarchyList):
    if len(dqHierarchyList) <= 0:
	print "No DQ information for this run found"
    for aDQ in dqHierarchyList:
        print "\nRunNumber: ", aDQ['RunNumber']
        print "LumiSectionNumber: ", aDQ['LumiSectionNumber']
        for aSubDQ in aDQ['DQFlagList']:
                print "      ", aSubDQ['Name'], aSubDQ['Value']
                for aSubSubDQ in aSubDQ['SubSysFlagList']:
                        print "                ", aSubSubDQ['Name'], aSubSubDQ['Value']
			for abSubSubDQ in aSubSubDQ['SubSysFlagList'] :
				print "                               ", abSubSubDQ['Name'], abSubSubDQ['Value']
Exemplo n.º 39
0
#!/usr/bin/env python
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
# API Unit tests for the DBS JavaServer.
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser

try:
  optManager  = DbsOptionParser()
  (opts,args) = optManager.getOpt()
  api = DbsApi(opts.__dict__)
  for dataset in api.listDatasetPaths():
	print "\n %s" %str(dataset)
	print "Dataset parent: %s" %str(api.listPathParents)
  	for block in api.listBlocks(dataset):
		print "block: %s" %str(block['Name'])
  		for parent in api.listBlockParents(block_name=block['Name']):
     			print "Parent:  %s" % str(parent['Name'])

except DbsApiException, ex:
  print "Caught API Exception %s: %s "  % (ex.getClassName(), ex.getErrorMessage() )
  if ex.getErrorCode() not in (None, ""):
    print "DBS Exception Error Code: ", ex.getErrorCode()


Exemplo n.º 40
0
#!/usr/bin/env python
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"

#
# API Unit tests for the DBS JavaServer.
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser

try:
  optManager  = DbsOptionParser()
  (opts,args) = optManager.getOpt()
  api = DbsApi(opts.__dict__)
  xml = api.listDatasetContents("/test_primary_001/TestProcessedDS001/SIM", "/test_primary_001/TestProcessedDS001/SIM#12345");
  print xml
  print api.insertDatasetContents(xml)

except DbsApiException, ex:
  print "Caught API Exception %s: %s "  % (ex.getClassName(), ex.getErrorMessage() )
  if ex.getErrorCode() not in (None, ""):
    print "DBS Exception Error Code: ", ex.getErrorCode()


Exemplo n.º 41
0
import time
import sys
import random
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser

dstURL = sys.argv[1]

optManager  = DbsOptionParser()
(opts,args) = optManager.getOpt()
args = {}
args['url'] = dstURL 
args['mode']='POST'
api = DbsApi(args)

try:
	
	print "\n\nListing Datasets "
	paths = api.listDatasetPaths()
	ranIndex = random.randint(1,len(paths))
	#myDataset = paths[ranIndex]
	myDataset = "/Wjets-sherpa/Summer08_IDEAL_V12_v1/GEN-SIM-RAW"
	print "Selected dataset is %s", myDataset

	print "\nListing Blocks "
	blocks = api.listBlocks(myDataset)
	#print blocks
	
	print "\n Listing Files "
Exemplo n.º 42
0
#!/usr/bin/env python
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser

optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
api = DbsApi(opts.__dict__)

try:
    print api.insertFileParent("NEW_TEST0005xxxxxxxxxx6", "NEW_TEST0002")

except DbsApiException, ex:
    print "Caught API Exception %s: %s " % (ex.getClassName(),
                                            ex.getErrorMessage())
    if ex.getErrorCode() not in (None, ""):
        print "DBS Exception Error Code: ", ex.getErrorCode()

print "Done"
Exemplo n.º 43
0
import sys
import os
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser
from DBSAPI.dbsApi import DbsApi


try:
	optManager  = DbsOptionParser()
	(opts,args) = optManager.getOpt()
	args = {}
	args['url']='http://cmssrv17.fnal.gov:8989/DBS_1_0_5_STABLE/servlet/DBSServlet' 
	#args['version']='DBS_1_0_7'
	args['mode']='POST'
	api = DbsApi(args)
      
	#api = DbsApi(opts.__dict__)
	srcURL = sys.argv[1]
	dstURL = sys.argv[2]
	path = sys.argv[3]
	block = ""
	if len(sys.argv) > 4 :
		block = sys.argv[4]

	api.migrateDatasetContents(srcURL, dstURL, path, block , False, True)
	#print api.listPrimaryDatasets();

except DbsApiException, ex:
	print "Caught API Exception %s: %s "  % (ex.getClassName(), ex.getErrorMessage() )
	if ex.getErrorCode() not in (None, ""):
Exemplo n.º 44
0
#!/usr/bin/env python
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
#
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser

try:
  optManager  = DbsOptionParser()
  (opts,args) = optManager.getOpt()
  api = DbsApi(opts.__dict__)
  
  print "Analysis Daatset...."
  #for analysis in api.listAnalysisDataset("*t005", "/test_primary_001/TestProcessedDS001/SIM"):
  #for analysis in api.listAnalysisDataset(version=1):
  #for analysis in api.listAnalysisDataset(path="/RelVal131QCD_pt600_800/CMSSW_1_3_1-1176201507/GEN-SIM-DIGI-RECO", version=0):
  for analysis in api.listAnalysisDataset():
     #print "  %s" % analysis
     print "  %s, %s" % (analysis['Name'], analysis['Version'])
  
except DbsApiException, ex:
  print "Caught API Exception %s: %s "  % (ex.getClassName(), ex.getErrorMessage() )
  if ex.getErrorCode() not in (None, ""):
    print "DBS Exception Error Code: ", ex.getErrorCode()

print "Done"
Exemplo n.º 45
0
#!/usr/bin/env python
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsRun import DbsRun
from DBSAPI.dbsOptions import DbsOptionParser

optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
api = DbsApi(opts.__dict__)

try:
    #select Dataset, Complete from ProcDSRuns where Run=361;
    #run=62243
    run = ""
    path = '/Cosmics/BeamCommissioning08_CRUZET4_V4P_CSCSkim_trial_v3/RECO'
    for runStatus in api.listProcDSRunStatus(path, run):
        print "RunNumber: " + str(runStatus['RunNumber']) + " Status:" + str(
            runStatus['Done'])

except DbsApiException, ex:
    print "Caught API Exception %s: %s " % (ex.getClassName(),
                                            ex.getErrorMessage())
    if ex.getErrorCode() not in (None, ""):
        print "DBS Exception Error Code: ", ex.getErrorCode()
Exemplo n.º 46
0
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsAlgorithm import DbsAlgorithm
from DBSAPI.dbsFileBlock import DbsFileBlock
from DBSAPI.dbsFile import DbsFile
from DBSAPI.dbsLumiSection import DbsLumiSection
from DBSAPI.dbsQueryableParameterSet import DbsQueryableParameterSet
from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset
from DBSAPI.dbsProcessedDataset import DbsProcessedDataset
from DBSAPI.dbsOptions import DbsOptionParser

optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
api = DbsApi(opts.__dict__)

try:
    merge_algo = DbsAlgorithm(
        ExecutableName="EdmFastMerge",
        ApplicationVersion="v101",
        ApplicationFamily="Merge",
    )
    path = "/test_primary_001/TestProcessedDS001/SIM"
    merge_proc = api.insertMergedDataset(path, "ThisISMergedDataset001",
                                         merge_algo)

    # File will go into THIS Block
    block = DbsFileBlock(StorageElement=['test1', 'test3'],
                         Name="/test_primary_001/TestProcessedDS001/SIM#12345")
Exemplo n.º 47
0

import sys
import random
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset
from DBSAPI.dbsFileBlock import DbsFileBlock
from DBSAPI.dbsProcessedDataset import DbsProcessedDataset
from DBSAPI.dbsOptions import DbsOptionParser


optManager  = DbsOptionParser()
(opts,args) = optManager.getOpt()
api = DbsApi(opts.__dict__)

#primary = DbsPrimaryDataset (Name = "test_primary_anzar_03")
#primary = DbsPrimaryDataset (Name = "TestPrimary1164750596.79")
primary = DbsPrimaryDataset (Name = "test_primary_anzar_001")

proc = DbsProcessedDataset (
         PrimaryDataset=primary,
         Name="TestProcessedDS001",
         TierList=['SIM', 'RECO'],
         )
#ran = str(int(random.random()*10000000))
#block = DbsFileBlock (
#		Name= "/this/isaqqqqqqqqqxstbddddlock#016712"+ ran,
#         )
Exemplo n.º 48
0
#!/usr/bin/env python
#
# API Unit tests for the DBS JavaServer.

import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser

try:
  optManager  = DbsOptionParser()
  (opts,args) = optManager.getOpt()
  api = DbsApi(opts.__dict__)
  for block in api.listBlocks("/test_primary_anzar_001/SIM/TestProcessedDS001"):
  #for block in api.listBlocks("/TestPrimary1167862926.47/SIM1167862926.47/TestProcessed1167862926.47", "/*hahah#12345"):
  #for block in api.listBlocks("/TestPrimary1167862926.47/SIM1167862926.47/TestProcessed1167862926.47", "/this/*"):
  #for block in api.listBlocks("/TestPrimary1167862926.47/SIM1167862926.47/TestProcessed1167862926.47", "/this/ff*"):
  #for block in api.listBlocks("/TestPrimary1167862926.47/SIM1167862926.47/TestProcessed1167862926.47", "/this/hahah#12345"):
  #for block in api.listBlocks("/TestPrimary1167862926.47/SIM1167862926.47/TestProcessed1167862926.47", "/this/hahah#12345"):
     print "  %s" % block

except DbsApiException, ex:
  print "Caught API Exception %s: %s "  % (ex.getClassName(), ex.getErrorMessage() )
  if ex.getErrorCode() not in (None, ""):
    print "DBS Exception Error Code: ", ex.getErrorCode()


#!/usr/bin/env python
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
#
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser

try:
  optManager  = DbsOptionParser()
  (opts,args) = optManager.getOpt()
  api = DbsApi(opts.__dict__)
  
  print "Analysis Daatset Defs...."
  #for analysisdef in api.listAnalysisDatasetDefinition("*"):
  for analysisdef in api.listAnalysisDatasetDefinition():
     print "  %s" % analysisdef
  
except DbsApiException, ex:
  print "Caught API Exception %s: %s "  % (ex.getClassName(), ex.getErrorMessage() )
  if ex.getErrorCode() not in (None, ""):
    print "DBS Exception Error Code: ", ex.getErrorCode()

print "Done"

Exemplo n.º 50
0
if (block != None) and (datasetPath != None):
    print "\n options --block or --datasetPath are mutually exclusive"
    print usage
    sys.exit(1)


print ">>>>> DBS URL : %s"%(url)

import logging
logging.disable(logging.INFO)

#  //
# // Get API to DBS
#//
args = {'url' : url , 'level' : 'ERROR'}
dbsapi = DbsApi(args)

#  //
# // Delete dataset
#//
if (datasetPath):
 print "Deleting datasetPath=%s"%datasetPath
 dbsapi.deleteProcDS(datasetPath)

if (block):
 dbsreader = DBSReader(url)
 getdatasetPath = dbsreader.blockToDatasetPath(block)
 print "Deleting block=%s from datasetPath=%s"%(block,getdatasetPath)
 dbsapi.deleteBlock(getdatasetPath,block)

Exemplo n.º 51
0
results = []
for result in myThread.dbi.processData(sql):
    results.extend(result.fetchall())

blocks = {}
for row in results:
    if not blocks.has_key(row[0]):
        blocks[row[0]] = []

    blocks[row[0]].append(row[1])

args = {}
args["url"] = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
dbsApi = DbsApi(args)

badBlocks = []
badFiles = []
for blockName in blocks.keys():
    print "%s:" % blockName
    blockFiles = []
    try:
        dbsFiles = dbsApi.listFiles(blockName = blockName)
    except Exception, ex:
        dbsFiles = []
        
    for dbsFile in dbsFiles:
        blockFiles.append(dbsFile["LogicalFileName"])

    if len(blockFiles) != len(blocks[blockName]):
Exemplo n.º 52
0
#!/usr/bin/env python
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
#
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsOptions import DbsOptionParser

try:
    optManager = DbsOptionParser()
    (opts, args) = optManager.getOpt()
    api = DbsApi(opts.__dict__)

    try:
        path = "/TestPrimary_002_20070205_11h52m29s/SIM_20070205_11h52m29s/TestProcessed_20070205_11h52m29s"
        token = path.split("/")
        proc = api.listProcessedDatasets(token[1], token[2], token[3])[0]
        print "proc fetched from DBS %s" % proc
        proc['Name'] = "TestProcessed_20070205_11h52m29s_MERGED"
        proc['ParentList'] = [path]

        print "proc modified to be inserted in DBS %s" % proc
        api.insertProcessedDataset(proc)
        print "Result: %s" % proc

    except DbsDatabaseError, e:
        print e