Esempi in Python per DbsApi.listFileArray

Esempio n. 1

File: harvestingInjector.py Progetto: dmwm/WMCore

def injectFilesFromDBS(inputFileset, datasetPath, runsWhiteList=[]):
    """
    _injectFilesFromDBS_

    """
    print("injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name))
    args = {}
    args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
    args["version"] = "DBS_2_1_1"
    args["mode"] = "GET"
    dbsApi = DbsApi(args)
    dbsResults = dbsApi.listFileArray(path=datasetPath, retriveList=["retrive_lumi", "retrive_run"])
    print("  found %d files, inserting into wmbs..." % (len(dbsResults)))

    for dbsResult in dbsResults:
        if runsWhiteList and str(dbsResult["LumiList"][0]["RunNumber"]) not in runsWhiteList:
            continue
        myFile = File(lfn=dbsResult["LogicalFileName"], size=dbsResult["FileSize"],
                      events=dbsResult["NumberOfEvents"], checksums={"cksum": dbsResult["Checksum"]},
                      locations="cmssrm.fnal.gov", merged=True)
        myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"])
        for lumi in dbsResult["LumiList"]:
            myRun.appendLumi(lumi["LumiSectionNumber"])
        myFile.addRun(myRun)
        myFile.create()
        inputFileset.addFile(myFile)

    if len(inputFileset) < 1:
        raise Exception("No files were selected!")

    inputFileset.commit()
    inputFileset.markOpen(False)
    return

Esempio n. 2

File: harvestingInjector.py Progetto: todor-ivanov/WMCore

def injectFilesFromDBS(inputFileset, datasetPath, runsWhiteList=[]):
    """
    _injectFilesFromDBS_

    """
    print("injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name))
    args = {}
    args["url"] = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
    args["version"] = "DBS_2_1_1"
    args["mode"] = "GET"
    dbsApi = DbsApi(args)
    dbsResults = dbsApi.listFileArray(path=datasetPath, retriveList=["retrive_lumi", "retrive_run"])
    print("  found %d files, inserting into wmbs..." % (len(dbsResults)))

    for dbsResult in dbsResults:
        if runsWhiteList and str(dbsResult["LumiList"][0]["RunNumber"]) not in runsWhiteList:
            continue
        myFile = File(lfn=dbsResult["LogicalFileName"], size=dbsResult["FileSize"],
                      events=dbsResult["NumberOfEvents"], checksums={"cksum": dbsResult["Checksum"]},
                      locations="cmssrm.fnal.gov", merged=True)
        myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"])
        for lumi in dbsResult["LumiList"]:
            myRun.appendLumi(lumi["LumiSectionNumber"])
        myFile.addRun(myRun)
        myFile.create()
        inputFileset.addFile(myFile)

    if len(inputFileset) < 1:
        raise Exception("No files were selected!")

    inputFileset.commit()
    inputFileset.markOpen(False)
    return

Esempio n. 3

File: injectAnalysisWorkflow.py Progetto: mialiu149/WMCore

def injectFilesFromDBS(inputFileset, datasetPath):
    """
    _injectFilesFromDBS_

    """
    print "injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name)
    args={}
    args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
    args["version"] = "DBS_2_0_9"
    args["mode"] = "GET"
    dbsApi = DbsApi(args)
    dbsResults = dbsApi.listFileArray(path = datasetPath, retriveList = ["retrive_block","retrive_lumi", "retrive_run"])

    # NOTE : this is to limit the number of jobs to create ... simply using first 10 files get for the needed dataset
    dbsResults =dbsResults[0:2]


    print "  found %d files, inserting into wmbs..." % (len(dbsResults))


    for dbsResult in dbsResults:
        myFile = File(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"],
                      events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]},
                      locations = set(['srm.ciemat.es','storm-se-01.ba.infn.it','storage01.lcg.cscs.ch']))

        myRun = Run(runNumber = dbsResult["LumiList"][0]["RunNumber"])
        for lumi in dbsResult["LumiList"]:
            myRun.lumis.append(lumi["LumiSectionNumber"])
        myFile.addRun(myRun)
        myFile.create()
        inputFileset.addFile(myFile)

    inputFileset.commit()
    inputFileset.markOpen(False)
    return

Esempio n. 4

def injectFilesFromDBS(inputFileset, datasetPath):
    """
    _injectFilesFromDBS_

    """
    print("injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name))
    args={}
    args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
    args["version"] = "DBS_2_0_9"
    args["mode"] = "GET"
    dbsApi = DbsApi(args)
    dbsResults = dbsApi.listFileArray(path = datasetPath, retriveList = ["retrive_block","retrive_lumi", "retrive_run"])

    # NOTE : this is to limit the number of jobs to create ... simply using first 10 files get for the needed dataset
    dbsResults =dbsResults[0:2]


    print("  found %d files, inserting into wmbs..." % (len(dbsResults)))


    for dbsResult in dbsResults:
        myFile = File(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"],
                      events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]},
                      locations = set(['srm.ciemat.es','storm-se-01.ba.infn.it','storage01.lcg.cscs.ch']))

        myRun = Run(runNumber = dbsResult["LumiList"][0]["RunNumber"])
        for lumi in dbsResult["LumiList"]:
            myRun.lumis.append(lumi["LumiSectionNumber"])
        myFile.addRun(myRun)
        myFile.create()
        inputFileset.addFile(myFile)

    inputFileset.commit()
    inputFileset.markOpen(False)
    return

Esempio n. 5

File: injectStoreResults.py Progetto: tsarangi/WMCore

def injectFilesFromDBS(inputFileset, datasetPath):
    """
    _injectFilesFromDBS_

    """
    print "injecting files from %s into %s, please wait..." % (
        datasetPath, inputFileset.name)
    args = {}
    args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
    args["mode"] = "GET"
    dbsApi = DbsApi(args)
    dbsResults = dbsApi.listFileArray(
        path=datasetPath, retriveList=["retrive_lumi", "retrive_run"])
    # Limiter on number of files
    dbsResults = dbsResults[0:20]
    print "  found %d files, inserting into wmbs..." % (len(dbsResults))

    for dbsResult in dbsResults:
        myFile = File(lfn=dbsResult["LogicalFileName"],
                      size=dbsResult["FileSize"],
                      events=dbsResult["NumberOfEvents"],
                      checksums={"cksum": dbsResult["Checksum"]},
                      locations="cmssrm.fnal.gov",
                      merged=True)
        myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"])
        for lumi in dbsResult["LumiList"]:
            myRun.lumis.append(lumi["LumiSectionNumber"])
        myFile.addRun(myRun)
        myFile.create()
        inputFileset.addFile(myFile)

        dbsFile = DBSBufferFile(lfn=dbsResult["LogicalFileName"],
                                size=dbsResult["FileSize"],
                                events=dbsResult["NumberOfEvents"],
                                checksums={"cksum": dbsResult["Checksum"]},
                                locations="cmssrm.fnal.gov",
                                status="NOTUPLOADED")
        dbsFile.setDatasetPath(datasetPath)
        dbsFile.setAlgorithm(appName="cmsRun",
                             appVer="Unknown",
                             appFam="Unknown",
                             psetHash="Unknown",
                             configContent="Unknown")
        dbsFile.create()

    inputFileset.commit()
    inputFileset.markOpen(False)
    return

Esempio n. 6

File: injectReRecoWorkflow.py Progetto: elasticaso/WMCore

def injectFilesFromDBS(inputFileset, datasetPath):
    """
    _injectFilesFromDBS_

    """
    print "injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name)
    args={}
    args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
    args["version"] = "DBS_2_0_9"
    args["mode"] = "GET"
    dbsApi = DbsApi(args)
    dbsResults = dbsApi.listFileArray(path = datasetPath, retriveList = ["retrive_lumi", "retrive_run"])
    dbsResults = dbsResults[0:10]
    print "  found %d files, inserting into wmbs..." % (len(dbsResults))

    for dbsResult in dbsResults:
        myFile = File(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"],
                      events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]},
                      locations = "cmssrm.fnal.gov", merged = True)
        myRun = Run(runNumber = dbsResult["LumiList"][0]["RunNumber"])
        for lumi in dbsResult["LumiList"]:
            myRun.lumis.append(lumi["LumiSectionNumber"])
        myFile.addRun(myRun)
        myFile.create()
        inputFileset.addFile(myFile)

        dbsFile = DBSBufferFile(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"],
                                events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]},
                                locations = "cmssrm.fnal.gov", status = "LOCAL")
        dbsFile.setDatasetPath(datasetPath)
        dbsFile.setAlgorithm(appName = "cmsRun", appVer = "Unknown", appFam = "Unknown",
                             psetHash = "Unknown", configContent = "Unknown")
        dbsFile.create()
        
    inputFileset.commit()
    inputFileset.markOpen(False)
    return

Esempio n. 7

File: dbsVerify.py Progetto: scarletnorberg/WMCore

    blocks[row[0]].append(row[1])

args = {}
args["url"] = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
dbsApi = DbsApi(args)

badBlocks = []
badFiles = []
for blockName in blocks.keys():
    print("%s:" % blockName)
    blockFiles = []
    try:
        dbsFiles = dbsApi.listFileArray(blockName=blockName)
    except Exception as ex:
        dbsFiles = []

    for dbsFile in dbsFiles:
        blockFiles.append(dbsFile["LogicalFileName"])

    if len(blockFiles) != len(blocks[blockName]):
        print("\tFile count mismatch: %s local, %s global" %
              (len(blocks[blockName]), len(blockFiles)))
        if blockName not in badBlocks:
            badBlocks.append(blockName)

    for blockFile in blocks[blockName]:
        if blockFile not in blockFiles:
            badFiles.append(blockFile)

Esempio n. 8

File: dbsVerify.py Progetto: AndresTanasijczuk/WMCore

    blocks[row[0]].append(row[1])

args = {}
args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
dbsApi = DbsApi(args)

badBlocks = []
badFiles = []
for blockName in blocks.keys():
    print("%s:" % blockName)
    blockFiles = []
    try:
        dbsFiles = dbsApi.listFileArray(blockName = blockName)
    except Exception as ex:
        dbsFiles = []
        
    for dbsFile in dbsFiles:
        blockFiles.append(dbsFile["LogicalFileName"])

    if len(blockFiles) != len(blocks[blockName]):
        print("\tFile count mismatch: %s local, %s global" % (len(blocks[blockName]), len(blockFiles)))
        if blockName not in badBlocks:
            badBlocks.append(blockName)

    for blockFile in blocks[blockName]:
        if blockFile not in blockFiles:
            badFiles.append(blockFile)
            #print "\t File missing: %s" % blockFile

Esempio n. 9

File: dbsFix.py Progetto: ramandeepkumar/WMCore

args = {}
#args["url"] = "https://cmst0dbs.cern.ch:8443/cms_dbs_prod_tier0_writer/servlet/DBSServlet"
args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSWriter"
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
dbsApi = DbsApi(args)

badBlocks = []
badFiles = {}

for blockName in blocks.keys():
    print "%s:" % blockName
    blockFiles = []
    try:
        dbsFiles = dbsApi.listFileArray(blockName = blockName)
    except Exception as ex:
        dbsFiles = []
        
    for dbsFile in dbsFiles:
        blockFiles.append(dbsFile["LogicalFileName"])

    if len(blockFiles) != len(blocks[blockName]):
        print "\tFile count mismatch: %s local, %s global" % (len(blocks[blockName]), len(blockFiles))
        if blockName not in badBlocks:
            badBlocks.append(blockName)

    for blockFile in blocks[blockName]:
        if blockFile not in blockFiles:
            if not badFiles.has_key(blockName):
                badFiles[blockName] = []

Esempio n. 10

File: dbsFix.py Progetto: tsarangi/WMCore

args = {}
#args["url"] = "https://cmst0dbs.cern.ch:8443/cms_dbs_prod_tier0_writer/servlet/DBSServlet"
args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSWriter"
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
dbsApi = DbsApi(args)

badBlocks = []
badFiles = {}

for blockName in blocks.keys():
    print "%s:" % blockName
    blockFiles = []
    try:
        dbsFiles = dbsApi.listFileArray(blockName=blockName)
    except Exception as ex:
        dbsFiles = []

    for dbsFile in dbsFiles:
        blockFiles.append(dbsFile["LogicalFileName"])

    if len(blockFiles) != len(blocks[blockName]):
        print "\tFile count mismatch: %s local, %s global" % (len(
            blocks[blockName]), len(blockFiles))
        if blockName not in badBlocks:
            badBlocks.append(blockName)

    for blockFile in blocks[blockName]:
        if blockFile not in blockFiles:
            if not badFiles.has_key(blockName):