Example #1
0
#ran = str(int(random.random()*10000000))
#block = DbsFileBlock (
#		Name= "/this/isaqqqqqqqqqxstbddddlock#016712"+ ran,
#         )


block = DbsFileBlock (
         Name="/this/hahah#12345"
         )

print "Creating block %s" % block

try:
    #api.insertBlock (proc, block)
    #api.insertBlock ('/TestPrimary1164751189.48/HIT1164751189.48/TestProcessed1164751189.48') 
    #api.insertBlock (proc)
    #print api.insertBlock ("/test_primary_anzar_001/SIM/TestProcessedDS002", block=None, storage_element=["thisIsMyOnlySE"])
    print api.insertBlock ("/test_primary_anzar_001/SIM/TestProcessedDS001", "/this/hahah#12345", storage_element=["thisIsMyOnlySE"])
    #api.insertBlock ("/test_primary_anzar_001/SIM/TestProcessedDS002" , "/this/hahah#12345" , ['se1', 'se2', 'se3'])
    #api.insertBlock ("/test_primary_anzar_001/SIM/TestProcessedDS002" , "/this/hahaah#12345" , ['sea1', 'sea2', 'sea3'])
    #api.insertBlock ("/TestPrimary1167862926.47/SIM1167862926.47/TestProcessed1167862926.47", "/this/hahah#12345", ['se1', 'se2', 'se3'])
    #print "Result: %s" % primary
except DbsApiException, ex:
  print "Caught API Exception %s: %s "  % (ex.getClassName(), ex.getErrorMessage() )
  if ex.getErrorCode() not in (None, ""):
    print "DBS Exception Error Code: ", ex.getErrorCode()


print "Done"

Example #2
0
	
    #"""	
    print "\n\nInserting primary %s" % primary
    print api.insertPrimaryDataset (primary)

    print "\n\nInserting Algorithm %s" % algo	
    print api.insertAlgorithm (algo)
    
    print "\n\nInserting Run %s" % run	
    print api.insertRun (run)
    
    print "\n\nInserting Processed %s" % proc	
    print api.insertProcessedDataset (proc)

    print "\n\nInserting Block %s" % block	
    print api.insertBlock (proc, block)
    
    print "\n\nInserting Files %s" % [myfile1, myfile2]	
    print api.insertFiles (proc, [myfile1, myfile2], block)

    print "\n\nInserting Child Processed %s" % procChild
    print api.insertProcessedDataset (procChild)

    print "\n\nInserting Block2 %s" % block2	
    print api.insertBlock (procChild, block2)
 
    print "\n\nInserting Block2 %s" % block21	
    print api.insertBlock (procChild, block21)

    print "\n\nInserting Files %s" % [myfile3, myfile4]	
    print api.insertFiles (procChild, [myfile3, myfile4], block2)
Example #3
0
    for blockFile in blocks[blockName]:
        if blockFile not in blockFiles:
            if not badFiles.has_key(blockName):
                badFiles[blockName] = []
                
            badFiles[blockName].append(blockFile)

    #sys.exit(0)

psetInstance = DbsQueryableParameterSet(Hash = "GIBBERISH")

for newBlockName in badFiles.keys():
    seName = blockLocation[newBlockName]
    (datasetPath, junk) = newBlockName.split("#", 1)
    dbsApi.insertBlock(datasetPath, newBlockName, storage_element_list = [seName])

    blockRef = dbsApi.listBlocks(dataset = datasetPath, block_name = newBlockName)[0]
    print blockRef

    newFiles = []
    for newFileLFN in badFiles[newBlockName]:
        localFile = DBSBufferFile(lfn = newFileLFN)
        localFile.load(parentage = 1)

        (primaryDS, procDS, tier) = datasetPath[1:].split("/", 3)
        primary = DbsPrimaryDataset(Name = primaryDS, Type = "mc")
        algo = DbsAlgorithm(ExecutableName = localFile["appName"],
                            ApplicationVersion = localFile["appVer"],
                            ApplicationFamily = localFile["appFam"],
                            ParameterSetID = psetInstance)
Example #4
0
        print "#################################################################################################"
        print " # files in json:           ",nFilesTot
        print " # files already published: ",nPublishedFiles
        print " # new files:               ",nFiles
        print "#################################################################################################"
            
        #--------------------
        # loop over files
        count = 0
        while count < len(files):

            # ------------------
            # insert block
            print "inserting block..."
            blockName = dbsApi.insertBlock(datasetPath, None ,
                                           storage_element_list = [seName])
            block = dbsApi.listBlocks(datasetPath,
                                       block_name = blockName,
                                       storage_element_name = seName)[0]
            print "...block inserted (name: ",blockName,")"

            #--------------------
            # loop over files
            start = count
            stop = min(count+blockSize,len(files))
            _files = files[start:stop]

            dbsFiles = []

            print "preparing files for 1 block..."
            
Example #5
0
primary = DbsPrimaryDataset (Name = "test_primary_001a")

proc = DbsProcessedDataset (
         PrimaryDataset=primary,
         TierList=['GEN', 'SIM'],
	 Path='/test_primary_001/TestProcessedDS001/GEN-SIM'
         #Name="TestProcessedDSWithADSParent",
         )

block = DbsFileBlock (
         #Name="/test_primary_001/TestProcessedDSWithADSParent/GEN-SIM#12345"
         Name="/test_primary_001/TestProcessedDS001/GEN-SIM#12345"
         )

print "Creating block %s" % block

try:
    # ALL Valid Options below
    #print api.insertBlock (proc)
    #print api.insertBlock (proc, block)
    #print api.insertBlock ("/test_primary_001/TestProcessedDS001/GEN-SIM", "/test_primary_001/TestProcessedDS001/GEN-SIM#123456")
    print api.insertBlock ("/test_primary_001/TestProcessedDS001/GEN-SIM", "/test_primary_001/TestProcessedDS001/GEN-SIM#123456" , ['se1', 'se2', 'se3'])

except DbsApiException, ex:
  print "Caught API Exception %s: %s "  % (ex.getClassName(), ex.getErrorMessage() )
  if ex.getErrorCode() not in (None, ""):
    print "DBS Exception Error Code: ", ex.getErrorCode()

print "Done"

Example #6
0
    for blockFile in blocks[blockName]:
        if blockFile not in blockFiles:
            if not badFiles.has_key(blockName):
                badFiles[blockName] = []

            badFiles[blockName].append(blockFile)

    #sys.exit(0)

psetInstance = DbsQueryableParameterSet(Hash="GIBBERISH")

for newBlockName in badFiles.keys():
    seName = blockLocation[newBlockName]
    (datasetPath, junk) = newBlockName.split("#", 1)
    dbsApi.insertBlock(datasetPath,
                       newBlockName,
                       storage_element_list=[seName])

    blockRef = dbsApi.listBlocks(dataset=datasetPath,
                                 block_name=newBlockName)[0]
    print blockRef

    newFiles = []
    for newFileLFN in badFiles[newBlockName]:
        localFile = DBSBufferFile(lfn=newFileLFN)
        localFile.load(parentage=1)

        (primaryDS, procDS, tier) = datasetPath[1:].split("/", 3)
        primary = DbsPrimaryDataset(Name=primaryDS, Type="mc")
        algo = DbsAlgorithm(ExecutableName=localFile["appName"],
                            ApplicationVersion=localFile["appVer"],
Example #7
0
proc = DbsProcessedDataset(PrimaryDataset=primary,
                           TierList=['GEN', 'SIM'],
                           Path='/test_primary_001/TestProcessedDS001/GEN-SIM'
                           #Name="TestProcessedDSWithADSParent",
                           )

block = DbsFileBlock(
    #Name="/test_primary_001/TestProcessedDSWithADSParent/GEN-SIM#12345"
    Name="/test_primary_001/TestProcessedDS001/GEN-SIM#12345")

print "Creating block %s" % block

try:
    # ALL Valid Options below
    #print api.insertBlock (proc)
    #print api.insertBlock (proc, block)
    #print api.insertBlock ("/test_primary_001/TestProcessedDS001/GEN-SIM", "/test_primary_001/TestProcessedDS001/GEN-SIM#123456")
    print api.insertBlock(
        "/test_primary_001/TestProcessedDS001/GEN-SIM",
        "/test_primary_001/TestProcessedDS001/GEN-SIM#123456",
        ['se1', 'se2', 'se3'])

except DbsApiException, ex:
    print "Caught API Exception %s: %s " % (ex.getClassName(),
                                            ex.getErrorMessage())
    if ex.getErrorCode() not in (None, ""):
        print "DBS Exception Error Code: ", ex.getErrorCode()

print "Done"
Example #8
0
                                        AlgoList=algoList,
                                        RunsList=runList,


                                )
    for t in tireList:
        tire=t 
        
    #Create block
    block = DbsFileBlock (Name= "/"+ primaryName+ "/"+ processedName + "/"+ tire +"#01")                   
    try:
         api.insertProcessedDataset (proc)
         print "## Inserted Processed: %s ##" % proc
         #import pdb
         #pdb.set_trace()
         api.insertBlock (proc, block)
         print "## Inserted Block: %s ##" % block
    except DbsApiException, ex:
                print "Caught API Exception %s: %s "  % (ex.getClassName(), ex.getErrorMessage() )
                if ex.getErrorCode() not in (None, ""):
                   print "DBS Exception Error Code: ", ex.getErrorCode()

#Now insert LFNs
numberOfFile = 0
fileList = []
LFNList = LFNDic.keys()
lastPsName = LFNPsDic[LFNList[0]]
for LFN in LFNList:
    if(lastPsName == LFNPsDic[LFN] and numberOfFile < 350): 
        numberOfFile = numberOfFile + 1
        fileList.append( DbsFile (Checksum= LFNChsmDic[LFN],
    Status="VALID",
    ValidationStatus="VALID",
    FileType="EDM",
    Dataset=proc,
    AlgoList=[algo],
    LumiList=[lumi1, lumi2],
    TierList=["GEN", "SIM"],
    BranchHash="001234565798685",
    # ParentList = ['NEW_TEST0003']
)

# Make a choice
block = DbsFileBlock(StorageElement=["test1", "test3"])

block["Name"] = "/test_primary_001/TestProcessedDS001/GEN-SIM#12345-" + str(HOW_MANY_FILES)
print "Inserting Files Into", api.insertBlock(proc, block)
# print "Wait........"
try:
    each_call = []
    time_taken = 0.0
    for i in range(HOW_MANY_FILES):
        rnd = str(os.popen("uuidgen").readline().strip())
        myfile1["LogicalFileName"] = "NEW_TEST" + rnd
        # print myfile1['LogicalFileName']
        prf = rnd + ".prof"
        p = hotshot.Profile(prf)
        # Insert in a Block
        out = p.run("api.insertFiles (proc, [myfile1], block)")
        stats = hotshot.stats.load(prf)
        stats.strip_dirs()
        stats.sort_stats("time", "calls")
Example #10
0
    ValidationStatus='VALID',
    FileType='EDM',
    Dataset=proc,
    AlgoList=[algo],
    LumiList=[lumi1, lumi2],
    TierList=['GEN', 'SIM'],
    BranchHash="001234565798685",
    #ParentList = ['NEW_TEST0003']
)

# Make a choice
block = DbsFileBlock(StorageElement=['test1', 'test3'], )

block['Name'] = "/test_primary_001/TestProcessedDS001/GEN-SIM#12345-" + str(
    HOW_MANY_FILES)
print "Inserting Files Into", api.insertBlock(proc, block)
#print "Wait........"
try:
    each_call = []
    time_taken = 0.0
    for i in range(HOW_MANY_FILES):
        rnd = str(os.popen('uuidgen').readline().strip())
        myfile1['LogicalFileName'] = 'NEW_TEST' + rnd
        #print myfile1['LogicalFileName']
        prf = rnd + '.prof'
        p = hotshot.Profile(prf)
        #Insert in a Block
        out = p.run("api.insertFiles (proc, [myfile1], block)")
        stats = hotshot.stats.load(prf)
        stats.strip_dirs()
        stats.sort_stats('time', 'calls')