#ran = str(int(random.random()*10000000)) #block = DbsFileBlock ( # Name= "/this/isaqqqqqqqqqxstbddddlock#016712"+ ran, # ) block = DbsFileBlock ( Name="/this/hahah#12345" ) print "Creating block %s" % block try: #api.insertBlock (proc, block) #api.insertBlock ('/TestPrimary1164751189.48/HIT1164751189.48/TestProcessed1164751189.48') #api.insertBlock (proc) #print api.insertBlock ("/test_primary_anzar_001/SIM/TestProcessedDS002", block=None, storage_element=["thisIsMyOnlySE"]) print api.insertBlock ("/test_primary_anzar_001/SIM/TestProcessedDS001", "/this/hahah#12345", storage_element=["thisIsMyOnlySE"]) #api.insertBlock ("/test_primary_anzar_001/SIM/TestProcessedDS002" , "/this/hahah#12345" , ['se1', 'se2', 'se3']) #api.insertBlock ("/test_primary_anzar_001/SIM/TestProcessedDS002" , "/this/hahaah#12345" , ['sea1', 'sea2', 'sea3']) #api.insertBlock ("/TestPrimary1167862926.47/SIM1167862926.47/TestProcessed1167862926.47", "/this/hahah#12345", ['se1', 'se2', 'se3']) #print "Result: %s" % primary except DbsApiException, ex: print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() ) if ex.getErrorCode() not in (None, ""): print "DBS Exception Error Code: ", ex.getErrorCode() print "Done"
#""" print "\n\nInserting primary %s" % primary print api.insertPrimaryDataset (primary) print "\n\nInserting Algorithm %s" % algo print api.insertAlgorithm (algo) print "\n\nInserting Run %s" % run print api.insertRun (run) print "\n\nInserting Processed %s" % proc print api.insertProcessedDataset (proc) print "\n\nInserting Block %s" % block print api.insertBlock (proc, block) print "\n\nInserting Files %s" % [myfile1, myfile2] print api.insertFiles (proc, [myfile1, myfile2], block) print "\n\nInserting Child Processed %s" % procChild print api.insertProcessedDataset (procChild) print "\n\nInserting Block2 %s" % block2 print api.insertBlock (procChild, block2) print "\n\nInserting Block2 %s" % block21 print api.insertBlock (procChild, block21) print "\n\nInserting Files %s" % [myfile3, myfile4] print api.insertFiles (procChild, [myfile3, myfile4], block2)
for blockFile in blocks[blockName]: if blockFile not in blockFiles: if not badFiles.has_key(blockName): badFiles[blockName] = [] badFiles[blockName].append(blockFile) #sys.exit(0) psetInstance = DbsQueryableParameterSet(Hash = "GIBBERISH") for newBlockName in badFiles.keys(): seName = blockLocation[newBlockName] (datasetPath, junk) = newBlockName.split("#", 1) dbsApi.insertBlock(datasetPath, newBlockName, storage_element_list = [seName]) blockRef = dbsApi.listBlocks(dataset = datasetPath, block_name = newBlockName)[0] print blockRef newFiles = [] for newFileLFN in badFiles[newBlockName]: localFile = DBSBufferFile(lfn = newFileLFN) localFile.load(parentage = 1) (primaryDS, procDS, tier) = datasetPath[1:].split("/", 3) primary = DbsPrimaryDataset(Name = primaryDS, Type = "mc") algo = DbsAlgorithm(ExecutableName = localFile["appName"], ApplicationVersion = localFile["appVer"], ApplicationFamily = localFile["appFam"], ParameterSetID = psetInstance)
print "#################################################################################################" print " # files in json: ",nFilesTot print " # files already published: ",nPublishedFiles print " # new files: ",nFiles print "#################################################################################################" #-------------------- # loop over files count = 0 while count < len(files): # ------------------ # insert block print "inserting block..." blockName = dbsApi.insertBlock(datasetPath, None , storage_element_list = [seName]) block = dbsApi.listBlocks(datasetPath, block_name = blockName, storage_element_name = seName)[0] print "...block inserted (name: ",blockName,")" #-------------------- # loop over files start = count stop = min(count+blockSize,len(files)) _files = files[start:stop] dbsFiles = [] print "preparing files for 1 block..."
primary = DbsPrimaryDataset (Name = "test_primary_001a") proc = DbsProcessedDataset ( PrimaryDataset=primary, TierList=['GEN', 'SIM'], Path='/test_primary_001/TestProcessedDS001/GEN-SIM' #Name="TestProcessedDSWithADSParent", ) block = DbsFileBlock ( #Name="/test_primary_001/TestProcessedDSWithADSParent/GEN-SIM#12345" Name="/test_primary_001/TestProcessedDS001/GEN-SIM#12345" ) print "Creating block %s" % block try: # ALL Valid Options below #print api.insertBlock (proc) #print api.insertBlock (proc, block) #print api.insertBlock ("/test_primary_001/TestProcessedDS001/GEN-SIM", "/test_primary_001/TestProcessedDS001/GEN-SIM#123456") print api.insertBlock ("/test_primary_001/TestProcessedDS001/GEN-SIM", "/test_primary_001/TestProcessedDS001/GEN-SIM#123456" , ['se1', 'se2', 'se3']) except DbsApiException, ex: print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() ) if ex.getErrorCode() not in (None, ""): print "DBS Exception Error Code: ", ex.getErrorCode() print "Done"
for blockFile in blocks[blockName]: if blockFile not in blockFiles: if not badFiles.has_key(blockName): badFiles[blockName] = [] badFiles[blockName].append(blockFile) #sys.exit(0) psetInstance = DbsQueryableParameterSet(Hash="GIBBERISH") for newBlockName in badFiles.keys(): seName = blockLocation[newBlockName] (datasetPath, junk) = newBlockName.split("#", 1) dbsApi.insertBlock(datasetPath, newBlockName, storage_element_list=[seName]) blockRef = dbsApi.listBlocks(dataset=datasetPath, block_name=newBlockName)[0] print blockRef newFiles = [] for newFileLFN in badFiles[newBlockName]: localFile = DBSBufferFile(lfn=newFileLFN) localFile.load(parentage=1) (primaryDS, procDS, tier) = datasetPath[1:].split("/", 3) primary = DbsPrimaryDataset(Name=primaryDS, Type="mc") algo = DbsAlgorithm(ExecutableName=localFile["appName"], ApplicationVersion=localFile["appVer"],
proc = DbsProcessedDataset(PrimaryDataset=primary, TierList=['GEN', 'SIM'], Path='/test_primary_001/TestProcessedDS001/GEN-SIM' #Name="TestProcessedDSWithADSParent", ) block = DbsFileBlock( #Name="/test_primary_001/TestProcessedDSWithADSParent/GEN-SIM#12345" Name="/test_primary_001/TestProcessedDS001/GEN-SIM#12345") print "Creating block %s" % block try: # ALL Valid Options below #print api.insertBlock (proc) #print api.insertBlock (proc, block) #print api.insertBlock ("/test_primary_001/TestProcessedDS001/GEN-SIM", "/test_primary_001/TestProcessedDS001/GEN-SIM#123456") print api.insertBlock( "/test_primary_001/TestProcessedDS001/GEN-SIM", "/test_primary_001/TestProcessedDS001/GEN-SIM#123456", ['se1', 'se2', 'se3']) except DbsApiException, ex: print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage()) if ex.getErrorCode() not in (None, ""): print "DBS Exception Error Code: ", ex.getErrorCode() print "Done"
AlgoList=algoList, RunsList=runList, ) for t in tireList: tire=t #Create block block = DbsFileBlock (Name= "/"+ primaryName+ "/"+ processedName + "/"+ tire +"#01") try: api.insertProcessedDataset (proc) print "## Inserted Processed: %s ##" % proc #import pdb #pdb.set_trace() api.insertBlock (proc, block) print "## Inserted Block: %s ##" % block except DbsApiException, ex: print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() ) if ex.getErrorCode() not in (None, ""): print "DBS Exception Error Code: ", ex.getErrorCode() #Now insert LFNs numberOfFile = 0 fileList = [] LFNList = LFNDic.keys() lastPsName = LFNPsDic[LFNList[0]] for LFN in LFNList: if(lastPsName == LFNPsDic[LFN] and numberOfFile < 350): numberOfFile = numberOfFile + 1 fileList.append( DbsFile (Checksum= LFNChsmDic[LFN],
Status="VALID", ValidationStatus="VALID", FileType="EDM", Dataset=proc, AlgoList=[algo], LumiList=[lumi1, lumi2], TierList=["GEN", "SIM"], BranchHash="001234565798685", # ParentList = ['NEW_TEST0003'] ) # Make a choice block = DbsFileBlock(StorageElement=["test1", "test3"]) block["Name"] = "/test_primary_001/TestProcessedDS001/GEN-SIM#12345-" + str(HOW_MANY_FILES) print "Inserting Files Into", api.insertBlock(proc, block) # print "Wait........" try: each_call = [] time_taken = 0.0 for i in range(HOW_MANY_FILES): rnd = str(os.popen("uuidgen").readline().strip()) myfile1["LogicalFileName"] = "NEW_TEST" + rnd # print myfile1['LogicalFileName'] prf = rnd + ".prof" p = hotshot.Profile(prf) # Insert in a Block out = p.run("api.insertFiles (proc, [myfile1], block)") stats = hotshot.stats.load(prf) stats.strip_dirs() stats.sort_stats("time", "calls")
ValidationStatus='VALID', FileType='EDM', Dataset=proc, AlgoList=[algo], LumiList=[lumi1, lumi2], TierList=['GEN', 'SIM'], BranchHash="001234565798685", #ParentList = ['NEW_TEST0003'] ) # Make a choice block = DbsFileBlock(StorageElement=['test1', 'test3'], ) block['Name'] = "/test_primary_001/TestProcessedDS001/GEN-SIM#12345-" + str( HOW_MANY_FILES) print "Inserting Files Into", api.insertBlock(proc, block) #print "Wait........" try: each_call = [] time_taken = 0.0 for i in range(HOW_MANY_FILES): rnd = str(os.popen('uuidgen').readline().strip()) myfile1['LogicalFileName'] = 'NEW_TEST' + rnd #print myfile1['LogicalFileName'] prf = rnd + '.prof' p = hotshot.Profile(prf) #Insert in a Block out = p.run("api.insertFiles (proc, [myfile1], block)") stats = hotshot.stats.load(prf) stats.strip_dirs() stats.sort_stats('time', 'calls')