#ran = str(int(random.random()*10000000))
#block = DbsFileBlock (
# Name= "/this/isaqqqqqqqqqxstbddddlock#016712"+ ran,
# )
block = DbsFileBlock (
Name="/this/hahah#12345"
)
print "Creating block %s" % block
try:
#api.insertBlock (proc, block)
#api.insertBlock ('/TestPrimary1164751189.48/HIT1164751189.48/TestProcessed1164751189.48')
#api.insertBlock (proc)
#print api.insertBlock ("/test_primary_anzar_001/SIM/TestProcessedDS002", block=None, storage_element=["thisIsMyOnlySE"])
print api.insertBlock ("/test_primary_anzar_001/SIM/TestProcessedDS001", "/this/hahah#12345", storage_element=["thisIsMyOnlySE"])
#api.insertBlock ("/test_primary_anzar_001/SIM/TestProcessedDS002" , "/this/hahah#12345" , ['se1', 'se2', 'se3'])
#api.insertBlock ("/test_primary_anzar_001/SIM/TestProcessedDS002" , "/this/hahaah#12345" , ['sea1', 'sea2', 'sea3'])
#api.insertBlock ("/TestPrimary1167862926.47/SIM1167862926.47/TestProcessed1167862926.47", "/this/hahah#12345", ['se1', 'se2', 'se3'])
#print "Result: %s" % primary
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() )
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
print "Done"
#"""
print "\n\nInserting primary %s" % primary
print api.insertPrimaryDataset (primary)
print "\n\nInserting Algorithm %s" % algo
print api.insertAlgorithm (algo)
print "\n\nInserting Run %s" % run
print api.insertRun (run)
print "\n\nInserting Processed %s" % proc
print api.insertProcessedDataset (proc)
print "\n\nInserting Block %s" % block
print api.insertBlock (proc, block)
print "\n\nInserting Files %s" % [myfile1, myfile2]
print api.insertFiles (proc, [myfile1, myfile2], block)
print "\n\nInserting Child Processed %s" % procChild
print api.insertProcessedDataset (procChild)
print "\n\nInserting Block2 %s" % block2
print api.insertBlock (procChild, block2)
print "\n\nInserting Block2 %s" % block21
print api.insertBlock (procChild, block21)
print "\n\nInserting Files %s" % [myfile3, myfile4]
print api.insertFiles (procChild, [myfile3, myfile4], block2)
for blockFile in blocks[blockName]:
if blockFile not in blockFiles:
if not badFiles.has_key(blockName):
badFiles[blockName] = []
badFiles[blockName].append(blockFile)
#sys.exit(0)
psetInstance = DbsQueryableParameterSet(Hash = "GIBBERISH")
for newBlockName in badFiles.keys():
seName = blockLocation[newBlockName]
(datasetPath, junk) = newBlockName.split("#", 1)
dbsApi.insertBlock(datasetPath, newBlockName, storage_element_list = [seName])
blockRef = dbsApi.listBlocks(dataset = datasetPath, block_name = newBlockName)[0]
print blockRef
newFiles = []
for newFileLFN in badFiles[newBlockName]:
localFile = DBSBufferFile(lfn = newFileLFN)
localFile.load(parentage = 1)
(primaryDS, procDS, tier) = datasetPath[1:].split("/", 3)
primary = DbsPrimaryDataset(Name = primaryDS, Type = "mc")
algo = DbsAlgorithm(ExecutableName = localFile["appName"],
ApplicationVersion = localFile["appVer"],
ApplicationFamily = localFile["appFam"],
ParameterSetID = psetInstance)
print "#################################################################################################"
print " # files in json: ",nFilesTot
print " # files already published: ",nPublishedFiles
print " # new files: ",nFiles
print "#################################################################################################"
#--------------------
# loop over files
count = 0
while count < len(files):
# ------------------
# insert block
print "inserting block..."
blockName = dbsApi.insertBlock(datasetPath, None ,
storage_element_list = [seName])
block = dbsApi.listBlocks(datasetPath,
block_name = blockName,
storage_element_name = seName)[0]
print "...block inserted (name: ",blockName,")"
#--------------------
# loop over files
start = count
stop = min(count+blockSize,len(files))
_files = files[start:stop]
dbsFiles = []
print "preparing files for 1 block..."
primary = DbsPrimaryDataset (Name = "test_primary_001a")
proc = DbsProcessedDataset (
PrimaryDataset=primary,
TierList=['GEN', 'SIM'],
Path='/test_primary_001/TestProcessedDS001/GEN-SIM'
#Name="TestProcessedDSWithADSParent",
)
block = DbsFileBlock (
#Name="/test_primary_001/TestProcessedDSWithADSParent/GEN-SIM#12345"
Name="/test_primary_001/TestProcessedDS001/GEN-SIM#12345"
)
print "Creating block %s" % block
try:
# ALL Valid Options below
#print api.insertBlock (proc)
#print api.insertBlock (proc, block)
#print api.insertBlock ("/test_primary_001/TestProcessedDS001/GEN-SIM", "/test_primary_001/TestProcessedDS001/GEN-SIM#123456")
print api.insertBlock ("/test_primary_001/TestProcessedDS001/GEN-SIM", "/test_primary_001/TestProcessedDS001/GEN-SIM#123456" , ['se1', 'se2', 'se3'])
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() )
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
print "Done"
for blockFile in blocks[blockName]:
if blockFile not in blockFiles:
if not badFiles.has_key(blockName):
badFiles[blockName] = []
badFiles[blockName].append(blockFile)
#sys.exit(0)
psetInstance = DbsQueryableParameterSet(Hash="GIBBERISH")
for newBlockName in badFiles.keys():
seName = blockLocation[newBlockName]
(datasetPath, junk) = newBlockName.split("#", 1)
dbsApi.insertBlock(datasetPath,
newBlockName,
storage_element_list=[seName])
blockRef = dbsApi.listBlocks(dataset=datasetPath,
block_name=newBlockName)[0]
print blockRef
newFiles = []
for newFileLFN in badFiles[newBlockName]:
localFile = DBSBufferFile(lfn=newFileLFN)
localFile.load(parentage=1)
(primaryDS, procDS, tier) = datasetPath[1:].split("/", 3)
primary = DbsPrimaryDataset(Name=primaryDS, Type="mc")
algo = DbsAlgorithm(ExecutableName=localFile["appName"],
ApplicationVersion=localFile["appVer"],
proc = DbsProcessedDataset(PrimaryDataset=primary,
TierList=['GEN', 'SIM'],
Path='/test_primary_001/TestProcessedDS001/GEN-SIM'
#Name="TestProcessedDSWithADSParent",
)
block = DbsFileBlock(
#Name="/test_primary_001/TestProcessedDSWithADSParent/GEN-SIM#12345"
Name="/test_primary_001/TestProcessedDS001/GEN-SIM#12345")
print "Creating block %s" % block
try:
# ALL Valid Options below
#print api.insertBlock (proc)
#print api.insertBlock (proc, block)
#print api.insertBlock ("/test_primary_001/TestProcessedDS001/GEN-SIM", "/test_primary_001/TestProcessedDS001/GEN-SIM#123456")
print api.insertBlock(
"/test_primary_001/TestProcessedDS001/GEN-SIM",
"/test_primary_001/TestProcessedDS001/GEN-SIM#123456",
['se1', 'se2', 'se3'])
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(),
ex.getErrorMessage())
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
print "Done"
AlgoList=algoList,
RunsList=runList,
)
for t in tireList:
tire=t
#Create block
block = DbsFileBlock (Name= "/"+ primaryName+ "/"+ processedName + "/"+ tire +"#01")
try:
api.insertProcessedDataset (proc)
print "## Inserted Processed: %s ##" % proc
#import pdb
#pdb.set_trace()
api.insertBlock (proc, block)
print "## Inserted Block: %s ##" % block
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() )
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
#Now insert LFNs
numberOfFile = 0
fileList = []
LFNList = LFNDic.keys()
lastPsName = LFNPsDic[LFNList[0]]
for LFN in LFNList:
if(lastPsName == LFNPsDic[LFN] and numberOfFile < 350):
numberOfFile = numberOfFile + 1
fileList.append( DbsFile (Checksum= LFNChsmDic[LFN],
Status="VALID",
ValidationStatus="VALID",
FileType="EDM",
Dataset=proc,
AlgoList=[algo],
LumiList=[lumi1, lumi2],
TierList=["GEN", "SIM"],
BranchHash="001234565798685",
# ParentList = ['NEW_TEST0003']
)
# Make a choice
block = DbsFileBlock(StorageElement=["test1", "test3"])
block["Name"] = "/test_primary_001/TestProcessedDS001/GEN-SIM#12345-" + str(HOW_MANY_FILES)
print "Inserting Files Into", api.insertBlock(proc, block)
# print "Wait........"
try:
each_call = []
time_taken = 0.0
for i in range(HOW_MANY_FILES):
rnd = str(os.popen("uuidgen").readline().strip())
myfile1["LogicalFileName"] = "NEW_TEST" + rnd
# print myfile1['LogicalFileName']
prf = rnd + ".prof"
p = hotshot.Profile(prf)
# Insert in a Block
out = p.run("api.insertFiles (proc, [myfile1], block)")
stats = hotshot.stats.load(prf)
stats.strip_dirs()
stats.sort_stats("time", "calls")
ValidationStatus='VALID',
FileType='EDM',
Dataset=proc,
AlgoList=[algo],
LumiList=[lumi1, lumi2],
TierList=['GEN', 'SIM'],
BranchHash="001234565798685",
#ParentList = ['NEW_TEST0003']
)
# Make a choice
block = DbsFileBlock(StorageElement=['test1', 'test3'], )
block['Name'] = "/test_primary_001/TestProcessedDS001/GEN-SIM#12345-" + str(
HOW_MANY_FILES)
print "Inserting Files Into", api.insertBlock(proc, block)
#print "Wait........"
try:
each_call = []
time_taken = 0.0
for i in range(HOW_MANY_FILES):
rnd = str(os.popen('uuidgen').readline().strip())
myfile1['LogicalFileName'] = 'NEW_TEST' + rnd
#print myfile1['LogicalFileName']
prf = rnd + '.prof'
p = hotshot.Profile(prf)
#Insert in a Block
out = p.run("api.insertFiles (proc, [myfile1], block)")
stats = hotshot.stats.load(prf)
stats.strip_dirs()
stats.sort_stats('time', 'calls')
