def dbsPublish(self, procds):
# Get file information
fileOps = FileOps(getCastor(procds['PathList'][0]), getDbsUser(procds['PathList'][0]))
# Check if user has dataset files, and DO NOT allow publish if they do not
if len(fileOps.getRootFiles()) == 0:
print "No dataset found on Castor, exiting"
sys.exit(1)
return None
# Check dbsAPI is valid
if self._dbsAPI is None:
print "DBS not connected"
return None
details = procds["PathList"][0].lstrip("/").split("/")
# Check that primary dataset exists
if not self._dbsAPI.listPrimaryDatasets(details[0]):
self._dbsAPI.insertPrimaryDataset(DbsPrimaryDataset(Name=details[0],Type="test"))
if self._dbsAPI.listPrimaryDatasets(details[0]):
parent = None
if len(procds["ParentList"]) > 0:
parent = procds["ParentList"][0].lstrip("/").split("/")
if parent is not None:
parentDatasets = self._dbsAPI.listProcessedDatasets(parent[0],parent[2],parent[1])
# If only one possible parent exists, use it as parent
if len(parentDatasets) == 1:
procds["ParentList"][0] = parentDatasets[0]['PathList'][0]
if len(parentDatasets) == 0:
#if the parent is
if not re.search("---",parent[1]):
self.dbsPublish(CastorToDbsFormatter(procds["ParentList"][0]).getDataset(), None)
else:
print "Parent dataset missing, please add parent before proceeding"
return None
else:
"Please use specific parent name, not REGEX"
return None
# Check if dataset already exists
dbsOutput = self._dbsAPI.listProcessedDatasets(details[0],details[2],details[1])
# Check if self._procds doesn't exist
if len(dbsOutput) == 0:
print "Adding Dataset to DBS"
# Add Dataset
self._dbsAPI.insertProcessedDataset(procds)
block = DbsFileBlock (Name=procds['PathList'][0]+"#1")
# Add the corresponding file block
self._dbsAPI.insertBlock(procds['PathList'][0], block)
else:
print "Dataset already on DBS"
return self._getDbsIDs(procds['PathList'][0])
else:
print "Primary dataset does not exist"
return None
def createDBSFileBlock(blockName):
"""
_createDBSFileBlock_
return a DbsFileBlock object with the block name provided
NOTE: This method DOES NOT create a new block in DBS
"""
return DbsFileBlock(Name=blockName)
FileSize= 12340,
Status= 'VALID',
ValidationStatus = 'VALID',
FileType= 'EDM',
Dataset= proc,
LumiList= [lumi1, lumi2],
TierList= ['SIM', 'GEN'],
AlgoList = [algo],
BranchList=['testbranch01', 'testbranch02'],
ParentList = ['NEW_TEST0004' + mytime]
)
block = DbsFileBlock (
StorageElement=['test1', 'test3'],
Name="/test_primary_001" + mytime + "/TestProcessedDS001" + mytime + "/GEN-SIM#12345"
)
block2 = DbsFileBlock (
StorageElement=['test1', 'test3'],
Name="/test_primary_001" + mytime + "/TestProcessedDS002" + mytime + "/GEN-SIM#44444"
)
block21 = DbsFileBlock (
StorageElement=['test1', 'test3'],
Name="/test_primary_001" + mytime + "/TestProcessedDS002" + mytime + "/GEN-SIM#3333"
)
block3 = DbsFileBlock (
StorageElement=['test1', 'test3'],
Name="/test_primary_001" + mytime + "/TestProcessedDS003" + mytime + "/GEN-SIM#3333"
'NumberOfEvents': 123
}, {
'TriggerTag': 'TestTrig002',
'NumberOfEvents': 345
}, {
'TriggerTag': 'TestTrig003',
'NumberOfEvents': 678
}],
FileAssoc=myfile4)
# Need to provide Block name if YOU want to control Block management (The block named must pre-exist), if NOT then DBS will throw this file in
# Open Block for this Dataset, and will do the Block management too.
# Make a choice
block = DbsFileBlock(
#Name="/test_primary_001/TestProcessedDSWithADSParent/GEN-SIM#123456"
Name="/test_primary_001/TestProcessedDS001/GEN-SIM#123456")
block2 = DbsFileBlock(
StorageElement=['test1', 'test3'],
Name="/test_primary_001/TestProcessedDS001/GEN-SIM#123456")
print "BUG to be fixed in server, cannot handle QueryableMetadata"
print "For now we don't have BLOCK Management on Server side so User need to providea BLOCK"
print "In future it will be an optional parameter"
print "Inserting files in processDS %s" % proc
try:
api.insertFiles(proc, [myfile1], block)
#!/usr/bin/env python
#
# Revision: 1.3 $"
# Id: DBSXMLParser.java,v 1.3 2006/10/26 18:26:04 afaq Exp $"
#
import sys
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsFileBlock import DbsFileBlock
from DBSAPI.dbsOptions import DbsOptionParser
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
block = DbsFileBlock(Name="/test_primary_001/TestProcessedDS001/GEN#123456")
print "Updating the SE in a block"
try:
api.updateSEBlock(block, "se1", "seaa1")
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(),
ex.getErrorMessage())
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
print "Done"
myfile6 = DbsFile(Checksum='000',
LogicalFileName='NEW_TEST00066',
NumberOfEvents=10000,
FileSize=12340,
Status='VALID',
ValidationStatus='VALID',
FileType='EDM',
Dataset=proc,
LumiList=[lumi1, lumi2],
TierList=['SIM', 'GEN'],
AlgoList=[algo],
BranchList=['testbranch01', 'testbranch02'],
ParentList=['NEW_TEST00056'])
block = DbsFileBlock(
StorageElement=['test1', 'test3'],
Name="/test_primary_0011/TestProcessedDS0011/GEN-SIM#123456")
block2 = DbsFileBlock(
StorageElement=['test1', 'test3'],
Name="/test_primary_0011/TestProcessedDS0021/GEN-SIM#444446")
block21 = DbsFileBlock(
StorageElement=['test1', 'test3'],
Name="/test_primary_0011/TestProcessedDS0021/GEN-SIM#33336")
block3 = DbsFileBlock(
StorageElement=['test1', 'test3'],
Name="/test_primary_0011/TestProcessedDS0031/GEN-SIM#33336")
try:
#QueryableMetadata= 'This is a test file',
NumberOfEvents=10000,
FileSize=12340,
Status='VALID',
ValidationStatus='VALID',
FileType='EVD',
Dataset=proc,
#Block= isDictType,
#LumiList= [lumi1, lumi2],
TierList=['SIM', 'RECO'],
AlgoList=[algo],
#ParentList = ['lfn01', 'lfn02']
)
block = DbsFileBlock(
#Name="/test_primary_anzar_001/TestProcessedDS002#879143ef-b527-44cb-867d-fff54f5730db"
Name="/this/hahah#12345")
print "BUG to be fixed in server, cannot handle QueryableMetadata"
print "For now we don't have BLOCK Management on Server side so User need to providea BLOCK"
print "In future it will be an optional parameter"
print "Inserting files in processDS %s" % proc
try:
api.insertFiles(proc, [myfile1, myfile2], None)
#api.insertFiles (proc, [myfile1, myfile2], block)
print "Result: %s" % proc
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(),
apiObj.run("/aa/does/notexist", "INVALID", excep = True)
apiObj.run(path, "IN;VALID", excep = True)
f.write("\n\n***********************updateProcDSStatus API tests***************************")
apiObj = DbsUnitTestApi(api.insertBlock, f)
apiObj.setVerboseLevel(opts.verbose)
f.write("\n\n***********************insertBlock API tests***************************")
#path = "/" + str(proc1['PrimaryDataset']['Name']) + "/" + str(proc1['TierList'][0]) + "/" + str(proc1['Name'])
#blockName = "/"+ mytime + "this/isatestblock#016712"
blockName = path + "#016712"
blockName1 = path + "#016713"
blockName2 = path + "#016714"
#blockName1 = "/"+ mytime + "this/isatestskljblock#016712"
#blockName2 = "/"+ mytime + "thislkss/isatestskljblock#016712"
block = DbsFileBlock (Path = path)
apiObj.run(path, excep = False)
block1 = DbsFileBlock (Name = blockName)
apiObj.run(path, blockName, excep = False)
block = DbsFileBlock (Name= blockName1)
apiObj.run(path, block, excep = False)
block = DbsFileBlock (Name= blockName2)
apiObj.run(proc1, block, excep = False)
apiObj.run(path, "/" + mytime + "this/isatestblock016712", excep = True)
apiObj.run("/absssssssc/dessssssf/hijaaaaaaa", excep = True)
apiObj.run("/abcaaaa/deaaaaaaf/hiaaaaaaaj", "/this/isatestblock#016712", excep = True)
apiObj.run(path, "/thisisatestblock#016712", excep = True)
EndEventNumber=endEvNum1,
LumiStartTime=stLumiTime1,
LumiEndTime=endLumiTime1,
RunNumber=runNumber,
)
lumiObj2 = DbsLumiSection(
LumiSectionNumber=lsNumber2,
StartEventNumber=stEvNum2,
EndEventNumber=endEvNum2,
LumiStartTime=stLumiTime2,
LumiEndTime=endLumiTime2,
RunNumber=runNumber,
)
blockObj1 = DbsFileBlock(Name=blockName1)
blockObj2 = DbsFileBlock(Name=blockName2)
blockObjM = DbsFileBlock(Name=blockNameM)
fileObj1 = DbsFile(Checksum=fileCkecksum1,
LogicalFileName=fileName1,
NumberOfEvents=fileNumEvents1,
FileSize=fileSize1,
Status=fileStatus1,
ValidationStatus=fileValidStatus1,
FileType=fileType1,
Dataset=procObj1,
AlgoList=[algoObj1],
LumiList=[lumiObj1],
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset
from DBSAPI.dbsFileBlock import DbsFileBlock
from DBSAPI.dbsProcessedDataset import DbsProcessedDataset
from DBSAPI.dbsOptions import DbsOptionParser
optManager = DbsOptionParser()
(opts,args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
block = DbsFileBlock (
#Name="/Wmunu/CMSSW_1_6_0-CSA07-2479/RAW#dd92dc6f-7538-4bde-9168-0a47922d83af"
Name="/Wmunu/CMSSW_1_6_0-HLT-1191261655/GEN-SIM-DIGI-RECO#0c8376b7-3467-48e5-ace1-31f53492d6c1"
)
print "Closing block %s" % block
try:
#print api.closeBlock ("/MTCC-070-os-DAQ-MTCC1/CMSSW_0_7_0-RAW-Run-00002120#53bfd044-b925-4446-b181-42ab98cc734a")
print api.closeBlock (block)
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(), ex.getErrorMessage() )
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
print "Done"
#Insert Processed Datatset apiObj = DbsUnitTestApi(api.insertProcessedDataset,f) proc1 = DbsProcessedDataset(PrimaryDataset=pri1, Name="TestProcessed" + mytime, PhysicsGroup="BPositive", Status="VALID", TierList=tierList, AlgoList=[algo1]) apiObj.run(proc1, excep = False) apiObj = DbsUnitTestApi(api.insertBlock, f) path = "/" + str(proc1['PrimaryDataset']['Name']) + "/" + str(proc1['TierList'][0]) + "/" + str(proc1['Name']) #Insert Block block1 = DbsFileBlock (Name = "/" + mytime + "this/isatestblock#016712", Path = path) apiObj.run(path, "/" + mytime + "this/isatestblock#016712" , excep = False) #Insert Run apiObj = DbsUnitTestApi(api.insertRun, f) runNumber1 = 101 + int(time.time()%1000) run1 = DbsRun (RunNumber=runNumber1, NumberOfEvents= 100, NumberOfLumiSections= 20, TotalLuminosity= 2222, StoreNumber= 123, StartOfRun= 'now', EndOfRun= 'never', ) apiObj.run(run1, excep = False)
f.write(
"\n***********************insertProcessedDataset API tests***************************"
)
apiObj = DbsUnitTestApi(api.insertBlock, f)
apiObj.setVerboseLevel(opts.verbose)
f.write(
"\n\n***********************insertBlock API tests***************************"
)
path = "/" + str(proc1['PrimaryDataset']['Name']) + "/" + str(
proc1['TierList'][0]) + "/" + str(proc1['Name'])
blockName = "/" + mytime + "this/isatestblock#016712"
blockName1 = "/" + mytime + "this/isatestskljblock#016712"
blockName2 = "/" + mytime + "thislkss/isatestskljblock#016712"
block = DbsFileBlock(Path=path)
apiObj.run(path, excep=False)
block1 = DbsFileBlock(Name=blockName)
apiObj.run(path, blockName, excep=False)
block = DbsFileBlock(Name=blockName1)
apiObj.run(path, block, excep=False)
block = DbsFileBlock(Name=blockName2)
apiObj.run(proc1, block, excep=False)
apiObj.run(path, blockName, excep=True)
apiObj.run(path, "/" + mytime + "this/isatestblock016712", excep=True)
apiObj.run("/absssssssc/dessssssf/hijaaaaaaa", excep=True)
apiObj.run("/abcaaaa/deaaaaaaf/hiaaaaaaaj",
import sys
import random
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset
from DBSAPI.dbsFileBlock import DbsFileBlock
from DBSAPI.dbsProcessedDataset import DbsProcessedDataset
from DBSAPI.dbsOptions import DbsOptionParser
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
name = sys.argv[1]
block = DbsFileBlock(Name=name)
#print "Closing block %s" % block
try:
#print api.closeBlock ("/MTCC-070-os-DAQ-MTCC1/CMSSW_0_7_0-RAW-Run-00002120#53bfd044-b925-4446-b181-42ab98cc734a")
print api.closeBlock(block)
print block
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(),
ex.getErrorMessage())
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
print "Done"
apiObj = DbsUnitTestApi(api.insertProcessedDataset,f)
proc1 = DbsProcessedDataset(PrimaryDataset=pri1,
Name="StressTestProcessed" + mytime,
PhysicsGroup="BPositive",
Status="VALID",
TierList=tierList,
AlgoList=[algo1])
apiObj.run(proc1, excep = False)
apiObj = DbsUnitTestApi(api.insertBlock, f)
path = "/" + str(proc1['PrimaryDataset']['Name']) + "/" + str(proc1['Name']) + "/" + tierName1 + "-" + tierName2
print "PATH: %s" % path
#Insert Block
block1 = DbsFileBlock (Name = path+'#01234-0567', Path = path)
apiObj.run(path, block1 , excep = False)
#Insert Run
apiObj = DbsUnitTestApi(api.insertRun, f)
runNumber1 = 101 + int(time.time()%1000)
run1 = DbsRun (RunNumber=runNumber1,
NumberOfEvents= 100,
NumberOfLumiSections= 20,
TotalLuminosity= 2222,
StoreNumber= 123,
StartOfRun= 'now',
EndOfRun= 'never',
)
apiObj.run(run1, excep = False)
from DBSAPI.dbsOptions import DbsOptionParser
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
primary = DbsPrimaryDataset(Name="test_primary_001a")
proc = DbsProcessedDataset(PrimaryDataset=primary,
TierList=['GEN', 'SIM'],
Path='/test_primary_001/TestProcessedDS001/GEN-SIM'
#Name="TestProcessedDSWithADSParent",
)
block = DbsFileBlock(
#Name="/test_primary_001/TestProcessedDSWithADSParent/GEN-SIM#12345"
Name="/test_primary_001/TestProcessedDS001/GEN-SIM#12345")
print "Creating block %s" % block
try:
# ALL Valid Options below
#print api.insertBlock (proc)
#print api.insertBlock (proc, block)
#print api.insertBlock ("/test_primary_001/TestProcessedDS001/GEN-SIM", "/test_primary_001/TestProcessedDS001/GEN-SIM#123456")
print api.insertBlock(
"/test_primary_001/TestProcessedDS001/GEN-SIM",
"/test_primary_001/TestProcessedDS001/GEN-SIM#123456",
['se1', 'se2', 'se3'])
except DbsApiException, ex:
import random
from DBSAPI.dbsApi import DbsApi
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset
from DBSAPI.dbsFileBlock import DbsFileBlock
from DBSAPI.dbsProcessedDataset import DbsProcessedDataset
from DBSAPI.dbsOptions import DbsOptionParser
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
block = DbsFileBlock(
Name=
"/SiStripCommissioning08-edm/Online/RAW#e7fff7a5-7681-4361-8dd1-cd568eafd362"
#Name="/Wmunu/CMSSW_1_6_0-HLT-1191261655/GEN-SIM-DIGI-RECO#0c8376b7-3467-48e5-ace1-31f53492d6c1"
)
print "Opening a block %s" % block
try:
print api.openBlock(block)
#print api.closeBlock (block)
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(),
ex.getErrorMessage())
if ex.getErrorCode() not in (None, ""):
print "DBS Exception Error Code: ", ex.getErrorCode()
print "Done"
EndEventNumber=endEvNum1,
LumiStartTime=stLumiTime1,
LumiEndTime=endLumiTime1,
RunNumber=runNumber,
)
lumiObj2 = DbsLumiSection(
LumiSectionNumber=lsNumber2,
StartEventNumber=stEvNum2,
EndEventNumber=endEvNum2,
LumiStartTime=stLumiTime2,
LumiEndTime=endLumiTime2,
RunNumber=runNumber,
)
blockObj1 = DbsFileBlock(Name=blockName1)
blockObj2 = DbsFileBlock(Name=blockName2)
blockObj3 = DbsFileBlock(Name=blockName3)
blockObj4 = DbsFileBlock(Name=blockName4)
blockObjM = DbsFileBlock(Name=blockNameM)
blockObjG = DbsFileBlock(Name=blockNameG)
fileObj1 = DbsFile(Checksum=fileCkecksum1,
Adler32=fileAdler321,
Md5=fileMd51,
LogicalFileName=fileName1,
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
try:
merge_algo = DbsAlgorithm(
ExecutableName="EdmFastMerge",
ApplicationVersion="v101",
ApplicationFamily="Merge",
)
path = "/test_primary_001/TestProcessedDS001/SIM"
merge_proc = api.insertMergedDataset(path, "ThisISMergedDataset001",
merge_algo)
# File will go into THIS Block
block = DbsFileBlock(StorageElement=['test1', 'test3'],
Name="/test_primary_001/TestProcessedDS001/SIM#12345")
merged_file = DbsFile(
Checksum='00000',
LogicalFileName='MERGEDFILE_001',
NumberOfEvents=10000,
FileSize=1000000,
Status='VALID',
ValidationStatus='VALID',
FileType='EVD',
Dataset=merge_proc,
Block=block,
AlgoList=[merge_algo],
)
#api.insertFiles (proc, [myfile1], block)
Checksum='999',
NumberOfEvents=10000,
FileSize=12340,
Status='VALID',
ValidationStatus='VALID',
FileType='EDM',
Dataset=proc,
AlgoList=[algo],
LumiList=[lumi1, lumi2],
TierList=['GEN', 'SIM'],
BranchHash="001234565798685",
#ParentList = ['NEW_TEST0003']
)
# Make a choice
block = DbsFileBlock(StorageElement=['test1', 'test3'], )
block['Name'] = "/test_primary_001/TestProcessedDS001/GEN-SIM#12345-" + str(
HOW_MANY_FILES)
print "Inserting Files Into", api.insertBlock(proc, block)
#print "Wait........"
try:
each_call = []
time_taken = 0.0
for i in range(HOW_MANY_FILES):
rnd = str(os.popen('uuidgen').readline().strip())
myfile1['LogicalFileName'] = 'NEW_TEST' + rnd
#print myfile1['LogicalFileName']
prf = rnd + '.prof'
p = hotshot.Profile(prf)
#Insert in a Block