def createProcessedDataset(primaryDataset,
algorithm,
datasetInfo,
apiRef=None):
"""
_createProcessedDataset_
"""
physicsGroup = datasetInfo.get("PhysicsGroup", "NoGroup")
status = datasetInfo.get("Status", "VALID")
dataTier = datasetInfo['DataTier']
globalTag = datasetInfo['Conditions']
### FEDE FOR PROCESSED DATASET ###
description = datasetInfo.get('Description', None)
if globalTag is None: globalTag = ''
parents = []
inputDataset = datasetInfo.get('ParentDataset', None)
if inputDataset != None:
parents.append(inputDataset)
tierList = makeTierList(datasetInfo['DataTier'])
name = datasetInfo['ProcessedDataset']
algolist = []
if algorithm not in ('', None):
algolist = [algorithm]
processedDataset = DbsProcessedDataset(
PrimaryDataset=primaryDataset,
AlgoList=algolist,
Name=name,
TierList=tierList,
ParentList=parents,
PhysicsGroup=physicsGroup,
Status=status,
GlobalTag=globalTag,
)
if apiRef != None:
apiRef.insertProcessedDataset(processedDataset)
### FEDE TO UPDATE THE DESCRIPTION FIELD ###
if description != None:
apiRef.updateProcDSDesc(processedDataset, description)
logging.debug(
"PrimaryDataset: %s ProcessedDataset: %s DataTierList: %s requested by PhysicsGroup: %s "
% (primaryDataset['Name'], name, tierList, physicsGroup))
return processedDataset
def createProcessedDataset(primaryDataset,
algorithm,
datasetInfo,
apiRef=None):
"""
_createProcessedDataset_
"""
physicsGroup = datasetInfo.get("PhysicsGroup", "NoGroup")
status = datasetInfo.get("Status", "VALID")
dataTier = datasetInfo['DataTier']
globalTag = datasetInfo.get('Conditions', None)
if globalTag is None: globalTag = ''
parents = []
inputDataset = datasetInfo.get('ParentDataset', None)
if inputDataset != None:
parents.append(inputDataset)
tierList = makeTierList(datasetInfo['DataTier'])
name = datasetInfo['ProcessedDataset']
algolist = []
if algorithm not in ('', None):
algolist = list(algorithm)
processedDataset = DbsProcessedDataset(
PrimaryDataset=primaryDataset,
AlgoList=algolist,
Name=name,
TierList=tierList,
ParentList=parents,
PhysicsGroup=physicsGroup,
Status=status,
GlobalTag=globalTag,
)
if apiRef != None:
apiRef.insertProcessedDataset(processedDataset)
#
logging.debug(
"PrimaryDataset: %s ProcessedDataset: %s DataTierList: %s requested by PhysicsGroup: %s "
% (primaryDataset['Name'], name, tierList, physicsGroup))
return processedDataset
def createProcessedDataset(algorithm,
apiRef,
primary,
processedName,
dataTier,
group="NoGroup",
status="VALID",
globalTag='',
parent=None):
"""
_createProcessedDataset_
Create a processed dataset
"""
if parent != None:
parents = [parent]
else:
parents = []
tierList = dataTier.split("-")
if not type(algorithm) == list:
algorithm = [algorithm]
processedDataset = DbsProcessedDataset(PrimaryDataset=primary,
AlgoList=algorithm,
Name=processedName,
TierList=tierList,
ParentList=parents,
PhysicsGroup=group,
Status=status,
GlobalTag=globalTag)
if apiRef != None:
try:
apiRef.insertProcessedDataset(processedDataset)
except DbsException as ex:
msg = "Error in DBSInterface.createProcessedDataset(%s)\n" % processedName
msg += formatEx(ex)
logging.error(msg)
raise DBSInterfaceError(msg)
logging.info("PrimaryDataset: %s ProcessedDataset: %s DataTierList: %s requested by PhysicsGroup: %s " \
% (primary['Name'], processedName, dataTier, group))
return processedDataset
def __init__(self, DSName, user):
parent = getUnknownParentWithDbs(getDbs(DSName))
path = getDbsWithUser(DSName, user)
parts = path.lstrip("/").split("/")
primary = DbsPrimaryDataset(Name=parts[0])
tierList = parts[2].split("-")
self.procds = DbsProcessedDataset(
PrimaryDataset=primary,
Name=parts[1],
PhysicsGroup="CMG",
Status="VALID",
TierList=tierList,
PathList=[
path,
],
DateCreated=datetime.now().strftime("%s"))
if parent is not None:
self.procds['ParentList'] = [
parent,
]
if user is not None:
self.procds['CreatedBy'] = user
lumi = DbsLumiSection (startEventNumber='10 0')
apiObj.run(lumi, excep = True)
f.write("\n***********************updateLumiSection API tests***************************")
f.write("\n\n***********************insertProcessedDataset API tests***************************")
tierList = [tierName1, tierName2]
apiObj = DbsUnitTestApi(api.insertProcessedDataset,f)
apiObj.setVerboseLevel(opts.verbose)
proc1 = DbsProcessedDataset(PrimaryDataset=pri1,
Name="TestProcessed" + mytime,
PhysicsGroup="BPositive",
Status="VALID",
TierList=tierList,
AlgoList=[algo1, algo2],
RunsList=[runNumber1],
)
apiObj.run(proc1, excep = False)
proc3 = DbsProcessedDataset(PrimaryDataset=pri2,
Name="TestProcessed" + mytime,
PhysicsGroup="BPositive",
Status="VALID",
TierList=tierList,
AlgoList=[algo1, algo2],
RunsList=[runNumber1],
)
apiObj.run(proc3, excep = False)
apiObj.run(algo1, excep = False)
#Insert Tier
apiObj = DbsUnitTestApi(api.insertTier, f)
tierName1 = "GEN"
tierName2 = "SIM"
apiObj.run(tierName1, excep = False)
apiObj.run(tierName2, excep = False)
tierList = [tierName1, tierName2]
#Insert Processed Datatset
apiObj = DbsUnitTestApi(api.insertProcessedDataset,f)
proc1 = DbsProcessedDataset(PrimaryDataset=pri1,
Name="StressTestProcessed" + mytime,
PhysicsGroup="BPositive",
Status="VALID",
TierList=tierList,
AlgoList=[algo1])
apiObj.run(proc1, excep = False)
apiObj = DbsUnitTestApi(api.insertBlock, f)
path = "/" + str(proc1['PrimaryDataset']['Name']) + "/" + str(proc1['Name']) + "/" + tierName1 + "-" + tierName2
print "PATH: %s" % path
#Insert Block
block1 = DbsFileBlock (Name = path+'#01234-0567', Path = path)
apiObj.run(path, block1 , excep = False)
#Insert Run
apiObj = DbsUnitTestApi(api.insertRun, f)
mytime = time.strftime("_%Y%m%d_%Hh%Mm%Ss",time.localtime())
primary = DbsPrimaryDataset (Name = "test_primary_001" + mytime, Type="test")
algo = DbsAlgorithm (
ExecutableName="TestExe01" + mytime,
ApplicationVersion= "TestVersion01" + mytime,
ApplicationFamily="AppFamily01" + mytime,
ParameterSetID=DbsQueryableParameterSet(
Hash="001234565798685",
)
)
proc = DbsProcessedDataset (
PrimaryDataset=primary,
Name="TestProcessedDS001" + mytime,
PhysicsGroup="BPositive",
Status="Valid",
TierList=['SIM', 'GEN'],
AlgoList=[algo],
)
procChild = DbsProcessedDataset (
PrimaryDataset=primary,
Name="TestProcessedDS002" + mytime,
PhysicsGroup="BPositive",
Status="Valid",
TierList=['SIM', 'GEN'],
AlgoList=[algo],
ParentList=['/test_primary_001' + mytime + '/TestProcessedDS001' + mytime +'/GEN-SIM']
)
procGrandChild = DbsProcessedDataset (
#args['version']='v00_00_05'
#args['level']='CRITICAL'
#args['level']='ERROR'
#api = DbsApi(args)
algo = DbsAlgorithm(ExecutableName="TestExe01",
ApplicationVersion="TestVersion01",
ApplicationFamily="AppFamily01",
ParameterSetID=DbsQueryableParameterSet(
Hash="001234565798685", ))
primary = DbsPrimaryDataset(Name="test_primary_001")
proc = DbsProcessedDataset(
PrimaryDataset=primary,
Name="TestProcessedDS001",
#Name="TestProcessedDSWithADSParent",
PhysicsGroup="BPositive",
Status="Valid",
TierList=['SIM', 'GEN'],
AlgoList=[algo],
)
lumi1 = DbsLumiSection(
LumiSectionNumber=1222,
StartEventNumber=100,
EndEventNumber=200,
LumiStartTime=1234,
LumiEndTime=1234,
RunNumber=1,
)
lumi2 = DbsLumiSection(
LumiSectionNumber=1333,
storage_element_list=[seName])
blockRef = dbsApi.listBlocks(dataset=datasetPath,
block_name=newBlockName)[0]
print blockRef
newFiles = []
for newFileLFN in badFiles[newBlockName]:
localFile = DBSBufferFile(lfn=newFileLFN)
localFile.load(parentage=1)
(primaryDS, procDS, tier) = datasetPath[1:].split("/", 3)
primary = DbsPrimaryDataset(Name=primaryDS, Type="mc")
algo = DbsAlgorithm(ExecutableName=localFile["appName"],
ApplicationVersion=localFile["appVer"],
ApplicationFamily=localFile["appFam"],
ParameterSetID=psetInstance)
processed = DbsProcessedDataset(PrimaryDataset=primary,
AlgoList=[algo],
Name=procDS,
TierList=[tier],
ParentList=[],
PhysicsGroup="NoGroup",
Status="VALID",
GlobalTag="")
newFiles.append(
DBSInterface.createDBSFileFromBufferFile(localFile, processed))
dbsApi.insertFiles(datasetPath, newFiles, blockRef)
dbsApi.closeBlock(block=newBlockName)
path[-1] = None
parent = ""
for i in path:
if i:
parent += "/" + i
# Create Processed dataset object to be published on Savannah and recorded on DBS
dataset = DbsProcessedDataset(
PrimaryDataset=primary,
Name=details[1],
PhysicsGroup="CMG",
Status="INVALID",
TierList=tiers,
AlgoList=[],
RunList=[],
PathList=[
dirOrFile,
],
ParentList=[],
CreatedBy=user,
DateCreated=datetime.datetime.now().strftime("%s"),
)
if parent != None: dataset['ParentList'] = [
parent,
]
try:
# Create DBLogger object to interact with Castor, DBS, and CMGDB
log = DBLogger.DBLogger(dirOrFile, targetTgz, None)
ApplicationFamily="AppFamily01",
ParameterSetID=DbsQueryableParameterSet(
Hash="001234565798685",
Name="MyFirstParam01",
Version="V001",
Type="test",
Annotation="This is test",
Content="int a= {}, b={c=1, d=33}, f={}, x, y, x"))
api.insertAlgorithm(algo)
# Parent Dataset
parent_procds = DbsProcessedDataset(
PrimaryDataset=primary,
Name="TestProcessedDS001-Parent",
PhysicsGroup="BPositive",
Status="VALID",
TierList=['GEN', 'SIM'],
AlgoList=[algo],
)
api.insertProcessedDataset(parent_procds)
# Lets say child dataset will have two runs
api.insertRun(
DbsRun(
RunNumber=1,
NumberOfEvents=100,
NumberOfLumiSections=10,
TotalLuminosity=1111,
StoreNumber=1234,
StartOfRun='now',
runObj = DbsRun(
RunNumber=runNumber,
NumberOfEvents=runNumEvents,
NumberOfLumiSections=numLumi,
TotalLuminosity=totalLumi,
StoreNumber=storeNum,
StartOfRun=startRun,
EndOfRun=endRun,
)
procObj1 = DbsProcessedDataset(PrimaryDataset=primObj,
Name=procName1,
AcquisitionEra=era,
GlobalTag=tag,
PhysicsGroup=phyGrp,
Status=procStatus,
TierList=[tier1, tier2],
AlgoList=[algoObj1, algoObj2],
XtCrossSection=1.1)
procObj2 = DbsProcessedDataset(PrimaryDataset=primObj,
Name=procName2,
AcquisitionEra=era,
GlobalTag=tag,
PhysicsGroup=phyGrp,
Status=procStatus,
TierList=[tier1, tier2],
AlgoList=[algoObj1, algoObj2],
ParentList=[path1],
RunsList=[runNumber],
from DBSAPI.dbsException import *
from DBSAPI.dbsApiException import *
from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset
from DBSAPI.dbsFileBlock import DbsFileBlock
from DBSAPI.dbsProcessedDataset import DbsProcessedDataset
from DBSAPI.dbsOptions import DbsOptionParser
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
primary = DbsPrimaryDataset(Name="test_primary_001a")
proc = DbsProcessedDataset(PrimaryDataset=primary,
TierList=['GEN', 'SIM'],
Path='/test_primary_001/TestProcessedDS001/GEN-SIM'
#Name="TestProcessedDSWithADSParent",
)
block = DbsFileBlock(
#Name="/test_primary_001/TestProcessedDSWithADSParent/GEN-SIM#12345"
Name="/test_primary_001/TestProcessedDS001/GEN-SIM#12345")
print "Creating block %s" % block
try:
# ALL Valid Options below
#print api.insertBlock (proc)
#print api.insertBlock (proc, block)
#print api.insertBlock ("/test_primary_001/TestProcessedDS001/GEN-SIM", "/test_primary_001/TestProcessedDS001/GEN-SIM#123456")
print api.insertBlock(
ApplicationFamily="AppFamily01",
ParameterSetID=DbsQueryableParameterSet(
Hash="001234565798685",
Name="MyFirstParam01",
Version="V001",
Type="test",
Annotation="This is test",
Content="int a= {}, b={c=1, d=33}, f={}, x, y, x"))
#primary = DbsPrimaryDataset (Name = "TestPrimary1164750596.79")
primary = DbsPrimaryDataset(Name="test_primary_001")
proc = DbsProcessedDataset(
PrimaryDataset=primary,
Name="TestProcessedDS001",
PhysicsGroup="BPositive",
Status="VALID",
TierList=['GEN', 'SIM'],
AlgoList=[algo],
RunsList=[1], # Provide a Run Number List that goes with this ProcDS
#ADSParent="/TestPrimary_001_20080320_13h37m20s/TestProcessed_20080320_13h37m20s/GEN-SIM/TestAnalysisDSDef_005_20080320_13h37m20s"
XtCrossSection=1.1)
print "Creating a processed dataset %s" % proc
try:
api.insertProcessedDataset(proc)
print "Result: %s" % proc
except DbsApiException, ex:
print "Caught API Exception %s: %s " % (ex.getClassName(),
ex.getErrorMessage())
if ex.getErrorCode() not in (None, ""):
algo = DbsAlgorithm(ExecutableName="TestExe01",
ApplicationVersion="TestVersion01",
ApplicationFamily="AppFamily01",
ParameterSetID=DbsQueryableParameterSet(
Hash="001234565798685",
Name="MyFirstParam01",
Version="V001",
Type="test",
Annotation="This is test",
Content="int a= {}, b={c=1, d=33}, f={}, x, y, x"))
primary = DbsPrimaryDataset(Name="test_primary_001")
proc = DbsProcessedDataset(
PrimaryDataset=primary,
Name="TestProcessedDS003",
PhysicsGroup="BPositive",
Status="Valid",
TierList=['SIM', 'RECO'],
AlgoList=[algo],
)
lumi1 = DbsLumiSection(
LumiSectionNumber=1222,
StartEventNumber=100,
EndEventNumber=200,
LumiStartTime='notime',
LumiEndTime='neverending',
RunNumber=1,
)
lumi2 = DbsLumiSection(
LumiSectionNumber=1333,
StartEventNumber=100,
