def dbsPublish(self, procds):
# Get file information
fileOps = FileOps(getCastor(procds['PathList'][0]), getDbsUser(procds['PathList'][0]))
# Check if user has dataset files, and DO NOT allow publish if they do not
if len(fileOps.getRootFiles()) == 0:
print "No dataset found on Castor, exiting"
sys.exit(1)
return None
# Check dbsAPI is valid
if self._dbsAPI is None:
print "DBS not connected"
return None
details = procds["PathList"][0].lstrip("/").split("/")
# Check that primary dataset exists
if not self._dbsAPI.listPrimaryDatasets(details[0]):
self._dbsAPI.insertPrimaryDataset(DbsPrimaryDataset(Name=details[0],Type="test"))
if self._dbsAPI.listPrimaryDatasets(details[0]):
parent = None
if len(procds["ParentList"]) > 0:
parent = procds["ParentList"][0].lstrip("/").split("/")
if parent is not None:
parentDatasets = self._dbsAPI.listProcessedDatasets(parent[0],parent[2],parent[1])
# If only one possible parent exists, use it as parent
if len(parentDatasets) == 1:
procds["ParentList"][0] = parentDatasets[0]['PathList'][0]
if len(parentDatasets) == 0:
#if the parent is
if not re.search("---",parent[1]):
self.dbsPublish(CastorToDbsFormatter(procds["ParentList"][0]).getDataset(), None)
else:
print "Parent dataset missing, please add parent before proceeding"
return None
else:
"Please use specific parent name, not REGEX"
return None
# Check if dataset already exists
dbsOutput = self._dbsAPI.listProcessedDatasets(details[0],details[2],details[1])
# Check if self._procds doesn't exist
if len(dbsOutput) == 0:
print "Adding Dataset to DBS"
# Add Dataset
self._dbsAPI.insertProcessedDataset(procds)
block = DbsFileBlock (Name=procds['PathList'][0]+"#1")
# Add the corresponding file block
self._dbsAPI.insertBlock(procds['PathList'][0], block)
else:
print "Dataset already on DBS"
return self._getDbsIDs(procds['PathList'][0])
else:
print "Primary dataset does not exist"
return None
def createPrimaryDataset(primaryName, primaryDatasetType='mc', apiRef=None):
"""
_createPrimaryDataset_
"""
logging.debug("Inserting PrimaryDataset %s with Type %s" \
% (primaryName, primaryDatasetType))
primary = DbsPrimaryDataset(Name=primaryName, Type=primaryDatasetType)
if apiRef:
try:
apiRef.insertPrimaryDataset(primary)
except DbsException, ex:
msg = "Error in DBSInterface.createPrimaryDataset(%s)\n" % primaryName
msg += formatEx(ex)
logging.error(msg)
raise DBSInterfaceError(msg)
def createPrimaryDataset(datasetInfo, apiRef=None):
"""
_createPrimaryDataset_
Create and return a Primary Dataset object.
If apiRef is not None, it is used to insert the dataset into the
DBS
"""
if datasetInfo.has_key('PrimaryDatasetType'):
PrimaryDatasetType = datasetInfo['PrimaryDatasetType']
else:
PrimaryDatasetType = 'mc'
logging.debug("Inserting PrimaryDataset %s with Type %s" %
(datasetInfo["PrimaryDataset"], PrimaryDatasetType))
primary = DbsPrimaryDataset(Name=datasetInfo["PrimaryDataset"],
Type=PrimaryDatasetType)
if apiRef != None:
apiRef.insertPrimaryDataset(primary)
return primary
def __init__(self, DSName, user):
parent = getUnknownParentWithDbs(getDbs(DSName))
path = getDbsWithUser(DSName, user)
parts = path.lstrip("/").split("/")
primary = DbsPrimaryDataset(Name=parts[0])
tierList = parts[2].split("-")
self.procds = DbsProcessedDataset(
PrimaryDataset=primary,
Name=parts[1],
PhysicsGroup="CMG",
Status="VALID",
TierList=tierList,
PathList=[
path,
],
DateCreated=datetime.now().strftime("%s"))
if parent is not None:
self.procds['ParentList'] = [
parent,
]
if user is not None:
self.procds['CreatedBy'] = user
serverInfo = api.getServerInfo()
isMYSQL = serverInfo['InstanceType']
isGlobal = serverInfo['InstanceName']
#mytime = str(time.time())
mytime = time.strftime("_%Y%m%d_%Hh%Mm%Ss",time.localtime())
f = open("result.txt", "a+")
apiObj = DbsUnitTestApi(api.insertPrimaryDataset, f)
apiObj.setVerboseLevel(opts.verbose)
f.write("\n\n***********************insertPrimaryDataset API tests***************************")
primary = 'TestPrimary_001' + mytime
pri1 = DbsPrimaryDataset (Name = primary, Type="test")
apiObj.run(pri1, excep = False)
primary = 'TestPrimary_002' + mytime
pri2 = DbsPrimaryDataset (Name = primary, Type="test")
apiObj.run(pri2, excep = False)
pri = DbsPrimaryDataset ()
apiObj.run(pri, excep = True)
pri = DbsPrimaryDataset (Name = "Test Het", Type="test")
apiObj.run(pri, excep = True)
pri = DbsPrimaryDataset (Name = "Test;Het", Type="test")
apiObj.run(pri, excep = True)
filename="bulkDataResult."+logext
#print "FILENAME: "+filename
f = open(filename, "w")
################# WRITING TO STDOUT COMMENT THIS LINE (UNCOMMENT ABOVE) TO WRITE TO A FILE ###########
#f=sys.stdout
for i in range(maxDS):
# Make this cycle unique
mytime = os.popen('uuidgen').readline().strip()
mytime += str(i)
fileList = []
#Insert Primary
apiObj = DbsUnitTestApi(api.insertPrimaryDataset, f)
primary = 'StressTestPrimary' + mytime
pri1 = DbsPrimaryDataset (Name = primary, Type='MC')
apiObj.run(pri1, excep = False)
#Insert Algorithm
apiObj = DbsUnitTestApi(api.insertAlgorithm,f)
algo1 = DbsAlgorithm (ExecutableName="StressTestExe01",
ApplicationVersion= "StressTestVersion01" + mytime,
ApplicationFamily="StressTestAppFamily01",
ParameterSetID=DbsQueryableParameterSet(Hash="001234565798685",
Name="StressTestParam01",
Version="V001",
Type="test",
Annotation="This is a stress test param",
Content="int a= {}, b={c=1, d=33}, f={}, x, y, x"
)
)
from DBSAPI.dbsFileBlock import DbsFileBlock
from DBSAPI.dbsRun import DbsRun
from DBSAPI.dbsFile import DbsFile
from DBSAPI.dbsLumiSection import DbsLumiSection
from DBSAPI.dbsQueryableParameterSet import DbsQueryableParameterSet
from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset
from DBSAPI.dbsProcessedDataset import DbsProcessedDataset
from DBSAPI.dbsOptions import DbsOptionParser
optManager = DbsOptionParser()
(opts,args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
mytime = time.strftime("_%Y%m%d_%Hh%Mm%Ss",time.localtime())
primary = DbsPrimaryDataset (Name = "test_primary_001" + mytime, Type="test")
algo = DbsAlgorithm (
ExecutableName="TestExe01" + mytime,
ApplicationVersion= "TestVersion01" + mytime,
ApplicationFamily="AppFamily01" + mytime,
ParameterSetID=DbsQueryableParameterSet(
Hash="001234565798685",
)
)
proc = DbsProcessedDataset (
PrimaryDataset=primary,
Name="TestProcessedDS001" + mytime,
PhysicsGroup="BPositive",
Status="Valid",
(opts, args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
#args={}
#args['url']='http://cmssrv17.fnal.gov:8989/DBS/servlet/DBSServlet'
#args['version']='v00_00_05'
#args['level']='CRITICAL'
#args['level']='ERROR'
#api = DbsApi(args)
algo = DbsAlgorithm(ExecutableName="TestExe01",
ApplicationVersion="TestVersion01",
ApplicationFamily="AppFamily01",
ParameterSetID=DbsQueryableParameterSet(
Hash="001234565798685", ))
primary = DbsPrimaryDataset(Name="test_primary_001")
proc = DbsProcessedDataset(
PrimaryDataset=primary,
Name="TestProcessedDS001",
#Name="TestProcessedDSWithADSParent",
PhysicsGroup="BPositive",
Status="Valid",
TierList=['SIM', 'GEN'],
AlgoList=[algo],
)
lumi1 = DbsLumiSection(
LumiSectionNumber=1222,
StartEventNumber=100,
EndEventNumber=200,
LumiStartTime=1234,
from DBSAPI.dbsApiException import *
from DBSAPI.dbsAlgorithm import DbsAlgorithm
from DBSAPI.dbsFileBlock import DbsFileBlock
from DBSAPI.dbsRun import DbsRun
from DBSAPI.dbsFile import DbsFile
from DBSAPI.dbsLumiSection import DbsLumiSection
from DBSAPI.dbsQueryableParameterSet import DbsQueryableParameterSet
from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset
from DBSAPI.dbsProcessedDataset import DbsProcessedDataset
from DBSAPI.dbsOptions import DbsOptionParser
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
primary = DbsPrimaryDataset(Name="test_primary_0011", Type="test")
algo = DbsAlgorithm(ExecutableName="TestExe01",
ApplicationVersion="TestVersion01",
ApplicationFamily="AppFamily01",
ParameterSetID=DbsQueryableParameterSet(
Hash="001234565798685", ))
proc = DbsProcessedDataset(
PrimaryDataset=primary,
Name="TestProcessedDS001",
PhysicsGroup="BPositive",
Status="Valid",
TierList=['SIM', 'GEN'],
AlgoList=[algo],
)
(datasetPath, junk) = newBlockName.split("#", 1)
dbsApi.insertBlock(datasetPath,
newBlockName,
storage_element_list=[seName])
blockRef = dbsApi.listBlocks(dataset=datasetPath,
block_name=newBlockName)[0]
print blockRef
newFiles = []
for newFileLFN in badFiles[newBlockName]:
localFile = DBSBufferFile(lfn=newFileLFN)
localFile.load(parentage=1)
(primaryDS, procDS, tier) = datasetPath[1:].split("/", 3)
primary = DbsPrimaryDataset(Name=primaryDS, Type="mc")
algo = DbsAlgorithm(ExecutableName=localFile["appName"],
ApplicationVersion=localFile["appVer"],
ApplicationFamily=localFile["appFam"],
ParameterSetID=psetInstance)
processed = DbsProcessedDataset(PrimaryDataset=primary,
AlgoList=[algo],
Name=procDS,
TierList=[tier],
ParentList=[],
PhysicsGroup="NoGroup",
Status="VALID",
GlobalTag="")
newFiles.append(
DBSInterface.createDBSFileFromBufferFile(localFile, processed))
targetDir = baseDir + "/" + args[0].lstrip("/").rstrip("/")
targetTgz = targetDir + "/Logger.tgz"
# Get array containing 1) Primary dataset name; 2) Dataset Name; 3+) Tiers
details = dirOrFile.lstrip("/").split("/")
# If element is not Primary dataset of dataset name, add to the Tiers array
count = 0
tiers = []
for i in details:
if count > 1:
tiers.append(i)
count += 1
# Create Primary dataset object (will reference complete object on DBS)
primary = DbsPrimaryDataset(Name=details[0])
path = dirOrFile.lstrip("/").split("/")
parent = None
if len(path) > 3:
path[-1] = None
parent = ""
for i in path:
if i:
parent += "/" + i
# Create Processed dataset object to be published on Savannah and recorded on DBS
dataset = DbsProcessedDataset(
PrimaryDataset=primary,
Name=details[1],
PhysicsGroup="CMG",
from DBSAPI.dbsOptions import DbsOptionParser
from DBSAPI.dbsAlgorithm import DbsAlgorithm
from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset
from DBSAPI.dbsRun import DbsRun
from DBSAPI.dbsQueryableParameterSet import DbsQueryableParameterSet
from DBSAPI.dbsProcessedDataset import DbsProcessedDataset
try:
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
import pdb
##Primary Dataset
primary = DbsPrimaryDataset(Name="test_primary_001", Type="TEST")
api.insertPrimaryDataset(primary)
#Algorithm used by Parent and Child Datasets for our test
algo = DbsAlgorithm(ExecutableName="TestExe01",
ApplicationVersion="TestVersion01",
ApplicationFamily="AppFamily01",
ParameterSetID=DbsQueryableParameterSet(
Hash="001234565798685",
Name="MyFirstParam01",
Version="V001",
Type="test",
Annotation="This is test",
Content="int a= {}, b={c=1, d=33}, f={}, x, y, x"))
api.insertAlgorithm(algo)
fileNameG = 'test_file_name_G_' + ran
fileNumEventsG = 234567
fileCkecksumG = 'test_cksum_G_' + ran
fileAdler32G = 'test_adler32_G_' + ran
fileMd5G = 'test_md5_G_' + ran
fileSizeG = 563226500
fileStatusG = validStatus
fileValidStatusG = validStatus
fileTypeG = 'STREAMER'
qim_name1 = "Tracker_Global"
qim_name2 = "TIB_Local"
qim_name3 = "TIB_Power"
qim_int = "TIB_Percentage"
primObj = DbsPrimaryDataset(Name=primName, Type=primType)
algoObj1 = DbsAlgorithm(ExecutableName=algoExe1,
ApplicationVersion=algoVer1,
ApplicationFamily=algoFam1,
ParameterSetID=DbsQueryableParameterSet(
Hash=psHash1,
Name=psName1,
Version=psVer1,
Type=psType1,
Annotation=psAnno1,
Content=psCon1))
algoObj2 = DbsAlgorithm(ExecutableName=algoExe2,
ApplicationVersion=algoVer2,
ApplicationFamily=algoFam2,
ParameterSetID=DbsQueryableParameterSet(
Hash=psHash2,
filename="bulkDataResult."+logext
#print "FILENAME: "+filename
f = open(filename, "w")
################# WRITING TO STDOUT COMMENT THIS LINE (UNCOMMENT ABOVE) TO WRITE TO A FILE ###########
#f=sys.stdout
for i in range(maxDS):
# Make this cycle unique
mytime = os.popen('uuidgen').readline().strip()
mytime += str(i)
fileList = []
#Insert Primary
apiObj = DbsUnitTestApi(api.insertPrimaryDataset, f)
primary = 'StressTestPrimary' + mytime
pri1 = DbsPrimaryDataset (Name = primary)
apiObj.run(pri1, excep = False)
#Insert Algorithm
apiObj = DbsUnitTestApi(api.insertAlgorithm,f)
algo1 = DbsAlgorithm (ExecutableName="StressTestExe01",
ApplicationVersion= "StressTestVersion01" + mytime,
ApplicationFamily="StressTestAppFamily01",
ParameterSetID=DbsQueryableParameterSet(Hash="001234565798685",
Name="StressTestParam01",
Version="V001",
Type="test",
Annotation="This is a stress test param",
Content="int a= {}, b={c=1, d=33}, f={}, x, y, x"
)
)
optManager = DbsOptionParser()
(opts,args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
mytime = str(time.time())
maxDS = 1
maxFiles = 5000
f = open("bulkDataResult.txt", "w")
fileList = []
for i in range(maxDS):
mytime = str(time.time())
#Insert Primary
apiObj = DbsUnitTestApi(api.insertPrimaryDataset, f)
primary = 'TestPrimary' + mytime
pri1 = DbsPrimaryDataset (Name = primary, Type='TEST')
apiObj.run(pri1, excep = False)
#Insert Algorithm
apiObj = DbsUnitTestApi(api.insertAlgorithm,f)
algo1 = DbsAlgorithm (ExecutableName="TestExe01",
ApplicationVersion= "TestVersion01" + mytime,
ApplicationFamily="AppFamily01",
ParameterSetID=DbsQueryableParameterSet(Hash="001234565798685",
Name="MyFirstParam01",
Version="V001",
Type="test",
Annotation="This is test",
Content="int a= {}, b={c=1, d=33}, f={}, x, y, x"
)
)
optManager = DbsOptionParser()
(opts, args) = optManager.getOpt()
api = DbsApi(opts.__dict__)
mytime = str(time.time())
maxDS = 1
maxFiles = 10000
f = open("bulkDataResult.txt", "w")
for i in range(maxDS):
mytime = str(time.time())
#Insert Primary
apiObj = DbsUnitTestApi(api.insertPrimaryDataset, f)
primary = 'TestPrimary' + mytime
pri1 = DbsPrimaryDataset(Name=primary, Type="TEST")
apiObj.run(pri1, excep=False)
#Insert Algorithm
apiObj = DbsUnitTestApi(api.insertAlgorithm, f)
algo1 = DbsAlgorithm(
ExecutableName="TestExe01",
ApplicationVersion="TestVersion01" + mytime,
ApplicationFamily="AppFamily01",
ParameterSetID=DbsQueryableParameterSet(
Hash="001234565798685",
Name="MyFirstParam01",
Version="V001",
Type="test",
Annotation="This is test",
Content="int a= {}, b={c=1, d=33}, f={}, x, y, x"))