def dbsPublish(self, procds): # Get file information fileOps = FileOps(getCastor(procds['PathList'][0]), getDbsUser(procds['PathList'][0])) # Check if user has dataset files, and DO NOT allow publish if they do not if len(fileOps.getRootFiles()) == 0: print "No dataset found on Castor, exiting" sys.exit(1) return None # Check dbsAPI is valid if self._dbsAPI is None: print "DBS not connected" return None details = procds["PathList"][0].lstrip("/").split("/") # Check that primary dataset exists if not self._dbsAPI.listPrimaryDatasets(details[0]): self._dbsAPI.insertPrimaryDataset(DbsPrimaryDataset(Name=details[0],Type="test")) if self._dbsAPI.listPrimaryDatasets(details[0]): parent = None if len(procds["ParentList"]) > 0: parent = procds["ParentList"][0].lstrip("/").split("/") if parent is not None: parentDatasets = self._dbsAPI.listProcessedDatasets(parent[0],parent[2],parent[1]) # If only one possible parent exists, use it as parent if len(parentDatasets) == 1: procds["ParentList"][0] = parentDatasets[0]['PathList'][0] if len(parentDatasets) == 0: #if the parent is if not re.search("---",parent[1]): self.dbsPublish(CastorToDbsFormatter(procds["ParentList"][0]).getDataset(), None) else: print "Parent dataset missing, please add parent before proceeding" return None else: "Please use specific parent name, not REGEX" return None # Check if dataset already exists dbsOutput = self._dbsAPI.listProcessedDatasets(details[0],details[2],details[1]) # Check if self._procds doesn't exist if len(dbsOutput) == 0: print "Adding Dataset to DBS" # Add Dataset self._dbsAPI.insertProcessedDataset(procds) block = DbsFileBlock (Name=procds['PathList'][0]+"#1") # Add the corresponding file block self._dbsAPI.insertBlock(procds['PathList'][0], block) else: print "Dataset already on DBS" return self._getDbsIDs(procds['PathList'][0]) else: print "Primary dataset does not exist" return None
def createPrimaryDataset(primaryName, primaryDatasetType='mc', apiRef=None): """ _createPrimaryDataset_ """ logging.debug("Inserting PrimaryDataset %s with Type %s" \ % (primaryName, primaryDatasetType)) primary = DbsPrimaryDataset(Name=primaryName, Type=primaryDatasetType) if apiRef: try: apiRef.insertPrimaryDataset(primary) except DbsException, ex: msg = "Error in DBSInterface.createPrimaryDataset(%s)\n" % primaryName msg += formatEx(ex) logging.error(msg) raise DBSInterfaceError(msg)
def createPrimaryDataset(datasetInfo, apiRef=None): """ _createPrimaryDataset_ Create and return a Primary Dataset object. If apiRef is not None, it is used to insert the dataset into the DBS """ if datasetInfo.has_key('PrimaryDatasetType'): PrimaryDatasetType = datasetInfo['PrimaryDatasetType'] else: PrimaryDatasetType = 'mc' logging.debug("Inserting PrimaryDataset %s with Type %s" % (datasetInfo["PrimaryDataset"], PrimaryDatasetType)) primary = DbsPrimaryDataset(Name=datasetInfo["PrimaryDataset"], Type=PrimaryDatasetType) if apiRef != None: apiRef.insertPrimaryDataset(primary) return primary
def __init__(self, DSName, user): parent = getUnknownParentWithDbs(getDbs(DSName)) path = getDbsWithUser(DSName, user) parts = path.lstrip("/").split("/") primary = DbsPrimaryDataset(Name=parts[0]) tierList = parts[2].split("-") self.procds = DbsProcessedDataset( PrimaryDataset=primary, Name=parts[1], PhysicsGroup="CMG", Status="VALID", TierList=tierList, PathList=[ path, ], DateCreated=datetime.now().strftime("%s")) if parent is not None: self.procds['ParentList'] = [ parent, ] if user is not None: self.procds['CreatedBy'] = user
serverInfo = api.getServerInfo() isMYSQL = serverInfo['InstanceType'] isGlobal = serverInfo['InstanceName'] #mytime = str(time.time()) mytime = time.strftime("_%Y%m%d_%Hh%Mm%Ss",time.localtime()) f = open("result.txt", "a+") apiObj = DbsUnitTestApi(api.insertPrimaryDataset, f) apiObj.setVerboseLevel(opts.verbose) f.write("\n\n***********************insertPrimaryDataset API tests***************************") primary = 'TestPrimary_001' + mytime pri1 = DbsPrimaryDataset (Name = primary, Type="test") apiObj.run(pri1, excep = False) primary = 'TestPrimary_002' + mytime pri2 = DbsPrimaryDataset (Name = primary, Type="test") apiObj.run(pri2, excep = False) pri = DbsPrimaryDataset () apiObj.run(pri, excep = True) pri = DbsPrimaryDataset (Name = "Test Het", Type="test") apiObj.run(pri, excep = True) pri = DbsPrimaryDataset (Name = "Test;Het", Type="test") apiObj.run(pri, excep = True)
filename="bulkDataResult."+logext #print "FILENAME: "+filename f = open(filename, "w") ################# WRITING TO STDOUT COMMENT THIS LINE (UNCOMMENT ABOVE) TO WRITE TO A FILE ########### #f=sys.stdout for i in range(maxDS): # Make this cycle unique mytime = os.popen('uuidgen').readline().strip() mytime += str(i) fileList = [] #Insert Primary apiObj = DbsUnitTestApi(api.insertPrimaryDataset, f) primary = 'StressTestPrimary' + mytime pri1 = DbsPrimaryDataset (Name = primary, Type='MC') apiObj.run(pri1, excep = False) #Insert Algorithm apiObj = DbsUnitTestApi(api.insertAlgorithm,f) algo1 = DbsAlgorithm (ExecutableName="StressTestExe01", ApplicationVersion= "StressTestVersion01" + mytime, ApplicationFamily="StressTestAppFamily01", ParameterSetID=DbsQueryableParameterSet(Hash="001234565798685", Name="StressTestParam01", Version="V001", Type="test", Annotation="This is a stress test param", Content="int a= {}, b={c=1, d=33}, f={}, x, y, x" ) )
from DBSAPI.dbsFileBlock import DbsFileBlock from DBSAPI.dbsRun import DbsRun from DBSAPI.dbsFile import DbsFile from DBSAPI.dbsLumiSection import DbsLumiSection from DBSAPI.dbsQueryableParameterSet import DbsQueryableParameterSet from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset from DBSAPI.dbsProcessedDataset import DbsProcessedDataset from DBSAPI.dbsOptions import DbsOptionParser optManager = DbsOptionParser() (opts,args) = optManager.getOpt() api = DbsApi(opts.__dict__) mytime = time.strftime("_%Y%m%d_%Hh%Mm%Ss",time.localtime()) primary = DbsPrimaryDataset (Name = "test_primary_001" + mytime, Type="test") algo = DbsAlgorithm ( ExecutableName="TestExe01" + mytime, ApplicationVersion= "TestVersion01" + mytime, ApplicationFamily="AppFamily01" + mytime, ParameterSetID=DbsQueryableParameterSet( Hash="001234565798685", ) ) proc = DbsProcessedDataset ( PrimaryDataset=primary, Name="TestProcessedDS001" + mytime, PhysicsGroup="BPositive", Status="Valid",
(opts, args) = optManager.getOpt() api = DbsApi(opts.__dict__) #args={} #args['url']='http://cmssrv17.fnal.gov:8989/DBS/servlet/DBSServlet' #args['version']='v00_00_05' #args['level']='CRITICAL' #args['level']='ERROR' #api = DbsApi(args) algo = DbsAlgorithm(ExecutableName="TestExe01", ApplicationVersion="TestVersion01", ApplicationFamily="AppFamily01", ParameterSetID=DbsQueryableParameterSet( Hash="001234565798685", )) primary = DbsPrimaryDataset(Name="test_primary_001") proc = DbsProcessedDataset( PrimaryDataset=primary, Name="TestProcessedDS001", #Name="TestProcessedDSWithADSParent", PhysicsGroup="BPositive", Status="Valid", TierList=['SIM', 'GEN'], AlgoList=[algo], ) lumi1 = DbsLumiSection( LumiSectionNumber=1222, StartEventNumber=100, EndEventNumber=200, LumiStartTime=1234,
from DBSAPI.dbsApiException import * from DBSAPI.dbsAlgorithm import DbsAlgorithm from DBSAPI.dbsFileBlock import DbsFileBlock from DBSAPI.dbsRun import DbsRun from DBSAPI.dbsFile import DbsFile from DBSAPI.dbsLumiSection import DbsLumiSection from DBSAPI.dbsQueryableParameterSet import DbsQueryableParameterSet from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset from DBSAPI.dbsProcessedDataset import DbsProcessedDataset from DBSAPI.dbsOptions import DbsOptionParser optManager = DbsOptionParser() (opts, args) = optManager.getOpt() api = DbsApi(opts.__dict__) primary = DbsPrimaryDataset(Name="test_primary_0011", Type="test") algo = DbsAlgorithm(ExecutableName="TestExe01", ApplicationVersion="TestVersion01", ApplicationFamily="AppFamily01", ParameterSetID=DbsQueryableParameterSet( Hash="001234565798685", )) proc = DbsProcessedDataset( PrimaryDataset=primary, Name="TestProcessedDS001", PhysicsGroup="BPositive", Status="Valid", TierList=['SIM', 'GEN'], AlgoList=[algo], )
(datasetPath, junk) = newBlockName.split("#", 1) dbsApi.insertBlock(datasetPath, newBlockName, storage_element_list=[seName]) blockRef = dbsApi.listBlocks(dataset=datasetPath, block_name=newBlockName)[0] print blockRef newFiles = [] for newFileLFN in badFiles[newBlockName]: localFile = DBSBufferFile(lfn=newFileLFN) localFile.load(parentage=1) (primaryDS, procDS, tier) = datasetPath[1:].split("/", 3) primary = DbsPrimaryDataset(Name=primaryDS, Type="mc") algo = DbsAlgorithm(ExecutableName=localFile["appName"], ApplicationVersion=localFile["appVer"], ApplicationFamily=localFile["appFam"], ParameterSetID=psetInstance) processed = DbsProcessedDataset(PrimaryDataset=primary, AlgoList=[algo], Name=procDS, TierList=[tier], ParentList=[], PhysicsGroup="NoGroup", Status="VALID", GlobalTag="") newFiles.append( DBSInterface.createDBSFileFromBufferFile(localFile, processed))
targetDir = baseDir + "/" + args[0].lstrip("/").rstrip("/") targetTgz = targetDir + "/Logger.tgz" # Get array containing 1) Primary dataset name; 2) Dataset Name; 3+) Tiers details = dirOrFile.lstrip("/").split("/") # If element is not Primary dataset of dataset name, add to the Tiers array count = 0 tiers = [] for i in details: if count > 1: tiers.append(i) count += 1 # Create Primary dataset object (will reference complete object on DBS) primary = DbsPrimaryDataset(Name=details[0]) path = dirOrFile.lstrip("/").split("/") parent = None if len(path) > 3: path[-1] = None parent = "" for i in path: if i: parent += "/" + i # Create Processed dataset object to be published on Savannah and recorded on DBS dataset = DbsProcessedDataset( PrimaryDataset=primary, Name=details[1], PhysicsGroup="CMG",
from DBSAPI.dbsOptions import DbsOptionParser from DBSAPI.dbsAlgorithm import DbsAlgorithm from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset from DBSAPI.dbsRun import DbsRun from DBSAPI.dbsQueryableParameterSet import DbsQueryableParameterSet from DBSAPI.dbsProcessedDataset import DbsProcessedDataset try: optManager = DbsOptionParser() (opts, args) = optManager.getOpt() api = DbsApi(opts.__dict__) import pdb ##Primary Dataset primary = DbsPrimaryDataset(Name="test_primary_001", Type="TEST") api.insertPrimaryDataset(primary) #Algorithm used by Parent and Child Datasets for our test algo = DbsAlgorithm(ExecutableName="TestExe01", ApplicationVersion="TestVersion01", ApplicationFamily="AppFamily01", ParameterSetID=DbsQueryableParameterSet( Hash="001234565798685", Name="MyFirstParam01", Version="V001", Type="test", Annotation="This is test", Content="int a= {}, b={c=1, d=33}, f={}, x, y, x")) api.insertAlgorithm(algo)
fileNameG = 'test_file_name_G_' + ran fileNumEventsG = 234567 fileCkecksumG = 'test_cksum_G_' + ran fileAdler32G = 'test_adler32_G_' + ran fileMd5G = 'test_md5_G_' + ran fileSizeG = 563226500 fileStatusG = validStatus fileValidStatusG = validStatus fileTypeG = 'STREAMER' qim_name1 = "Tracker_Global" qim_name2 = "TIB_Local" qim_name3 = "TIB_Power" qim_int = "TIB_Percentage" primObj = DbsPrimaryDataset(Name=primName, Type=primType) algoObj1 = DbsAlgorithm(ExecutableName=algoExe1, ApplicationVersion=algoVer1, ApplicationFamily=algoFam1, ParameterSetID=DbsQueryableParameterSet( Hash=psHash1, Name=psName1, Version=psVer1, Type=psType1, Annotation=psAnno1, Content=psCon1)) algoObj2 = DbsAlgorithm(ExecutableName=algoExe2, ApplicationVersion=algoVer2, ApplicationFamily=algoFam2, ParameterSetID=DbsQueryableParameterSet( Hash=psHash2,
filename="bulkDataResult."+logext #print "FILENAME: "+filename f = open(filename, "w") ################# WRITING TO STDOUT COMMENT THIS LINE (UNCOMMENT ABOVE) TO WRITE TO A FILE ########### #f=sys.stdout for i in range(maxDS): # Make this cycle unique mytime = os.popen('uuidgen').readline().strip() mytime += str(i) fileList = [] #Insert Primary apiObj = DbsUnitTestApi(api.insertPrimaryDataset, f) primary = 'StressTestPrimary' + mytime pri1 = DbsPrimaryDataset (Name = primary) apiObj.run(pri1, excep = False) #Insert Algorithm apiObj = DbsUnitTestApi(api.insertAlgorithm,f) algo1 = DbsAlgorithm (ExecutableName="StressTestExe01", ApplicationVersion= "StressTestVersion01" + mytime, ApplicationFamily="StressTestAppFamily01", ParameterSetID=DbsQueryableParameterSet(Hash="001234565798685", Name="StressTestParam01", Version="V001", Type="test", Annotation="This is a stress test param", Content="int a= {}, b={c=1, d=33}, f={}, x, y, x" ) )
optManager = DbsOptionParser() (opts,args) = optManager.getOpt() api = DbsApi(opts.__dict__) mytime = str(time.time()) maxDS = 1 maxFiles = 5000 f = open("bulkDataResult.txt", "w") fileList = [] for i in range(maxDS): mytime = str(time.time()) #Insert Primary apiObj = DbsUnitTestApi(api.insertPrimaryDataset, f) primary = 'TestPrimary' + mytime pri1 = DbsPrimaryDataset (Name = primary, Type='TEST') apiObj.run(pri1, excep = False) #Insert Algorithm apiObj = DbsUnitTestApi(api.insertAlgorithm,f) algo1 = DbsAlgorithm (ExecutableName="TestExe01", ApplicationVersion= "TestVersion01" + mytime, ApplicationFamily="AppFamily01", ParameterSetID=DbsQueryableParameterSet(Hash="001234565798685", Name="MyFirstParam01", Version="V001", Type="test", Annotation="This is test", Content="int a= {}, b={c=1, d=33}, f={}, x, y, x" ) )
optManager = DbsOptionParser() (opts, args) = optManager.getOpt() api = DbsApi(opts.__dict__) mytime = str(time.time()) maxDS = 1 maxFiles = 10000 f = open("bulkDataResult.txt", "w") for i in range(maxDS): mytime = str(time.time()) #Insert Primary apiObj = DbsUnitTestApi(api.insertPrimaryDataset, f) primary = 'TestPrimary' + mytime pri1 = DbsPrimaryDataset(Name=primary, Type="TEST") apiObj.run(pri1, excep=False) #Insert Algorithm apiObj = DbsUnitTestApi(api.insertAlgorithm, f) algo1 = DbsAlgorithm( ExecutableName="TestExe01", ApplicationVersion="TestVersion01" + mytime, ApplicationFamily="AppFamily01", ParameterSetID=DbsQueryableParameterSet( Hash="001234565798685", Name="MyFirstParam01", Version="V001", Type="test", Annotation="This is test", Content="int a= {}, b={c=1, d=33}, f={}, x, y, x"))