def testDropCount(self):
"""
_testDropCount_
Verify that dropping a fileset and counting the files in a fileset works
correctly.
"""
testCollectionA = CouchCollection(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="Thunderstruck")
testCollectionB = CouchCollection(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="StruckThunder")
testFiles = []
for i in range(5):
testFile = File(lfn=makeUUID(), size=random.randint(1024, 4096),
events=random.randint(1024, 4096))
testFiles.append(testFile)
testFilesetA = CouchFileset(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="TestFilesetA")
testFilesetB = CouchFileset(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="TestFilesetB")
testFilesetC = CouchFileset(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="TestFilesetC")
testCollectionA.addFileset(testFilesetA)
testCollectionB.addFileset(testFilesetB)
testCollectionB.addFileset(testFilesetC)
testFilesetA.add(testFiles)
testFilesetB.add(testFiles)
testFilesetC.add(testFiles)
testFilesetC.drop()
testCollectionC = CouchCollection(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="StruckThunder")
testCollectionC.populate()
self.assertEqual(len(testCollectionC["filesets"]), 1,
"Error: There should be one fileset in this collection.")
self.assertEqual(testCollectionC["filesets"][0].fileCount(), 5,
"Error: Wrong number of files in fileset.")
testCollectionD = CouchCollection(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="Thunderstruck")
testCollectionD.populate()
self.assertEqual(len(testCollectionD["filesets"]), 1,
"Error: There should be one fileset in this collection.")
self.assertEqual(testCollectionD["filesets"][0].fileCount(), 5,
"Error: Wrong number of files in fileset.")
return
def getDataCollection(self, collName):
"""
_getDataCollection_
Get a data collection by name
"""
coll = CouchCollection(name=collName, database=self.database, url=self.url)
coll.populate()
return coll
def getDataCollection(self, collName):
"""
_getDataCollection_
Get a data collection by name
"""
coll = CouchCollection(name=collName, database=self.database, url=self.url)
coll.populate()
return coll
def getDataCollection(self, collName, user = "******",
group = "cmsdataops"):
"""
_getDataCollection_
Get a data collection by name
"""
coll = CouchCollection(name = collName, database = self.database,
url = self.url)
coll.owner = self.newOwner(group, user)
coll.populate()
return coll
def getDataCollection(self,
collName,
user="******",
group="cmsdataops"):
"""
_getDataCollection_
Get a data collection by name
"""
coll = CouchCollection(name=collName,
database=self.database,
url=self.url)
coll.owner = self.newOwner(group, user)
coll.populate()
return coll
def listCollections(self, owner):
"""
_listCollections_
List the collections belonging to an owner.
"""
params = {"startkey": [owner.group.name, owner.name],
"endkey": [owner.group.name, owner.name, {}],
"reduce": True, "group_level": 3}
result = self.couchdb.loadView("ACDC", "owner_coll_fileset_docs",
params)
for row in result["rows"]:
coll = CouchCollection(name = row["key"][2],
database = self.database, url = self.url)
coll.setOwner(owner)
coll.populate()
yield coll
def listCollections(self, owner):
"""
_listCollections_
List the collections belonging to an owner.
"""
params = {
"startkey": [owner.group.name, owner.name],
"endkey": [owner.group.name, owner.name, {}],
"reduce": True,
"group_level": 3
}
result = self.couchdb.loadView("ACDC", "owner_coll_fileset_docs",
params)
for row in result["rows"]:
coll = CouchCollection(name=row["key"][2],
database=self.database,
url=self.url)
coll.setOwner(owner)
coll.populate()
yield coll
def testCreatePopulateDrop(self):
"""
_testCreatePopulateDrop_
Test creating, populating and dropping a collection.
"""
testCollectionA = CouchCollection(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="Thunderstruck")
testCollectionB = CouchCollection(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="StruckThunder")
testCollectionA.setOwner(self.owner)
testCollectionB.setOwner(self.owner)
testCollectionA.create()
testCollectionB.create()
# There should be nothing in couch. Documents are only added for
# filesets and files.
testFilesA = []
for i in range(5):
testFile = File(lfn=makeUUID(),
size=random.randint(1024, 4096),
events=random.randint(1024, 4096))
testFilesA.append(testFile)
testFilesB = []
for i in range(10):
testFile = File(lfn=makeUUID(),
size=random.randint(1024, 4096),
events=random.randint(1024, 4096))
testFilesB.append(testFile)
testFilesetA = CouchFileset(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="TestFilesetA")
testFilesetB = CouchFileset(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="TestFilesetB")
testFilesetC = CouchFileset(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="TestFilesetC")
testCollectionA.addFileset(testFilesetA)
testCollectionB.addFileset(testFilesetB)
testCollectionB.addFileset(testFilesetC)
testFilesetA.add(testFilesA)
testFilesetB.add(testFilesA)
testFilesetC.add(testFilesA)
testFilesetC.add(testFilesB)
# Drop testCollectionA
testCollectionA.drop()
# Try to populate testFilesetA
testCollectionC = CouchCollection(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="ThunderStruck")
testCollectionC.setOwner(self.owner)
testCollectionC.populate()
self.assertEqual(
len(testCollectionC["filesets"]), 0,
"Error: There should be no filesets in this collect.")
# Try to populate testFilesetB
testCollectionD = CouchCollection(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="StruckThunder")
testCollectionD.setOwner(self.owner)
testCollectionD.populate()
for fileset in testCollectionD["filesets"]:
testFiles = testFilesA
if fileset["name"] == "TestFilesetC":
testFiles.extend(testFilesB)
self.assertEqual(len(testFiles), len(fileset.files.keys()),
"Error: Wrong number of files in fileset.")
for testFile in testFiles:
self.assertTrue(testFile["lfn"] in fileset.files.keys(),
"Error: File is missing.")
self.assertEqual(testFile["events"],
fileset.files[testFile["lfn"]]["events"],
"Error: Wrong number of events.")
self.assertEqual(testFile["size"],
fileset.files[testFile["lfn"]]["size"],
"Error: Wrong file size.")
return
def testL_CascadeCloseOutAnnnouncement(self):
"""
_testL_CascadeCloseOutAnnouncement_
Test the cascade closeout REST call, also
check that when announced a request deletes all ACDC records in the system.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put("request", schema)[0]
originalRequest = result['RequestName']
self.setupACDCDatabase(originalRequest, "/%s/DataProcessing" % originalRequest,
result['Requestor'], result['Group'])
depth = 2
nReq = 3
requests = [originalRequest]
def createChildrenRequest(parentRequest, i, nReq):
createdRequests = []
resubSchema = utils.getResubmissionSchema(parentRequest, "/%s/DataProcessing" % parentRequest,
groupName, userName)
result = self.jsonSender.put("request", resubSchema)[0]
requestName = result['RequestName']
self.setupACDCDatabase(requestName, "/%s/DataProcessing" % requestName, result['Requestor'], result['Group'])
createdRequests.append(requestName)
if i:
for _ in range(nReq):
createdRequests.extend(createChildrenRequest(requestName, i - 1, nReq))
return createdRequests
requests.extend(createChildrenRequest(originalRequest, depth, nReq))
for request in requests:
self.changeStatusAndCheck(request, 'assignment-approved')
for request in requests:
self.jsonSender.put("assignment?team=%s&requestName=%s" % (teamName, request))
for status in ['acquired',
'running-open', 'running-closed',
'completed']:
for request in requests:
self.changeStatusAndCheck(request, status)
self.jsonSender.post('closeout?requestName=%s&cascade=True' % originalRequest)
svc = CouchService(url = self.testInit.couchUrl,
database = "%s_acdc" % self.couchDBName)
owner = svc.newOwner(groupName, userName)
for request in requests:
result = self.jsonSender.get('request/%s' % request)
self.assertEqual(result[0]['RequestStatus'], 'closed-out')
testCollection = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = request)
testCollection.setOwner(owner)
testCollection.populate()
self.assertNotEqual(len(testCollection["filesets"]), 0)
self.jsonSender.post('announce?requestName=%s&cascade=True' % originalRequest)
for request in requests:
result = self.jsonSender.get('request/%s' % request)
self.assertEqual(result[0]['RequestStatus'], 'announced')
testCollection = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = request)
testCollection.setOwner(owner)
testCollection.populate()
self.assertEqual(len(testCollection["filesets"]), 0)
def testL_CascadeCloseOutAnnnouncement(self):
"""
_testL_CascadeCloseOutAnnouncement_
Test the cascade closeout REST call, also
check that when announced a request deletes all ACDC records in the system.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put("request", schema)[0]
originalRequest = result['RequestName']
self.setupACDCDatabase(originalRequest, "/%s/DataProcessing" % originalRequest,
result['Requestor'], result['Group'])
depth = 2
nReq = 3
requests = [originalRequest]
def createChildrenRequest(parentRequest, i, nReq):
createdRequests = []
resubSchema = utils.getResubmissionSchema(parentRequest, "/%s/DataProcessing" % parentRequest,
groupName, userName)
result = self.jsonSender.put("request", resubSchema)[0]
requestName = result['RequestName']
self.setupACDCDatabase(requestName, "/%s/DataProcessing" % requestName, result['Requestor'], result['Group'])
createdRequests.append(requestName)
if i:
for _ in range(nReq):
createdRequests.extend(createChildrenRequest(requestName, i - 1, nReq))
return createdRequests
requests.extend(createChildrenRequest(originalRequest, depth, nReq))
for request in requests:
self.changeStatusAndCheck(request, 'assignment-approved')
for request in requests:
self.jsonSender.put("assignment?team=%s&requestName=%s" % (teamName, request))
for status in ['acquired',
'running-open', 'running-closed',
'completed']:
for request in requests:
self.changeStatusAndCheck(request, status)
self.jsonSender.post('closeout?requestName=%s&cascade=True' % originalRequest)
svc = CouchService(url = self.testInit.couchUrl,
database = "%s_acdc" % self.couchDBName)
owner = svc.newOwner(groupName, userName)
for request in requests:
result = self.jsonSender.get('request/%s' % request)
self.assertEqual(result[0]['RequestStatus'], 'closed-out')
testCollection = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = request)
testCollection.setOwner(owner)
testCollection.populate()
self.assertNotEqual(len(testCollection["filesets"]), 0)
self.jsonSender.post('announce?requestName=%s&cascade=True' % originalRequest)
for request in requests:
result = self.jsonSender.get('request/%s' % request)
self.assertEqual(result[0]['RequestStatus'], 'announced')
testCollection = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = request)
testCollection.setOwner(owner)
testCollection.populate()
self.assertEqual(len(testCollection["filesets"]), 0)
def testCreatePopulateDrop(self):
"""
_testCreatePopulateDrop_
Test creating, populating and dropping a collection.
"""
testCollectionA = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = "Thunderstruck")
testCollectionB = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = "StruckThunder")
testCollectionA.setOwner(self.owner)
testCollectionB.setOwner(self.owner)
testCollectionA.create()
testCollectionB.create()
# There should be nothing in couch. Documents are only added for
# filesets and files.
testFilesA = []
for i in range(5):
testFile = File(lfn = makeUUID(), size = random.randint(1024, 4096),
events = random.randint(1024, 4096))
testFilesA.append(testFile)
testFilesB = []
for i in range(10):
testFile = File(lfn = makeUUID(), size = random.randint(1024, 4096),
events = random.randint(1024, 4096))
testFilesB.append(testFile)
testFilesetA = CouchFileset(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = "TestFilesetA")
testFilesetB = CouchFileset(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = "TestFilesetB")
testFilesetC = CouchFileset(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = "TestFilesetC")
testCollectionA.addFileset(testFilesetA)
testCollectionB.addFileset(testFilesetB)
testCollectionB.addFileset(testFilesetC)
testFilesetA.add(testFilesA)
testFilesetB.add(testFilesA)
testFilesetC.add(testFilesA)
testFilesetC.add(testFilesB)
# Drop testCollectionA
testCollectionA.drop()
# Try to populate testFilesetA
testCollectionC = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = "ThunderStruck")
testCollectionC.setOwner(self.owner)
testCollectionC.populate()
self.assertEqual(len(testCollectionC["filesets"]), 0,
"Error: There should be no filesets in this collect.")
# Try to populate testFilesetB
testCollectionD = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = "StruckThunder")
testCollectionD.setOwner(self.owner)
testCollectionD.populate()
for fileset in testCollectionD["filesets"]:
testFiles = testFilesA
if fileset["name"] == "TestFilesetC":
testFiles.extend(testFilesB)
self.assertEqual(len(testFiles), len(fileset.files.keys()),
"Error: Wrong number of files in fileset.")
for testFile in testFiles:
self.assertTrue(testFile["lfn"] in fileset.files.keys(),
"Error: File is missing.")
self.assertEqual(testFile["events"],
fileset.files[testFile["lfn"]]["events"],
"Error: Wrong number of events.")
self.assertEqual(testFile["size"],
fileset.files[testFile["lfn"]]["size"],
"Error: Wrong file size.")
return
