def testListCollectionsFilesets(self):
"""
_testListCollectionsFilesets_
Verify that collections and filesets in ACDC can be listed.
"""
svc = CouchService(url=self.testInit.couchUrl, database=self.testInit.couchDbName)
ownerA = svc.newOwner("somegroup", "someuserA")
ownerB = svc.newOwner("somegroup", "someuserB")
testCollectionA = CouchCollection(
database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="Thunderstruck"
)
testCollectionA.setOwner(ownerA)
testCollectionB = CouchCollection(
database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="Struckthunder"
)
testCollectionB.setOwner(ownerA)
testCollectionC = CouchCollection(
database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="Thunderstruck"
)
testCollectionC.setOwner(ownerB)
testCollectionD = CouchCollection(
database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="Thunderstruck"
)
testCollectionD.setOwner(ownerB)
testFilesetA = CouchFileset(database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="TestFilesetA")
testCollectionA.addFileset(testFilesetA)
testFilesetB = CouchFileset(database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="TestFilesetB")
testCollectionB.addFileset(testFilesetB)
testFilesetC = CouchFileset(database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="TestFilesetC")
testCollectionC.addFileset(testFilesetC)
testFilesetD = CouchFileset(database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="TestFilesetD")
testCollectionC.addFileset(testFilesetD)
testFiles = []
for i in range(5):
testFile = File(lfn=makeUUID(), size=random.randint(1024, 4096), events=random.randint(1024, 4096))
testFiles.append(testFile)
testFilesetA.add(testFiles)
testFilesetB.add(testFiles)
testFilesetC.add(testFiles)
testFilesetD.add(testFiles)
goldenCollectionNames = ["Thunderstruck", "Struckthunder"]
for collection in svc.listCollections(ownerA):
self.assertTrue(collection["name"] in goldenCollectionNames, "Error: Missing collection name.")
goldenCollectionNames.remove(collection["name"])
self.assertEqual(len(goldenCollectionNames), 0, "Error: Missing collections.")
goldenFilesetNames = ["TestFilesetC", "TestFilesetD"]
for fileset in svc.listFilesets(testCollectionD):
self.assertTrue(fileset["name"] in goldenFilesetNames, "Error: Missing fileset.")
goldenFilesetNames.remove(fileset["name"])
self.assertEqual(len(goldenFilesetNames), 0, "Error: Missing filesets.")
return
def setupACDCDatabase(self, collectionName, taskPath,
user, group):
"""
_setupACDCDatabase_
Populate an ACDC database with bogus records
associated to certain collection name, user and task path.
"""
acdcServer = CouchService(url = self.testInit.couchUrl,
database = "%s_acdc" % self.couchDBName)
owner = acdcServer.newOwner(group, user)
testCollection = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = collectionName)
testCollection.setOwner(owner)
testFileset = CouchFileset(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = taskPath)
testCollection.addFileset(testFileset)
testFiles = []
for _ in range(5):
testFile = File(lfn = makeUUID(), size = random.randint(1024, 4096),
events = random.randint(1024, 4096))
testFiles.append(testFile)
testFileset.add(testFiles)
def populateCouchDB(self):
"""
_populateCouchDB_
Populate the ACDC records
"""
svc = CouchService(url=self.testInit.couchUrl,
database=self.testInit.couchDbName)
ownerA = svc.newOwner("somegroup", "someuserA")
ownerB = svc.newOwner("somegroup", "someuserB")
testCollectionA = CouchCollection(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="Thunderstruck")
testCollectionA.setOwner(ownerA)
testCollectionB = CouchCollection(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="Struckthunder")
testCollectionB.setOwner(ownerA)
testCollectionC = CouchCollection(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="Thunderstruck")
testCollectionC.setOwner(ownerB)
testCollectionD = CouchCollection(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="Thunderstruck")
testCollectionD.setOwner(ownerB)
testFilesetA = CouchFileset(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="TestFilesetA")
testCollectionA.addFileset(testFilesetA)
testFilesetB = CouchFileset(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="TestFilesetB")
testCollectionB.addFileset(testFilesetB)
testFilesetC = CouchFileset(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="TestFilesetC")
testCollectionC.addFileset(testFilesetC)
testFilesetD = CouchFileset(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="TestFilesetD")
testCollectionC.addFileset(testFilesetD)
testFiles = []
for i in range(5):
testFile = File(lfn=makeUUID(),
size=random.randint(1024, 4096),
events=random.randint(1024, 4096))
testFiles.append(testFile)
testFilesetA.add(testFiles)
time.sleep(1)
testFilesetB.add(testFiles)
time.sleep(1)
testFilesetC.add(testFiles)
time.sleep(2)
testFilesetD.add(testFiles)
def setupACDCDatabase(self, collectionName, taskPath,
user, group):
"""
_setupACDCDatabase_
Populate an ACDC database with bogus records
associated to certain collection name, user and task path.
"""
acdcServer = CouchService(url = self.testInit.couchUrl,
database = "%s_acdc" % self.couchDBName)
owner = acdcServer.newOwner(group, user)
testCollection = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = collectionName)
testCollection.setOwner(owner)
testFileset = CouchFileset(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = taskPath)
testCollection.addFileset(testFileset)
testFiles = []
for _ in range(5):
testFile = File(lfn = makeUUID(), size = random.randint(1024, 4096),
events = random.randint(1024, 4096))
testFiles.append(testFile)
testFileset.add(testFiles)
def populateCouchDB(self):
"""
_populateCouchDB_
Populate the ACDC records
"""
svc = CouchService(url = self.testInit.couchUrl,
database = self.testInit.couchDbName)
ownerA = svc.newOwner("somegroup", "someuserA")
ownerB = svc.newOwner("somegroup", "someuserB")
testCollectionA = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = "Thunderstruck")
testCollectionA.setOwner(ownerA)
testCollectionB = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = "Struckthunder")
testCollectionB.setOwner(ownerA)
testCollectionC = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = "Thunderstruck")
testCollectionC.setOwner(ownerB)
testCollectionD = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = "Thunderstruck")
testCollectionD.setOwner(ownerB)
testFilesetA = CouchFileset(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = "TestFilesetA")
testCollectionA.addFileset(testFilesetA)
testFilesetB = CouchFileset(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = "TestFilesetB")
testCollectionB.addFileset(testFilesetB)
testFilesetC = CouchFileset(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = "TestFilesetC")
testCollectionC.addFileset(testFilesetC)
testFilesetD = CouchFileset(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = "TestFilesetD")
testCollectionC.addFileset(testFilesetD)
testFiles = []
for i in range(5):
testFile = File(lfn = makeUUID(), size = random.randint(1024, 4096),
events = random.randint(1024, 4096))
testFiles.append(testFile)
testFilesetA.add(testFiles)
time.sleep(1)
testFilesetB.add(testFiles)
time.sleep(1)
testFilesetC.add(testFiles)
time.sleep(2)
testFilesetD.add(testFiles)
def testOwners(self):
"""
_testOwners_
Verify that owners can be created, listed and removed.
"""
svc = CouchService(url=self.testInit.couchUrl, database=self.testInit.couchDbName)
self.assertEqual(svc.listOwners(), [])
owner = svc.newOwner("somegroup", "someuser")
self.assertTrue(len(svc.listOwners()) == 1)
owner2 = svc.listOwners()[0]
self.assertEqual(str(owner2["group"]), owner["group"])
self.assertEqual(str(owner2["name"]), owner["name"])
svc.removeOwner(owner2)
self.assertTrue(len(svc.listOwners()) == 0)
return
def testOwners(self):
"""
_testOwners_
Verify that owners can be created, listed and removed.
"""
svc = CouchService(url=self.testInit.couchUrl,
database=self.testInit.couchDbName)
self.assertEqual(svc.listOwners(), [])
owner = svc.newOwner("somegroup", "someuser")
self.assertTrue(len(svc.listOwners()) == 1)
owner2 = svc.listOwners()[0]
self.assertEqual(str(owner2['group']), owner['group'])
self.assertEqual(str(owner2['name']), owner['name'])
svc.removeOwner(owner2)
self.assertTrue(len(svc.listOwners()) == 0)
return
def testL_CascadeCloseOutAnnnouncement(self):
"""
_testL_CascadeCloseOutAnnouncement_
Test the cascade closeout REST call, also
check that when announced a request deletes all ACDC records in the system.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put("request", schema)[0]
originalRequest = result['RequestName']
self.setupACDCDatabase(originalRequest, "/%s/DataProcessing" % originalRequest,
result['Requestor'], result['Group'])
depth = 2
nReq = 3
requests = [originalRequest]
def createChildrenRequest(parentRequest, i, nReq):
createdRequests = []
resubSchema = utils.getResubmissionSchema(parentRequest, "/%s/DataProcessing" % parentRequest,
groupName, userName)
result = self.jsonSender.put("request", resubSchema)[0]
requestName = result['RequestName']
self.setupACDCDatabase(requestName, "/%s/DataProcessing" % requestName, result['Requestor'], result['Group'])
createdRequests.append(requestName)
if i:
for _ in range(nReq):
createdRequests.extend(createChildrenRequest(requestName, i - 1, nReq))
return createdRequests
requests.extend(createChildrenRequest(originalRequest, depth, nReq))
for request in requests:
self.changeStatusAndCheck(request, 'assignment-approved')
for request in requests:
self.jsonSender.put("assignment?team=%s&requestName=%s" % (teamName, request))
for status in ['acquired',
'running-open', 'running-closed',
'completed']:
for request in requests:
self.changeStatusAndCheck(request, status)
self.jsonSender.post('closeout?requestName=%s&cascade=True' % originalRequest)
svc = CouchService(url = self.testInit.couchUrl,
database = "%s_acdc" % self.couchDBName)
owner = svc.newOwner(groupName, userName)
for request in requests:
result = self.jsonSender.get('request/%s' % request)
self.assertEqual(result[0]['RequestStatus'], 'closed-out')
testCollection = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = request)
testCollection.setOwner(owner)
testCollection.populate()
self.assertNotEqual(len(testCollection["filesets"]), 0)
self.jsonSender.post('announce?requestName=%s&cascade=True' % originalRequest)
for request in requests:
result = self.jsonSender.get('request/%s' % request)
self.assertEqual(result[0]['RequestStatus'], 'announced')
testCollection = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = request)
testCollection.setOwner(owner)
testCollection.populate()
self.assertEqual(len(testCollection["filesets"]), 0)
def testListCollectionsFilesets(self):
"""
_testListCollectionsFilesets_
Verify that collections and filesets in ACDC can be listed.
"""
svc = CouchService(url=self.testInit.couchUrl,
database=self.testInit.couchDbName)
ownerA = svc.newOwner("somegroup", "someuserA")
ownerB = svc.newOwner("somegroup", "someuserB")
testCollectionA = CouchCollection(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="Thunderstruck")
testCollectionA.setOwner(ownerA)
testCollectionB = CouchCollection(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="Struckthunder")
testCollectionB.setOwner(ownerA)
testCollectionC = CouchCollection(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="Thunderstruck")
testCollectionC.setOwner(ownerB)
testCollectionD = CouchCollection(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="Thunderstruck")
testCollectionD.setOwner(ownerB)
testFilesetA = CouchFileset(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="TestFilesetA")
testCollectionA.addFileset(testFilesetA)
testFilesetB = CouchFileset(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="TestFilesetB")
testCollectionB.addFileset(testFilesetB)
testFilesetC = CouchFileset(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="TestFilesetC")
testCollectionC.addFileset(testFilesetC)
testFilesetD = CouchFileset(database=self.testInit.couchDbName,
url=self.testInit.couchUrl,
name="TestFilesetD")
testCollectionC.addFileset(testFilesetD)
testFiles = []
for i in range(5):
testFile = File(lfn=makeUUID(),
size=random.randint(1024, 4096),
events=random.randint(1024, 4096))
testFiles.append(testFile)
testFilesetA.add(testFiles)
testFilesetB.add(testFiles)
testFilesetC.add(testFiles)
testFilesetD.add(testFiles)
goldenCollectionNames = ["Thunderstruck", "Struckthunder"]
for collection in svc.listCollections(ownerA):
self.assertTrue(collection["name"] in goldenCollectionNames,
"Error: Missing collection name.")
goldenCollectionNames.remove(collection["name"])
self.assertEqual(len(goldenCollectionNames), 0,
"Error: Missing collections.")
goldenFilesetNames = ["TestFilesetC", "TestFilesetD"]
for fileset in svc.listFilesets(testCollectionD):
self.assertTrue(fileset["name"] in goldenFilesetNames,
"Error: Missing fileset.")
goldenFilesetNames.remove(fileset["name"])
self.assertEqual(len(goldenFilesetNames), 0,
"Error: Missing filesets.")
return
def testL_CascadeCloseOutAnnnouncement(self):
"""
_testL_CascadeCloseOutAnnouncement_
Test the cascade closeout REST call, also
check that when announced a request deletes all ACDC records in the system.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put("request", schema)[0]
originalRequest = result['RequestName']
self.setupACDCDatabase(originalRequest, "/%s/DataProcessing" % originalRequest,
result['Requestor'], result['Group'])
depth = 2
nReq = 3
requests = [originalRequest]
def createChildrenRequest(parentRequest, i, nReq):
createdRequests = []
resubSchema = utils.getResubmissionSchema(parentRequest, "/%s/DataProcessing" % parentRequest,
groupName, userName)
result = self.jsonSender.put("request", resubSchema)[0]
requestName = result['RequestName']
self.setupACDCDatabase(requestName, "/%s/DataProcessing" % requestName, result['Requestor'], result['Group'])
createdRequests.append(requestName)
if i:
for _ in range(nReq):
createdRequests.extend(createChildrenRequest(requestName, i - 1, nReq))
return createdRequests
requests.extend(createChildrenRequest(originalRequest, depth, nReq))
for request in requests:
self.changeStatusAndCheck(request, 'assignment-approved')
for request in requests:
self.jsonSender.put("assignment?team=%s&requestName=%s" % (teamName, request))
for status in ['acquired',
'running-open', 'running-closed',
'completed']:
for request in requests:
self.changeStatusAndCheck(request, status)
self.jsonSender.post('closeout?requestName=%s&cascade=True' % originalRequest)
svc = CouchService(url = self.testInit.couchUrl,
database = "%s_acdc" % self.couchDBName)
owner = svc.newOwner(groupName, userName)
for request in requests:
result = self.jsonSender.get('request/%s' % request)
self.assertEqual(result[0]['RequestStatus'], 'closed-out')
testCollection = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = request)
testCollection.setOwner(owner)
testCollection.populate()
self.assertNotEqual(len(testCollection["filesets"]), 0)
self.jsonSender.post('announce?requestName=%s&cascade=True' % originalRequest)
for request in requests:
result = self.jsonSender.get('request/%s' % request)
self.assertEqual(result[0]['RequestStatus'], 'announced')
testCollection = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = request)
testCollection.setOwner(owner)
testCollection.populate()
self.assertEqual(len(testCollection["filesets"]), 0)
