Exemplo n.º 1
0
    def populateCouchDB(self):
        """
        _populateCouchDB_

        Populate the ACDC records
        """
        svc = CouchService(url=self.testInit.couchUrl,
                           database=self.testInit.couchDbName)

        ownerA = svc.newOwner("somegroup", "someuserA")
        ownerB = svc.newOwner("somegroup", "someuserB")

        testCollectionA = CouchCollection(database=self.testInit.couchDbName,
                                          url=self.testInit.couchUrl,
                                          name="Thunderstruck")
        testCollectionA.setOwner(ownerA)
        testCollectionB = CouchCollection(database=self.testInit.couchDbName,
                                          url=self.testInit.couchUrl,
                                          name="Struckthunder")
        testCollectionB.setOwner(ownerA)
        testCollectionC = CouchCollection(database=self.testInit.couchDbName,
                                          url=self.testInit.couchUrl,
                                          name="Thunderstruck")
        testCollectionC.setOwner(ownerB)
        testCollectionD = CouchCollection(database=self.testInit.couchDbName,
                                          url=self.testInit.couchUrl,
                                          name="Thunderstruck")
        testCollectionD.setOwner(ownerB)

        testFilesetA = CouchFileset(database=self.testInit.couchDbName,
                                    url=self.testInit.couchUrl,
                                    name="TestFilesetA")
        testCollectionA.addFileset(testFilesetA)
        testFilesetB = CouchFileset(database=self.testInit.couchDbName,
                                    url=self.testInit.couchUrl,
                                    name="TestFilesetB")
        testCollectionB.addFileset(testFilesetB)
        testFilesetC = CouchFileset(database=self.testInit.couchDbName,
                                    url=self.testInit.couchUrl,
                                    name="TestFilesetC")
        testCollectionC.addFileset(testFilesetC)
        testFilesetD = CouchFileset(database=self.testInit.couchDbName,
                                    url=self.testInit.couchUrl,
                                    name="TestFilesetD")
        testCollectionC.addFileset(testFilesetD)

        testFiles = []
        for i in range(5):
            testFile = File(lfn=makeUUID(),
                            size=random.randint(1024, 4096),
                            events=random.randint(1024, 4096))
            testFiles.append(testFile)

        testFilesetA.add(testFiles)
        time.sleep(1)
        testFilesetB.add(testFiles)
        time.sleep(1)
        testFilesetC.add(testFiles)
        time.sleep(2)
        testFilesetD.add(testFiles)
Exemplo n.º 2
0
    def setupACDCDatabase(self, collectionName, taskPath,
                          user, group):
        """
        _setupACDCDatabase_

        Populate an ACDC database with bogus records
        associated to certain collection name, user and task path.
        """
        acdcServer = CouchService(url = self.testInit.couchUrl,
                                  database = "%s_acdc" % self.couchDBName)
        owner = acdcServer.newOwner(group, user)
        testCollection = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl,
                                          name = collectionName)
        testCollection.setOwner(owner)
        testFileset = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = taskPath)
        testCollection.addFileset(testFileset)

        testFiles = []
        for _ in range(5):
            testFile = File(lfn = makeUUID(), size = random.randint(1024, 4096),
                            events = random.randint(1024, 4096))
            testFiles.append(testFile)

        testFileset.add(testFiles)
Exemplo n.º 3
0
def changeStatus(requestName, status, wmstatUrl, acdcUrl):
    """ Changes the status for this request """
    request = GetRequest.getRequestByName(requestName)
    if not status in RequestStatus.StatusList:
        raise RuntimeError, "Bad status code " + status
    if not request.has_key('RequestStatus'):
        raise RuntimeError, "Cannot find status for request " + requestName
    oldStatus = request['RequestStatus']
    if not status in RequestStatus.NextStatus[oldStatus]:
        raise RuntimeError, "Cannot change status from %s to %s.  Allowed values are %s" % (
           oldStatus, status,  RequestStatus.NextStatus[oldStatus])

    if status == 'aborted':
        # delete from the workqueue
        if not privileged() and not ownsRequest(request):
            raise cherrypy.HTTPError(403, "You are not allowed to abort this request")
        elif not privileged():
            raise cherrypy.HTTPError(403, "You are not allowed to change the state for this request")
        # delete from the workqueue if it's been assigned to one
        if status in RequestStatus.NextStatus[oldStatus]:
            abortRequest(requestName)
        else:
            raise cherrypy.HTTPError(400, "You cannot abort a request in state %s" % oldStatus)
        
    if status == 'announced':
        # cleanup acdc database, if possible
        if acdcUrl:
            url, database = WMCore.Lexicon.splitCouchServiceURL(acdcUrl)
            acdcService = CouchService(url = url, database = database)
            acdcService.removeFilesetsByCollectionName(requestName)

    # finally, perform the transition, have to do it in both Oracle and CouchDB
    # and in WMStats
    ChangeState.changeRequestStatus(requestName, status, wmstatUrl=wmstatUrl)
Exemplo n.º 4
0
def changeStatus(requestName, status, wmstatUrl, acdcUrl):
    """ Changes the status for this request """
    request = GetRequest.getRequestByName(requestName)
    if not status in RequestStatus.StatusList:
        raise RuntimeError, "Bad status code " + status
    if not request.has_key('RequestStatus'):
        raise RuntimeError, "Cannot find status for request " + requestName
    oldStatus = request['RequestStatus']
    if not status in RequestStatus.NextStatus[oldStatus]:
        raise RuntimeError, "Cannot change status from %s to %s.  Allowed values are %s" % (
           oldStatus, status,  RequestStatus.NextStatus[oldStatus])

    if status == 'aborted' or status == 'force-complete':
        # delete from the workqueue
        if not privileged() and not ownsRequest(request):
            raise cherrypy.HTTPError(403, "You are not allowed to %s this request" % status)
        elif not privileged():
            raise cherrypy.HTTPError(403, "You are not allowed to change the state for this request")
        # delete from the workqueue if it's been assigned to one
        if status in RequestStatus.NextStatus[oldStatus]:
            abortRequest(requestName)
        else:
            raise cherrypy.HTTPError(400, "You cannot abort a request in state %s" % oldStatus)
        
    if status == 'announced':
        # cleanup acdc database, if possible
        if acdcUrl:
            url, database = WMCore.Lexicon.splitCouchServiceURL(acdcUrl)
            acdcService = CouchService(url = url, database = database)
            acdcService.removeFilesetsByCollectionName(requestName)

    # finally, perform the transition, have to do it in both Oracle and CouchDB
    # and in WMStats
    ChangeState.changeRequestStatus(requestName, status, wmstatUrl=wmstatUrl)
Exemplo n.º 5
0
    def testListCollectionsFilesets(self):
        """
        _testListCollectionsFilesets_

        Verify that collections and filesets in ACDC can be listed.
        """
        svc = CouchService(url=self.testInit.couchUrl,
                           database=self.testInit.couchDbName)

        testCollectionA = CouchCollection(database=self.testInit.couchDbName,
                                          url=self.testInit.couchUrl,
                                          name="Thunderstruck")
        testCollectionB = CouchCollection(database=self.testInit.couchDbName,
                                          url=self.testInit.couchUrl,
                                          name="Struckthunder")
        testCollectionC = CouchCollection(database=self.testInit.couchDbName,
                                          url=self.testInit.couchUrl,
                                          name="Thunderstruck")
        testCollectionD = CouchCollection(database=self.testInit.couchDbName,
                                          url=self.testInit.couchUrl,
                                          name="Thunderstruck")

        testFilesetA = CouchFileset(database=self.testInit.couchDbName,
                                    url=self.testInit.couchUrl,
                                    name="TestFilesetA")
        testCollectionA.addFileset(testFilesetA)
        testFilesetB = CouchFileset(database=self.testInit.couchDbName,
                                    url=self.testInit.couchUrl,
                                    name="TestFilesetB")
        testCollectionB.addFileset(testFilesetB)
        testFilesetC = CouchFileset(database=self.testInit.couchDbName,
                                    url=self.testInit.couchUrl,
                                    name="TestFilesetC")
        testCollectionC.addFileset(testFilesetC)
        testFilesetD = CouchFileset(database=self.testInit.couchDbName,
                                    url=self.testInit.couchUrl,
                                    name="TestFilesetD")
        testCollectionC.addFileset(testFilesetD)

        testFiles = []
        for i in range(5):
            testFile = File(lfn=makeUUID(),
                            size=random.randint(1024, 4096),
                            events=random.randint(1024, 4096))
            testFiles.append(testFile)

        testFilesetA.add(testFiles)
        testFilesetB.add(testFiles)
        testFilesetC.add(testFiles)
        testFilesetD.add(testFiles)

        goldenFilesetNames = ["TestFilesetA", "TestFilesetC", "TestFilesetD"]
        for fileset in svc.listFilesets(testCollectionD):
            self.assertTrue(fileset["name"] in goldenFilesetNames,
                            "Error: Missing fileset.")
            goldenFilesetNames.remove(fileset["name"])
        self.assertEqual(len(goldenFilesetNames), 0,
                         "Error: Missing filesets.")

        return
Exemplo n.º 6
0
    def testTimestampAccounting(self):
        """
        _testTimestampAccounting_

        Check the correct functioning of the timestamp view in the ACDC
        couchapp and the function to remove old filesets.
        """
        self.populateCouchDB()
        svc = CouchService(url=self.testInit.couchUrl,
                           database=self.testInit.couchDbName)

        currentTime = time.time()
        database = CouchServer(self.testInit.couchUrl).connectDatabase(
            self.testInit.couchDbName)
        results = database.loadView("ACDC", "byTimestamp",
                                    {"endkey": currentTime})
        self.assertEqual(len(results["rows"]), 4)
        results = database.loadView("ACDC", "byTimestamp",
                                    {"endkey": currentTime - 2})
        self.assertEqual(len(results["rows"]), 3)
        results = database.loadView("ACDC", "byTimestamp",
                                    {"endkey": currentTime - 3})
        self.assertEqual(len(results["rows"]), 2)
        results = database.loadView("ACDC", "byTimestamp",
                                    {"endkey": currentTime - 4})
        self.assertEqual(len(results["rows"]), 1)
        results = database.loadView("ACDC", "byTimestamp",
                                    {"endkey": currentTime - 5})
        self.assertEqual(len(results["rows"]), 0)
        svc.removeOldFilesets(0)
        results = database.loadView("ACDC", "byTimestamp",
                                    {"endkey": currentTime})
        self.assertEqual(len(results["rows"]), 0)
        return
Exemplo n.º 7
0
    def setupACDCDatabase(self, collectionName, taskPath,
                          user, group):
        """
        _setupACDCDatabase_

        Populate an ACDC database with bogus records
        associated to certain collection name, user and task path.
        """
        acdcServer = CouchService(url = self.testInit.couchUrl,
                                  database = "%s_acdc" % self.couchDBName)
        owner = acdcServer.newOwner(group, user)
        testCollection = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl,
                                          name = collectionName)
        testCollection.setOwner(owner)
        testFileset = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = taskPath)
        testCollection.addFileset(testFileset)

        testFiles = []
        for _ in range(5):
            testFile = File(lfn = makeUUID(), size = random.randint(1024, 4096),
                            events = random.randint(1024, 4096))
            testFiles.append(testFile)

        testFileset.add(testFiles)
Exemplo n.º 8
0
    def testTimestampAccounting(self):
        """
        _testTimestampAccounting_

        Check the correct functioning of the timestamp view in the ACDC
        couchapp and the function to remove old filesets.
        """
        self.populateCouchDB()
        svc = CouchService(url=self.testInit.couchUrl, database=self.testInit.couchDbName)

        currentTime = time.time()
        database = CouchServer(self.testInit.couchUrl).connectDatabase(self.testInit.couchDbName)
        results = database.loadView("ACDC", "byTimestamp", {"endkey": currentTime})
        self.assertEqual(len(results["rows"]), 4)
        results = database.loadView("ACDC", "byTimestamp", {"endkey": currentTime - 2})
        self.assertEqual(len(results["rows"]), 3)
        results = database.loadView("ACDC", "byTimestamp", {"endkey": currentTime - 3})
        self.assertEqual(len(results["rows"]), 2)
        results = database.loadView("ACDC", "byTimestamp", {"endkey": currentTime - 4})
        self.assertEqual(len(results["rows"]), 1)
        results = database.loadView("ACDC", "byTimestamp", {"endkey": currentTime - 5})
        self.assertEqual(len(results["rows"]), 0)
        svc.removeOldFilesets(0)
        results = database.loadView("ACDC", "byTimestamp", {"endkey": currentTime})
        self.assertEqual(len(results["rows"]), 0)
        return
Exemplo n.º 9
0
    def testListCollectionsFilesets(self):
        """
        _testListCollectionsFilesets_

        Verify that collections and filesets in ACDC can be listed.
        """
        svc = CouchService(url=self.testInit.couchUrl, database=self.testInit.couchDbName)

        ownerA = svc.newOwner("somegroup", "someuserA")
        ownerB = svc.newOwner("somegroup", "someuserB")

        testCollectionA = CouchCollection(
            database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="Thunderstruck"
        )
        testCollectionA.setOwner(ownerA)
        testCollectionB = CouchCollection(
            database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="Struckthunder"
        )
        testCollectionB.setOwner(ownerA)
        testCollectionC = CouchCollection(
            database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="Thunderstruck"
        )
        testCollectionC.setOwner(ownerB)
        testCollectionD = CouchCollection(
            database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="Thunderstruck"
        )
        testCollectionD.setOwner(ownerB)

        testFilesetA = CouchFileset(database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="TestFilesetA")
        testCollectionA.addFileset(testFilesetA)
        testFilesetB = CouchFileset(database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="TestFilesetB")
        testCollectionB.addFileset(testFilesetB)
        testFilesetC = CouchFileset(database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="TestFilesetC")
        testCollectionC.addFileset(testFilesetC)
        testFilesetD = CouchFileset(database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="TestFilesetD")
        testCollectionC.addFileset(testFilesetD)

        testFiles = []
        for i in range(5):
            testFile = File(lfn=makeUUID(), size=random.randint(1024, 4096), events=random.randint(1024, 4096))
            testFiles.append(testFile)

        testFilesetA.add(testFiles)
        testFilesetB.add(testFiles)
        testFilesetC.add(testFiles)
        testFilesetD.add(testFiles)

        goldenCollectionNames = ["Thunderstruck", "Struckthunder"]
        for collection in svc.listCollections(ownerA):
            self.assertTrue(collection["name"] in goldenCollectionNames, "Error: Missing collection name.")
            goldenCollectionNames.remove(collection["name"])
        self.assertEqual(len(goldenCollectionNames), 0, "Error: Missing collections.")

        goldenFilesetNames = ["TestFilesetC", "TestFilesetD"]
        for fileset in svc.listFilesets(testCollectionD):
            self.assertTrue(fileset["name"] in goldenFilesetNames, "Error: Missing fileset.")
            goldenFilesetNames.remove(fileset["name"])
        self.assertEqual(len(goldenFilesetNames), 0, "Error: Missing filesets.")

        return
Exemplo n.º 10
0
    def populateCouchDB(self):
        """
        _populateCouchDB_

        Populate the ACDC records
        """
        svc = CouchService(url = self.testInit.couchUrl,
                           database = self.testInit.couchDbName)

        ownerA = svc.newOwner("somegroup", "someuserA")
        ownerB = svc.newOwner("somegroup", "someuserB")

        testCollectionA = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl,
                                          name = "Thunderstruck")
        testCollectionA.setOwner(ownerA)
        testCollectionB = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl,
                                          name = "Struckthunder")
        testCollectionB.setOwner(ownerA)
        testCollectionC = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl,
                                          name = "Thunderstruck")
        testCollectionC.setOwner(ownerB)
        testCollectionD = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl,
                                          name = "Thunderstruck")
        testCollectionD.setOwner(ownerB)

        testFilesetA = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = "TestFilesetA")
        testCollectionA.addFileset(testFilesetA)
        testFilesetB = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = "TestFilesetB")
        testCollectionB.addFileset(testFilesetB)
        testFilesetC = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = "TestFilesetC")
        testCollectionC.addFileset(testFilesetC)
        testFilesetD = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = "TestFilesetD")
        testCollectionC.addFileset(testFilesetD)

        testFiles = []
        for i in range(5):
            testFile = File(lfn = makeUUID(), size = random.randint(1024, 4096),
                            events = random.randint(1024, 4096))
            testFiles.append(testFile)

        testFilesetA.add(testFiles)
        time.sleep(1)
        testFilesetB.add(testFiles)
        time.sleep(1)
        testFilesetC.add(testFiles)
        time.sleep(2)
        testFilesetD.add(testFiles)
Exemplo n.º 11
0
    def __init__(self, rest, config):

        super(CouchDBCleanup, self).__init__(config)
        self.reqDB = RequestDBReader(config.reqmgrdb_url)
        self.reqmgrAux = ReqMgrAux(config.reqmgr2_url, logger=self.logger)
        # statuses that we want to keep the transfer documents
        self.transferStatuses = [
            "assigned", "staging", "staged", "acquired", "failed",
            "running-open", "running-closed"
        ]

        baseURL, acdcDB = splitCouchServiceURL(config.acdc_url)
        self.acdcService = CouchService(url=baseURL, database=acdcDB)
Exemplo n.º 12
0
    def testListCollectionsFilesets(self):
        """
        _testListCollectionsFilesets_

        Verify that collections and filesets in ACDC can be listed.
        """
        testCollection = self.populateCouchDB()

        svc = CouchService(url=self.testInit.couchUrl,
                           database=self.testInit.couchDbName)

        goldenFilesetNames = ["TestFilesetA", "TestFilesetC", "TestFilesetD"]
        for fileset in svc.listFilesets(testCollection):
            self.assertTrue(fileset["name"] in goldenFilesetNames, "Error: Missing fileset.")
            goldenFilesetNames.remove(fileset["name"])
        self.assertEqual(len(goldenFilesetNames), 0, "Error: Missing filesets.")

        return
Exemplo n.º 13
0
    def acdcCleanup(self, config):
        """
        gather active data statistics
        """

        reqDB = RequestDBReader(config.reqmgrdb_url)

        from WMCore.ACDC.CouchService import CouchService
        baseURL, acdcDB = splitCouchServiceURL(config.acdc_url)
        acdcService = CouchService(url=baseURL, database=acdcDB)
        originalRequests = acdcService.listCollectionNames()

        if len(originalRequests) == 0:
            return
        # filter requests
        results = reqDB._getCouchView("byrequest", {}, originalRequests)
        # checkt he status of the requests [announced, rejected-archived, aborted-archived, normal-archived]
        deleteStates = [
            "announced", "rejected-archived", "aborted-archived",
            "normal-archived"
        ]
        filteredRequests = []
        for row in results["rows"]:
            if row["value"][0] in deleteStates:
                filteredRequests.append(row["key"])

        total = 0
        for req in filteredRequests:
            try:
                deleted = acdcService.removeFilesetsByCollectionName(req)
                if deleted == None:
                    self.logger.warning("request alread deleted %s", req)
                else:
                    total += len(deleted)
                    self.logger.info("request %s deleted", req)
            except Exception as ex:
                self.logger.error(
                    "request deleted failed: will try again %s: %s", req,
                    str(ex))
        self.logger.info("total %s requests deleted", total)
        return
Exemplo n.º 14
0
    def testRemoveByCollectionName(self):
        """
        _testRemoveByCollectionName_

        Check the function to obliterate all the filesets of a collection
        """
        self.populateCouchDB()
        svc = CouchService(url=self.testInit.couchUrl, database=self.testInit.couchDbName)
        database = CouchServer(self.testInit.couchUrl).connectDatabase(self.testInit.couchDbName)

        results = database.loadView("ACDC", "byCollectionName", keys=["Thunderstruck"])
        self.assertTrue(len(results["rows"]) > 0)
        svc.removeFilesetsByCollectionName("Thunderstruck")
        results = database.loadView("ACDC", "byCollectionName", keys=["Thunderstruck"])
        self.assertEqual(len(results["rows"]), 0)
        results = database.loadView("ACDC", "byCollectionName", keys=["Struckthunder"])
        self.assertTrue(len(results["rows"]) > 0)
        svc.removeFilesetsByCollectionName("Struckthunder")
        results = database.loadView("ACDC", "byCollectionName", keys=["Struckthunder"])
        self.assertEqual(len(results["rows"]), 0)
        return
Exemplo n.º 15
0
    def testRemoveByCollectionName(self):
        """
        _testRemoveByCollectionName_

        Check the function to obliterate all the filesets of a collection
        """
        self.populateCouchDB()
        svc = CouchService(url=self.testInit.couchUrl,
                           database=self.testInit.couchDbName)
        database = CouchServer(self.testInit.couchUrl).connectDatabase(
            self.testInit.couchDbName)

        results = database.loadView("ACDC",
                                    "byCollectionName",
                                    keys=["Thunderstruck"])
        self.assertTrue(len(results["rows"]) > 0)
        svc.removeFilesetsByCollectionName("Thunderstruck")
        results = database.loadView("ACDC",
                                    "byCollectionName",
                                    keys=["Thunderstruck"])
        self.assertEqual(len(results["rows"]), 0)
        results = database.loadView("ACDC",
                                    "byCollectionName",
                                    keys=["Struckthunder"])
        self.assertTrue(len(results["rows"]) > 0)
        svc.removeFilesetsByCollectionName("Struckthunder")
        results = database.loadView("ACDC",
                                    "byCollectionName",
                                    keys=["Struckthunder"])
        self.assertEqual(len(results["rows"]), 0)
        return
Exemplo n.º 16
0
    def acdcCleanup(self, config):
        """
        gather active data statistics
        """
        
        reqDB = RequestDBReader(config.reqmgrdb_url)

        from WMCore.ACDC.CouchService import CouchService
        baseURL, acdcDB = splitCouchServiceURL(config.acdc_url)
        acdcService = CouchService(url = baseURL, database = acdcDB)
        originalRequests = acdcService.listCollectionNames()
        
        if len(originalRequests) == 0:
            return 
        # filter requests
        results = reqDB._getCouchView("byrequest", {}, originalRequests)
        # checkt he status of the requests [announced, rejected-archived, aborted-archived, normal-archived]
        deleteStates = ["announced", "rejected-archived", "aborted-archived", "normal-archived"]
        filteredRequests = []
        for row in results["rows"]:
            if row["value"][0] in deleteStates:
                filteredRequests.append(row["key"])
                
        total = 0
        for req in filteredRequests:
            try:
                deleted = acdcService.removeFilesetsByCollectionName(req)
                if deleted == None:
                    self.logger.warning("request alread deleted %s" % req)
                else:
                    total += len(deleted)
                    self.logger.info("request %s deleted" % req)
            except:
                self.logger.error("request deleted failed: will try again %s" % req)
        self.logger.info("total %s requests deleted" % total)        
        return
Exemplo n.º 17
0
    def testOwners(self):
        """
        _testOwners_

        Verify that owners can be created, listed and removed.
        """
        svc = CouchService(url=self.testInit.couchUrl,
                           database=self.testInit.couchDbName)
        self.assertEqual(svc.listOwners(), [])

        owner = svc.newOwner("somegroup", "someuser")

        self.assertTrue(len(svc.listOwners()) == 1)

        owner2 = svc.listOwners()[0]
        self.assertEqual(str(owner2['group']), owner['group'])
        self.assertEqual(str(owner2['name']), owner['name'])

        svc.removeOwner(owner2)
        self.assertTrue(len(svc.listOwners()) == 0)
        return
Exemplo n.º 18
0
    def testOwners(self):
        """
        _testOwners_

        Verify that owners can be created, listed and removed.
        """
        svc = CouchService(url=self.testInit.couchUrl, database=self.testInit.couchDbName)
        self.assertEqual(svc.listOwners(), [])

        owner = svc.newOwner("somegroup", "someuser")

        self.assertTrue(len(svc.listOwners()) == 1)

        owner2 = svc.listOwners()[0]
        self.assertEqual(str(owner2["group"]), owner["group"])
        self.assertEqual(str(owner2["name"]), owner["name"])

        svc.removeOwner(owner2)
        self.assertTrue(len(svc.listOwners()) == 0)
        return
Exemplo n.º 19
0
 def __init__(self, url, database, **opts):
     CouchService.__init__(self, url=url, database=database, **opts)
Exemplo n.º 20
0
    def testL_CascadeCloseOutAnnnouncement(self):
        """
        _testL_CascadeCloseOutAnnouncement_

        Test the cascade closeout REST call, also
        check that when announced a request deletes all ACDC records in the system.
        """
        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")

        result = self.jsonSender.put("request", schema)[0]
        originalRequest = result['RequestName']
        self.setupACDCDatabase(originalRequest, "/%s/DataProcessing" % originalRequest,
                               result['Requestor'], result['Group'])
        depth = 2
        nReq = 3
        requests = [originalRequest]
        def createChildrenRequest(parentRequest, i, nReq):
            createdRequests = []
            resubSchema = utils.getResubmissionSchema(parentRequest, "/%s/DataProcessing" % parentRequest,
                                                      groupName, userName)
            result = self.jsonSender.put("request", resubSchema)[0]
            requestName = result['RequestName']
            self.setupACDCDatabase(requestName, "/%s/DataProcessing" % requestName, result['Requestor'], result['Group'])
            createdRequests.append(requestName)
            if i:
                for _ in range(nReq):
                    createdRequests.extend(createChildrenRequest(requestName, i - 1, nReq))
            return createdRequests
        requests.extend(createChildrenRequest(originalRequest, depth, nReq))
        for request in requests:
            self.changeStatusAndCheck(request, 'assignment-approved')
        for request in requests:
            self.jsonSender.put("assignment?team=%s&requestName=%s" % (teamName, request))
        for status in ['acquired',
                       'running-open', 'running-closed',
                       'completed']:
            for request in requests:
                self.changeStatusAndCheck(request, status)
        self.jsonSender.post('closeout?requestName=%s&cascade=True' % originalRequest)
        svc = CouchService(url = self.testInit.couchUrl,
                                  database = "%s_acdc" % self.couchDBName)

        owner = svc.newOwner(groupName, userName)
        for request in requests:
            result = self.jsonSender.get('request/%s' % request)
            self.assertEqual(result[0]['RequestStatus'], 'closed-out')
            testCollection = CouchCollection(database = self.testInit.couchDbName,
                                             url = self.testInit.couchUrl,
                                             name = request)
            testCollection.setOwner(owner)
            testCollection.populate()
            self.assertNotEqual(len(testCollection["filesets"]), 0)

        self.jsonSender.post('announce?requestName=%s&cascade=True' % originalRequest)
        for request in requests:

            result = self.jsonSender.get('request/%s' % request)
            self.assertEqual(result[0]['RequestStatus'], 'announced')
            testCollection = CouchCollection(database = self.testInit.couchDbName,
                                             url = self.testInit.couchUrl,
                                             name = request)
            testCollection.setOwner(owner)
            testCollection.populate()
            self.assertEqual(len(testCollection["filesets"]), 0)
Exemplo n.º 21
0
class CouchDBCleanup(CherryPyPeriodicTask):
    def __init__(self, rest, config):

        super(CouchDBCleanup, self).__init__(config)
        self.reqDB = RequestDBReader(config.reqmgrdb_url)
        self.reqmgrAux = ReqMgrAux(config.reqmgr2_url, logger=self.logger)
        # statuses that we want to keep the transfer documents
        self.transferStatuses = [
            "assigned", "staging", "staged", "acquired", "failed",
            "running-open", "running-closed"
        ]

        baseURL, acdcDB = splitCouchServiceURL(config.acdc_url)
        self.acdcService = CouchService(url=baseURL, database=acdcDB)

    def setConcurrentTasks(self, config):
        """
        sets the list of functions which
        """
        self.concurrentTasks = [{
            'func': self.acdcCleanup,
            'duration': config.acdcCleanDuration
        }, {
            'func': self.auxCouchCleanup,
            'duration': config.auxCleanDuration
        }]

    def auxCouchCleanup(self, config):
        """
        Cleanup TRANSFER documents from the reqmgr_auxiliary CouchDB.
        The list of status can be expanded in the future
        """
        self.logger.info("Fetching TRANSFER documents from CouchDB...")

        transferDocs = self.reqmgrAux.getTransferInfo("ALL_DOCS")
        if not transferDocs:
            self.logger.info(
                "  there are no transfer documents in the database.")
            return
        auxDocs = []
        for row in transferDocs:
            auxDocs.append(row['workflowName'])

        results = self.reqDB._getCouchView("bystatus", {},
                                           self.transferStatuses)
        activeRequests = []
        for row in results["rows"]:
            activeRequests.append(row["id"])

        # now find transfer docs that are not active in the system
        transferDocs = []
        for transferDoc in auxDocs:
            if transferDoc not in activeRequests:
                transferDocs.append(transferDoc)
        self.logger.info("Found %d transfer documents to delete",
                         len(transferDocs))

        for wflowName in transferDocs:
            self.logger.info("Deleting transfer document: %s", wflowName)
            try:
                self.reqmgrAux.deleteConfigDoc("transferinfo", wflowName)
            except Exception as exc:
                self.logger.warning(
                    "Failed to delete transfer doc: %s. Error: %s", wflowName,
                    str(exc))
        self.logger.info("Transfer documents cleanup completed.")

    def acdcCleanup(self, config):
        """
        gather active data statistics
        """
        self.logger.info("Fetching ACDC collection names...")
        originalRequests = self.acdcService.listCollectionNames()
        if not originalRequests:
            self.logger.info("  there are no collection documents to delete.")
            return

        # filter requests
        results = self.reqDB._getCouchView("byrequest", {}, originalRequests)
        # filter requests only in the following status
        deleteStates = [
            "announced", "rejected-archived", "aborted-archived",
            "normal-archived"
        ]
        filteredRequests = []
        for row in results["rows"]:
            if row["value"][0] in deleteStates:
                filteredRequests.append(row["key"])

        total = 0
        for req in filteredRequests:
            try:
                self.logger.info("Removing ACDC collection for: %s", req)
                deleted = self.acdcService.removeFilesetsByCollectionName(req)
                if deleted is None:
                    self.logger.warning("  request '%s' already deleted", req)
                else:
                    total += len(deleted)
                    self.logger.info("request %s deleted", req)
            except Exception as ex:
                self.logger.error(
                    "Failed to delete request: %s, will try again later. Error: %s",
                    req, str(ex))
        self.logger.info("total %s requests deleted", total)
        return
Exemplo n.º 22
0
 def __init__(self, url, database, **opts):
     CouchService.__init__(self, url = url,
                           database = database,
                           **opts)
Exemplo n.º 23
0
    def testL_CascadeCloseOutAnnnouncement(self):
        """
        _testL_CascadeCloseOutAnnouncement_

        Test the cascade closeout REST call, also
        check that when announced a request deletes all ACDC records in the system.
        """
        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")

        result = self.jsonSender.put("request", schema)[0]
        originalRequest = result['RequestName']
        self.setupACDCDatabase(originalRequest, "/%s/DataProcessing" % originalRequest,
                               result['Requestor'], result['Group'])
        depth = 2
        nReq = 3
        requests = [originalRequest]
        def createChildrenRequest(parentRequest, i, nReq):
            createdRequests = []
            resubSchema = utils.getResubmissionSchema(parentRequest, "/%s/DataProcessing" % parentRequest,
                                                      groupName, userName)
            result = self.jsonSender.put("request", resubSchema)[0]
            requestName = result['RequestName']
            self.setupACDCDatabase(requestName, "/%s/DataProcessing" % requestName, result['Requestor'], result['Group'])
            createdRequests.append(requestName)
            if i:
                for _ in range(nReq):
                    createdRequests.extend(createChildrenRequest(requestName, i - 1, nReq))
            return createdRequests
        requests.extend(createChildrenRequest(originalRequest, depth, nReq))
        for request in requests:
            self.changeStatusAndCheck(request, 'assignment-approved')
        for request in requests:
            self.jsonSender.put("assignment?team=%s&requestName=%s" % (teamName, request))
        for status in ['acquired',
                       'running-open', 'running-closed',
                       'completed']:
            for request in requests:
                self.changeStatusAndCheck(request, status)
        self.jsonSender.post('closeout?requestName=%s&cascade=True' % originalRequest)
        svc = CouchService(url = self.testInit.couchUrl,
                                  database = "%s_acdc" % self.couchDBName)

        owner = svc.newOwner(groupName, userName)
        for request in requests:
            result = self.jsonSender.get('request/%s' % request)
            self.assertEqual(result[0]['RequestStatus'], 'closed-out')
            testCollection = CouchCollection(database = self.testInit.couchDbName,
                                             url = self.testInit.couchUrl,
                                             name = request)
            testCollection.setOwner(owner)
            testCollection.populate()
            self.assertNotEqual(len(testCollection["filesets"]), 0)

        self.jsonSender.post('announce?requestName=%s&cascade=True' % originalRequest)
        for request in requests:

            result = self.jsonSender.get('request/%s' % request)
            self.assertEqual(result[0]['RequestStatus'], 'announced')
            testCollection = CouchCollection(database = self.testInit.couchDbName,
                                             url = self.testInit.couchUrl,
                                             name = request)
            testCollection.setOwner(owner)
            testCollection.populate()
            self.assertEqual(len(testCollection["filesets"]), 0)